diff --git a/README.md b/README.md index 42a5636..ad08ed2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Binary Data Format (or BDF) is designed to store data in a tree-like binary structure, like Notch's NBT format, but also open source and free like JSON. The format is -fast and allows multiple data types, it uses 32-bit integers, so BDF files can +fast and allows multiple data types. It uses 32-bit integers, so BDF files can be fast and work well on 32-bit systems, but have a maximum size of 2 GB. BDF allows human readable serialization to see what is going on for debugging purposes, but it currently can't parse the human readable serialized string to an object. @@ -28,6 +28,7 @@ purposes, but it currently can't parse the human readable serialized string to a ### Data types +- Undefined - Boolean - Integer - Long @@ -35,20 +36,33 @@ purposes, but it currently can't parse the human readable serialized string to a - Byte - Double - Float +- Boolean Array +- Integer Array +- Long Array +- Short Array +- Byte Array +- Double Array +- Float Array - String - Array - Named List -- Empty ### Creating an object -You will need to create a new object to store any data, use a `BdfObject` instance. -You can input serialized data into `BdfObject` via a `BdfDatabase`. +You will need to generate a BdfObject to serialize anything, +this can be done by first generating a BdfReader, or generating +a new object via an existing BdfObject. ```java -// New BDF object -BdfObject bdf = new BdfObject(); +// Create a reader object +BdfReader reader = new BdfReader(); + +// Get the BdfObject instance +BdfObject bdf = reader.getObject(); + +// Generate another BdfObject instance +BdfObject bdf_new = bdf.newObject(); // Get an integer int v = bdf.getInteger(); @@ -56,6 +70,9 @@ int v = bdf.getInteger(); // Set an integer bdf.setInteger(5); +// Set a "smart" integer +bdf.setSmartInteger(53); + // Get the type of variable of the object int type = bdf.getType(); @@ -66,65 +83,58 @@ if(type == BdfTypes.INTEGER) } // Serialize the BDF object -byte[] data = bdf.serialize().getBytes(); - - +IBdfDatabase data = bdf.serialize(); // Load another BDF object with the serialized bytes BdfObject bdf2 = new BdfObject(new BdfDatabase(data)); ``` -A `BdfFileManager` -instance can be used in the same way as a `BdfObject`, but it also needs a String parameter -for the path of the file. The file can be written with `BdfFileManager.saveDatabase()`. -A `BdfFileManager` is an instance of `BdfObject`, a `BdfFileManager` can be casted to -a `BdfObject`. +A file manager instance can be used in the same way as a reader object, +but it also needs a String parameter for the path of the file. The file +manager instance also has the capacity to use compression (by default this +uses the GZIP compression algorithm). ```java -// Open a file -BdfFileManager bdf = new BdfFileManager("file.bdf"); +// Open a file with compression enabled +BdfFileManager reader = new BdfFileManager("file.bdf", true); // Save the database -bdf.saveDatabase(); +reader.saveDatabase(); -// The file can be casted to a BdfObject -BdfObject bdf2 = (BdfObject) bdf; +// The file can be casted to a BdfReader +BdfReader reader2 = (BdfReader) reader; -// Bdf -System.out.println(bdf instanceof BdfObject); // true +// Can be used just as any reader instance +BdfObject bdf = reader.getObject(); ``` ### Arrays -Arrays can be used to store lists of information, they hold `BdfObject`. -The array is called with `new BdfArray()`. It can hold information, get -the size of the array with `BdfArray.size()`, remove elements with -`BdfArray.remove(index)`, set indexes with `BdfArray.set(index, BdfObject)`, -and add elements with `BdfArray.add(BdfObject)`. Arrays also -have support for Iterators and are an instance of `Iterable`. +Arrays can be used to store lists of information, they hold instances of +BdfObject. Arrays have support for Iterators and are an instance of Iterable. ```java -// New BDF Object -BdfObject bdf = new BdfObject(); +BdfReader reader = new BdfReader(); +BdfObject bdf = reader.getObject(); -// New BDF Array -BdfArray array = new BdfArray(); +// Can be created from a bdf object +BdfArray array = bdf.newArray(); -// Size +// Get the length of an array int size = array.size(); -// Remove -array.remove(3); // Could be any number +// Remove any index from an array +array.remove(3); -// Set - Could be any number with any object -array.set(4, BdfObject.withString("A String")); +// Set an object to an index of an array +array.set(4, bdf.newObject().setString("A String")); -// Add - Could be any object -array.add(BdfObject.withByte(53)); +// Add an object to an array +array.add(bdf.newObject().setByte(53)); // Set the array to the bdf object bdf.setArray(array); @@ -139,23 +149,27 @@ for(BdfObject o : array) ### Named lists -Named lists can be used to store data under strings, +Named lists can be used to store data under ids/strings to be used like variables in a program. A named list -can be created with `new BdfNamedList()` and it -has the ability to set with `BdfNamedList.set(String, BdfObject)`, -remove with `BdfNamedList.remove(String)`, and check -for a key with `BdfNamedList.contains(String)`. It also has -features to get all the keys with `BdfNamedList.getKeys()`. -Named lists also have Iterator support and are an instance of -`Iterable`. +can be created similar to an array. ```java -// New bdf named list -BdfNamedList list = new BdfNamedList(); +BdfReader reader = new BdfReader(); +BdfObject bdf = new BdfObject(); -// Set an element with a value -list.set("key1", BdfObject.withInteger(5)); +// New named list +BdfNamedList nl = bdf.newNamedList(); + +// Set an element to the named list +nl.set("key1", bdf.newObject().setInteger(5)); + +// Use ids instead of strings for optimisation +// if set/get is being called multiple times +// on the same key. + +int key2 = nl.getKeyLocation("key2"); +nl.set(key2, bdf.newObject().setFloat(42.0F)); // Get an elements value int v = list.get("key1").getInteger(); @@ -164,125 +178,45 @@ int v = list.get("key1").getInteger(); boolean has_key = list.contains("key1"); // Get the lists keys -String[] keys = list.getKeys(); +int[] keys = list.getKeys(); // Iterate over the lists keys -for(String key : keys) +for(int key : keys) { - + // Get the keys name + String key_name = nl.getKeyName(key); } ``` -### Saving classes +### Further optimisations -Classes can be saved with `BdfClassManager` and by -implementing the `IBdfClassManager` interface, -adding 2 functions `BdfClassLoad` and `BdfClassSave`. -`BdfClassLoad` is for checking and loading data from -bdf into the classes variables, while `BdfClassSave` -is for packing pre-existing variables into bdf format. -A BdfClassManager can be used to pass the `IBdfClassManager` -interface into. - -A class with `IBdfClassManager` to save the data -could look like this: - -```java - -class HelloWorld implements IBdfClassManager -{ - int iterator = 0; - - @Override - public void BdfClassLoad(BdfObject bdf) - { - // Load scripts here - - // Get the named list - BdfNamedList nl = bdf.getNamedList(); - - // Set the iterator stored in bdf - int iterator = nl.get("iterator").getInteger(); - } - - @Override - public void BdfClassSave(BdfObject bdf) - { - // Save scripts here - - // Create a named list - BdfNamedList nl = new BdfNamedList(); - - // Set the iterator to the named list - nl.set("iterator", BdfObject.withInteger(iterator)); - - // Store the named list - bdf.setNamedList(nl); - } - - public void hello() - { - // Increase the iterator by 1 - iterator++; - - // Say "Hello, World! Script executed times!" - System.out.println("Hello, World! Script executed "+iterator+" times!"); - } - -} - -``` - -A script to manage this could look something like this: - -```java - -/* - Get a new BdfObject instance, it could be existing, - or from another file, BdfArray, or BdfNamedList instance. -*/ -BdfObject bdf = new BdfObject(); - -// Create the HelloWorld class -HelloWorld hello = new HelloWorld(); - -// Get a new BdfClassManager instance to deal with BDF data -BdfClassManager manager = new BdfClassManager(hello); - -// Give the manager an existing BdfObject instance -manager.setBdf(bdf); - -// Load the classes bdf data -manager.load(); - -// Call the hello world function -hello.hello(); - -// Save the classes bdf data -manager.save(); - -``` ### Implementation details -All integer data types used are signed and Big Endian. +All integer data types are in the Big Endian layout. -**Type (1 byte)** +**Flags (1 unsigned byte)** +This holds 3 values: +- Type (0-17) +- Size type (0-2) +- Parent payload (0-2) + +**Type** ``` -0: BOOLEAN (1 byte, 0x00 or 0x01) -1: INTEGER (4 bytes) -2: LONG (8 bytes) -3: SHORT (2 bytes) -4: BYTE (1 byte) -5: DOUBLE (8 bytes) -6: FLOAT (4 bytes) +0: UNDEFINED (0 bytes) -7: STRING -8: ARRAY -9: NAMED_LIST +1: BOOLEAN (1 byte, 0x00 or 0x01) +2: INTEGER (4 bytes) +3: LONG (8 bytes) +4: SHORT (2 bytes) +5: BYTE (1 byte) +6: DOUBLE (8 bytes) +7: FLOAT (4 bytes) -10: EMPTY (0 bytes) +8: STRING +9: ARRAY +10: NAMED_LIST 11: ARRAY_BOOLEAN 12: ARRAY_INTEGER @@ -291,18 +225,23 @@ All integer data types used are signed and Big Endian. 15: ARRAY_BYTE 16: ARRAY_DOUBLE 17: ARRAY_FLOAT + ``` +**Size Type** +This value holds info for how big the size of +the size of the payload is, in bytes. The purpose +of this is to reduce the size as much as possible +by throwing out unneccicary zeros. + **Object** -- Type (signed byte, 1 byte) +- Flags (unsigned byte, 1 byte) +- Size (variable length) - Payload (Any type, variable length) **NamedList** -- Key size (signed int, 4 bytes) -- Key (variable length) -- Payload size (signed int, 4 bytes) +- Key ID (variable length) - Payload (Object, variable length) **Array** -- Payload size (signed int, 4 bytes) - Payload (Object, variable length) diff --git a/data.hbdf b/data.hbdf new file mode 100644 index 0000000..bbdc4fb --- /dev/null +++ b/data.hbdf @@ -0,0 +1,22 @@ +{ + "name": "Josua", + "age": 17I, + "anArray": [ + "hi =)", + 69B + ], + "anIntArray": int( + 432I, + 234I, + 69I, + 2I, + 423I + ), + "array2": [ + bool( + false, true , + true , true , + true , false, + ) + ] +} diff --git a/src/bdf/data/BdfDatabase.java b/src/bdf/data/BdfDatabase.java index 650dd02..7648f7a 100644 --- a/src/bdf/data/BdfDatabase.java +++ b/src/bdf/data/BdfDatabase.java @@ -5,16 +5,12 @@ import java.io.OutputStream; public class BdfDatabase implements IBdfDatabase { - private static final int STREAM_CHUNK_SIZE = 1024*1024; + static final int STREAM_CHUNK_SIZE = 1024*1024; - private byte[] database; - private int location; - private int size; + byte[] database; public BdfDatabase(byte[] bytes) { database = bytes; - size = bytes.length; - location = 0; } public BdfDatabase(String str) { @@ -23,28 +19,28 @@ public class BdfDatabase implements IBdfDatabase public BdfDatabase(int size) { this.database = new byte[size]; - this.location = 0; - this.size = size; - } - - private BdfDatabase() { } @Override - public IBdfDatabase getAt(int start, int end) + public IBdfDatabase getCopy(int start, int end) { byte[] database = new byte[end - start]; for(int i=0;i= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(); + + if(!(c == '\n' || c == '\t' || c == ' ')) { + return; + } + + increment(); + } + } + + // In the format "abc\n\t\u0003..." + public String getQuotedString() + { + if(getChar() != '"') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, this); + } + + increment(); + String str = ""; + + while(true) + { + if(offset >= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(); + + // Check for back slashes + if(c == '\\') + { + increment(1); + c = getChar(); + + switch(c) + { + case 'n': + str += "\n"; + break; + case 't': + str += "\t"; + break; + case '"': + str += "\""; + break; + case '\\': + str += "\\"; + break; + case 'u': // \u0000 + { + if(offset + 5 >= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, getPointer(1)); + } + + char[] hex = getCharArray(1, 4); + char unicode = (char)0; + int m = 1; + + for(int j=hex.length-1;j>=0;j--) + { + c = hex[j]; + + if(c >= '0' && c <= '9') { + unicode += (char)(m * (c - '0')); + } + + else if(c >= 'a' && c <= 'f') { + unicode += (char)(m * (c - 'a' + 10)); + } + + else { + throw BdfError.createError(BdfError.ERROR_SYNTAX, getPointer(1 + (hex.length-j-1))); + } + + m *= 16; + } + + str += unicode; + increment(5); + + break; + } + default: + str += "\\" + c; + } + } + + else if(c == '"') { + increment(); + break; + } + + else { + increment(); + str += c; + } + } + + return str; + } + + public boolean isNext(String check) + { + if(check.length() + offset >= data.length) { + return false; + } + + for(int i=0;i= data.length) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, this); + } + + char c = getChar(i); + c = (char)((c >= 'A' && c <= 'Z') ? (c + 32) : c); + + if(c != check.charAt(i)) { + return false; + } + } + + increment(check.length()); + + return true; + } + + public boolean isInteger() + { + for(int i=offset;i= '0' && c <= '9') { + continue; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, new BdfStringPointer(data, i)); + } + + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, new BdfStringPointer(data, data.length - 1)); + } +} diff --git a/src/bdf/data/IBdfDatabase.java b/src/bdf/data/IBdfDatabase.java index 30cd911..8ca3658 100644 --- a/src/bdf/data/IBdfDatabase.java +++ b/src/bdf/data/IBdfDatabase.java @@ -5,7 +5,7 @@ import java.io.OutputStream; public interface IBdfDatabase { - public IBdfDatabase getAt(int start, int end); + public IBdfDatabase getCopy(int start, int end); public IBdfDatabase getPointer(int location, int size); public IBdfDatabase getPointer(int location); @@ -17,9 +17,18 @@ public interface IBdfDatabase public byte[] getBytes(); public byte[] getBytes(int start, int size); + public byte getByte(); public byte getByte(int i); public String getString(); - public void setBytes(int pos, byte[] bytes); + public void setBytes(byte[] bytes, int offset, int length); + public void setBytes(byte[] bytes, int offset); + public void setBytes(byte[] bytes); + + public void setBytes(IBdfDatabase bytes, int offset, int length); + public void setBytes(IBdfDatabase bytes, int offset); + public void setBytes(IBdfDatabase bytes); + public void setByte(int pos, byte b); + public void setByte(byte b); } diff --git a/src/bdf/types/BdfArray.java b/src/bdf/types/BdfArray.java index aa9b645..35d53f4 100644 --- a/src/bdf/types/BdfArray.java +++ b/src/bdf/types/BdfArray.java @@ -5,21 +5,68 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Iterator; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; +import bdf.util.BdfError; import bdf.util.DataHelpers; public class BdfArray implements IBdfType, Iterable { - protected ArrayList elements = new ArrayList(); + protected ArrayList elements; protected BdfLookupTable lookupTable; - BdfArray(BdfLookupTable lookupTable) { + BdfArray(BdfLookupTable lookupTable, BdfStringPointer ptr) + { this.lookupTable = lookupTable; + this.elements = new ArrayList(); + + ptr.increment(); + + // [..., ...] + while(true) + { + ptr.ignoreBlanks(); + + if(ptr.getChar() == ']') { + ptr.increment(); + return; + } + + add(new BdfObject(lookupTable, ptr)); + + // There should be a comma after this + ptr.ignoreBlanks(); + + char c = ptr.getChar(); + + if(c == ']') { + ptr.increment(); + return; + } + + if(c != ',') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + } + } + + BdfArray(BdfLookupTable lookupTable, int size) + { + this.lookupTable = lookupTable; + this.elements = new ArrayList(size); + + for(int i=0;i(); // Create an iterator value to loop over the data int i = 0; @@ -54,12 +101,12 @@ public class BdfArray implements IBdfType, Iterable } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) { int pos = 0; for(BdfObject o : elements) { - pos += o.serialize(database.getPointer(pos), locations); + pos += o.serialize(database.getPointer(pos), locations, map, (byte)0); } return pos; diff --git a/src/bdf/types/BdfLookupTable.java b/src/bdf/types/BdfLookupTable.java index ed60d51..add1dba 100644 --- a/src/bdf/types/BdfLookupTable.java +++ b/src/bdf/types/BdfLookupTable.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import bdf.data.IBdfDatabase; import bdf.util.DataHelpers; @@ -26,8 +28,8 @@ class BdfLookupTable implements IBdfType for(int i=0;i uses[loc_0]) + { + int v_l = locations[loc_0]; + locations[loc_0] = locations[loc_1]; + locations[loc_1] = v_l; + + int v_u = uses[loc_0]; + uses[loc_0] = uses[loc_1]; + uses[loc_1] = v_u; + + int v_m = map_copy[j]; + map_copy[j] = map_copy[j+1]; + map_copy[j+1] = v_m; + + changed = true; + } + } + + if(!changed) { + return map_copy; + } + } + + return map_copy; + } + + public int[] serializeGetLocations(int[] locations) + { + int[] uses = new int[keys.size()]; int next = 0; - reader.bdf.getLocationUses(locations); + reader.bdf.getLocationUses(uses); - for(int i=0;i 0) { + for(int i=0;i 0) { locations[i] = next; next += 1; } else { @@ -111,7 +154,18 @@ class BdfLookupTable implements IBdfType } } - return locations; + int[] map = new int[next]; + next = 0; + + for(int i=0;i elements = new ArrayList(); protected BdfLookupTable lookupTable; + BdfNamedList(BdfLookupTable lookupTable, BdfStringPointer ptr) + { + this.lookupTable = lookupTable; + ptr.increment(); + + // {"key": ..., "key2": ...} + while(true) + { + ptr.ignoreBlanks(); + + char c = ptr.getChar(); + + if(c == '}') { + ptr.increment(); + break; + } + + if(c != '"') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + String key = ptr.getQuotedString(); + + // There should be a colon after this + ptr.ignoreBlanks(); + if(ptr.getChar() != ':') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + + set(key, new BdfObject(lookupTable, ptr)); + + // There should be a comma after this + ptr.ignoreBlanks(); + + c = ptr.getChar(); + + if(c == '}') { + ptr.increment(); + return; + } + + if(c != ',') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + } + } + BdfNamedList(BdfLookupTable lookupTable) { this.lookupTable = lookupTable; } @@ -33,13 +89,45 @@ public class BdfNamedList implements IBdfType // Loop over the data while(i < data.size()) { - // Get the key - int key = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt(); - i += 4; - // Get the object - int object_size = BdfObject.getSize(data.getPointer(i)); + int key_size = 0; + IBdfDatabase flag_ptr = data.getPointer(i); + int object_size = BdfObject.getSize(flag_ptr); + byte key_size_bytes = BdfObject.getParentFlags(flag_ptr); BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size)); + i += object_size; + + switch(key_size_bytes) + { + case 2: + key_size = 1; + break; + case 1: + key_size = 2; + break; + case 0: + key_size = 4; + break; + } + + // Get the key + ByteBuffer key_buff = DataHelpers.getByteBuffer(data.getPointer(i, key_size)); + int key = 0; + + switch(key_size_bytes) + { + case 2: + key = 0xff & key_buff.get(); + break; + case 1: + key = 0xffff & key_buff.getShort(); + break; + case 0: + key = key_buff.getInt(); + break; + } + + i += key_size; // Create a new element and save some data to it Element element = new Element(); @@ -48,24 +136,41 @@ public class BdfNamedList implements IBdfType // Add the object to the elements list elements.add(element); - - // Increase the iterator by the amount of bytes - i += object_size; } } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte flags) { int pos = 0; for(Element o : elements) { - database.setBytes(pos, DataHelpers.serializeInt(locations[o.key])); + int location = locations[o.key]; + byte size_bytes_tag; + byte size_bytes; - int size = o.object.serialize(database.getPointer(pos + 4), locations); + if(location > 65535) { // >= 2 ^ 16 + size_bytes_tag = 0; + size_bytes = 4; + } else if(location > 255) { // >= 2 ^ 8 + size_bytes_tag = 1; + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes_tag = 2; + size_bytes = 1; + } - pos += size + 4; + int size = o.object.serialize(database.getPointer(pos), locations, map, size_bytes_tag); + int offset = pos + size; + + byte[] bytes = DataHelpers.serializeInt(location); + + for(int i=0;i 65535) { // >= 2 ^ 16 + size += 4; + } else if(location > 255) { // >= 2 ^ 8 + size += 2; + } else { // < 2 ^ 8 + size += 1; + } + size += o.object.serializeSeeker(locations); } @@ -231,14 +345,6 @@ public class BdfNamedList implements IBdfType return keys; } - public int getKeyLocation(String key) { - return lookupTable.getLocation(key.getBytes()); - } - - public String getKeyName(int key) { - return new String(lookupTable.getName(key)); - } - public boolean contains(String key) { return contains(lookupTable.getLocation(key.getBytes())); } diff --git a/src/bdf/types/BdfObject.java b/src/bdf/types/BdfObject.java index 0f6fc1e..473682b 100644 --- a/src/bdf/types/BdfObject.java +++ b/src/bdf/types/BdfObject.java @@ -6,8 +6,11 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import bdf.data.BdfDatabase; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; +import bdf.util.BdfError; import bdf.util.DataHelpers; +import tests.Tests; public class BdfObject implements IBdfType { @@ -15,6 +18,7 @@ public class BdfObject implements IBdfType protected Object object = null; protected byte type = BdfTypes.UNDEFINED; protected BdfLookupTable lookupTable; + protected int last_seek; BdfObject(BdfLookupTable lookupTable) { this.lookupTable = lookupTable; @@ -26,14 +30,16 @@ public class BdfObject implements IBdfType this.lookupTable = lookupTable; // Get the type and database values - byte flags = data.getByte(0); - - type = flags & 0x0f; + int flags = 0xff & data.getByte(0); + type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + int size_bytes = getSizeBytes(flags % 3); + database = data.getPointer(1); // Skip the size bytes if size is stored if(shouldStoreSize(type)) { - database = database.getPointer(4 - (flags & 0b00110000) >> 4); + database = database.getPointer(size_bytes); } // Set the object variable if there is an object specified @@ -46,26 +52,431 @@ public class BdfObject implements IBdfType } } + BdfObject(BdfLookupTable lookupTable, BdfStringPointer ptr) + { + this.lookupTable = lookupTable; + + char c = ptr.getChar(); + + if(c == '{') { + setNamedList(new BdfNamedList(lookupTable, ptr)); + return; + } + + if(c == '[') { + setArray(new BdfArray(lookupTable, ptr)); + return; + } + + if(c == '"') { + setString(ptr.getQuotedString()); + return; + } + + boolean isPrimitiveArray = false; + byte type = 0; + + if(ptr.isNext("int")) { + type = BdfTypes.ARRAY_INTEGER; + isPrimitiveArray = true; + } + + else if(ptr.isNext("long")) { + type = BdfTypes.ARRAY_LONG; + isPrimitiveArray = true; + } + + else if(ptr.isNext("byte")) { + type = BdfTypes.ARRAY_BYTE; + isPrimitiveArray = true; + } + + else if(ptr.isNext("short")) { + type = BdfTypes.ARRAY_SHORT; + isPrimitiveArray = true; + } + + else if(ptr.isNext("bool")) { + type = BdfTypes.ARRAY_BOOLEAN; + isPrimitiveArray = true; + } + + else if(ptr.isNext("double")) { + type = BdfTypes.ARRAY_DOUBLE; + isPrimitiveArray = true; + } + + else if(ptr.isNext("float")) { + type = BdfTypes.ARRAY_FLOAT; + isPrimitiveArray = true; + } + + // Deserialize a primitive array + if(isPrimitiveArray) + { + ptr.ignoreBlanks(); + + if(ptr.getChar() != '(') { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + ptr.increment(); + ptr.ignoreBlanks(); + + // Get the size of the array + int size = 0; + + // Get a copy of the pointer + BdfStringPointer ptr2 = ptr.getPointer(0); + + for(;;) + { + if(ptr2.isNext("true") || ptr2.isNext("false")) { + size += 1; + } + + else + { + for(;;) + { + c = ptr2.getChar(); + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + ptr2.increment(); + continue; + } + + if(c == 'B' || c == 'S' || c == 'I' || c == 'L' || c == 'D' || c == 'F') { + ptr2.increment(); + size += 1; + break; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr2); + } + } + + ptr2.ignoreBlanks(); + + if(ptr2.getChar() == ',') { + ptr2.increment(); + ptr2.ignoreBlanks(); + } + + if(ptr2.getChar() == ')') { + ptr2.increment(); + break; + } + } + + Object array = null; + + switch(type) + { + case BdfTypes.ARRAY_BOOLEAN: + array = new boolean[size]; + break; + case BdfTypes.ARRAY_BYTE: + array = new byte[size]; + break; + case BdfTypes.ARRAY_DOUBLE: + array = new double[size]; + break; + case BdfTypes.ARRAY_FLOAT: + array = new float[size]; + break; + case BdfTypes.ARRAY_INTEGER: + array = new int[size]; + break; + case BdfTypes.ARRAY_LONG: + array = new long[size]; + break; + case BdfTypes.ARRAY_SHORT: + array = new short[size]; + break; + } + + for(int i=0;;i++) + { + if(ptr.isNext("true")) + { + if(type != BdfTypes.ARRAY_BOOLEAN) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + boolean[] a = (boolean[]) array; + a[i] = true; + } + + else if(ptr.isNext("false")) + { + if(type != BdfTypes.ARRAY_BOOLEAN) { + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + boolean[] a = (boolean[]) array; + a[i] = false; + } + + else + { + // Parse a number + String number = ""; + + for(;;) + { + c = ptr.getChar(); + + if(ptr.getDataLocation() > ptr.getDataLength()) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); + } + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + ptr.increment(); + number += c; + continue; + } + + switch(c) + { + case 'D': + { + if(type != BdfTypes.ARRAY_DOUBLE) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + double[] a = (double[]) array; + a[i] = Double.parseDouble(number); + + ptr.increment(); + break; + } + + case 'F': + { + if(type != BdfTypes.ARRAY_FLOAT) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + float[] a = (float[]) array; + a[i] = Float.parseFloat(number); + + ptr.increment(); + break; + } + + case 'I': + { + if(type != BdfTypes.ARRAY_INTEGER) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + int[] a = (int[]) array; + a[i] = Integer.parseInt(number); + + ptr.increment(); + break; + } + + case 'L': + { + if(type != BdfTypes.ARRAY_LONG) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + long[] a = (long[]) array; + a[i] = Long.parseLong(number); + + ptr.increment(); + break; + } + + case 'S': + { + if(type != BdfTypes.ARRAY_SHORT) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + short[] a = (short[]) array; + a[i] = Short.parseShort(number); + + ptr.increment(); + break; + } + + case 'B': + { + if(type != BdfTypes.ARRAY_BYTE) + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + + byte[] a = (byte[]) array; + a[i] = Byte.parseByte(number); + + ptr.increment(); + break; + } + + default: + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + + break; + } + } + + // int (420I, 23I ) + + ptr.ignoreBlanks(); + + if(ptr.getChar() == ',') { + ptr.increment(); + ptr.ignoreBlanks(); + } + + if(ptr.getChar() == ')') { + ptr.increment(); + break; + } + } + + switch(type) + { + case BdfTypes.ARRAY_BOOLEAN: + setBooleanArray((boolean[])array); + break; + case BdfTypes.ARRAY_BYTE: + setByteArray((byte[])array); + break; + case BdfTypes.ARRAY_DOUBLE: + setDoubleArray((double[])array); + break; + case BdfTypes.ARRAY_FLOAT: + setFloatArray((float[])array); + break; + case BdfTypes.ARRAY_INTEGER: + setIntegerArray((int[])array); + break; + case BdfTypes.ARRAY_LONG: + setLongArray((long[])array); + break; + case BdfTypes.ARRAY_SHORT: + setShortArray((short[])array); + break; + } + + return; + } + + if(ptr.isNext("true")) { + setBoolean(true); + return; + } + + if(ptr.isNext("false")) { + setBoolean(false); + return; + } + + if(ptr.isNext("undefined")) { + return; + } + + // Parse a number + String number = ""; + + for(;;) + { + c = ptr.getChar(); + ptr.increment(); + + if(ptr.getDataLocation() > ptr.getDataLength()) { + throw BdfError.createError(BdfError.ERROR_END_OF_FILE, ptr); + } + + if(c >= '0' && c <= '9' || c == '.' || c == 'e' || c == 'E' || c == '-') { + number += c; + continue; + } + + switch(c) + { + case 'D': + setDouble(Double.parseDouble(number)); + return; + case 'F': + setFloat(Float.parseFloat(number)); + return; + case 'B': + setByte(Byte.parseByte(number)); + return; + case 'S': + setShort(Short.parseShort(number)); + return; + case 'I': + setInteger(Integer.parseInt(number)); + return; + case 'L': + setLong(Long.parseLong(number)); + return; + } + + throw BdfError.createError(BdfError.ERROR_SYNTAX, ptr); + } + } + private boolean shouldStoreSize(byte b) { return b > 7; } + static private int getSizeBytes(int size_bytes_tag) + { + switch(size_bytes_tag) + { + case 0: return 4; + case 1: return 2; + case 2: return 1; + default: return 4; + } + } + + static byte getParentFlags(IBdfDatabase db) + { + int flags = 0xff & db.getByte(0); + + byte type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + + byte size_bytes = (byte)(flags % 3); + flags = (byte)((flags - size_bytes) / 3); + + byte parent_flags = (byte)(flags % 3); + flags = (byte)((flags - parent_flags) / 3); + + return parent_flags; + } + static int getSize(IBdfDatabase db) { - byte type = db.getByte(0); + int flags = 0xff & db.getByte(0); + byte type = (byte)(flags % 18); + flags = (byte)((flags - type) / 18); + + int size_bytes = getSizeBytes(flags % 3); int size = getSize(type); if(size != -1) { return size; } - return DataHelpers.getByteBuffer(db.getPointer(1, 4)).getInt() + 5; + ByteBuffer size_buff = DataHelpers.getByteBuffer(db.getPointer(1, size_bytes)); + + switch(size_bytes) + { + case 4: return size_buff.getInt(); + case 2: return (0xffff & size_buff.getShort()); + case 1: return (0xff & size_buff.get()); + } + + return 0; } static int getSize(byte type) { - type &= 0x0f; - switch(type) { case BdfTypes.BOOLEAN: @@ -98,40 +509,63 @@ public class BdfObject implements IBdfType } @Override - public int serialize(IBdfDatabase database, int[] locations) + public int serialize(IBdfDatabase database, int[] locations, int[] map, byte parent_flags) { - int size; - + int size = last_seek; boolean storeSize = shouldStoreSize(type); + byte size_bytes_tag = 0; + int size_bytes = 0; + + if(storeSize) + { + if(size > 65535) { // >= 2 ^ 16 + size_bytes_tag = 0; + size_bytes = 4; + } else if(size > 255) { // >= 2 ^ 8 + size_bytes_tag = 1; + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes_tag = 2; + size_bytes = 1; + } + } + + int offset = size_bytes + 1; + byte flags = (byte)(type + (size_bytes_tag * 18) + (parent_flags * 3 * 18)); + // Objects switch(type) { case BdfTypes.ARRAY: - size = ((BdfArray)object).serialize(database.getPointer(5), locations) + 5; + size = ((BdfArray)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; break; case BdfTypes.NAMED_LIST: - size = ((BdfNamedList)object).serialize(database.getPointer(5), locations) + 5; + size = ((BdfNamedList)object).serialize(database.getPointer(offset), locations, map, (byte)0) + offset; break; case BdfTypes.STRING: byte[] str = ((String)object).getBytes(); - size = str.length + 5; - database.setBytes(5, str); + size = str.length + offset; + database.setBytes(str, offset); break; default: - int o = storeSize ? 5 : 1; - size = this.database.size() + o; - database.setBytes(o, this.database.getBytes()); + size = this.database.size() + offset; + database.setBytes(this.database, offset); break; } - database.setByte(0, type); + database.setByte(0, flags); - if(storeSize) { - database.setBytes(1, DataHelpers.serializeInt(size - 5)); + if(storeSize) + { + byte[] bytes = DataHelpers.serializeInt(size); + + for(int i=0;i 65531) { // >= 2 ^ 16 + size_bytes = 4; + } else if(size > 253) { // >= 2 ^ 8 + size_bytes = 2; + } else { // < 2 ^ 8 + size_bytes = 1; + } + + size += size_bytes; + last_seek = size; + + return size; } private String calcIndent(BdfIndent indent, int it) { @@ -213,7 +669,7 @@ public class BdfObject implements IBdfType break; case BdfTypes.ARRAY_INTEGER: { - stream.write(("(" + calcIndent(indent, it)).getBytes()); + stream.write(("int(" + calcIndent(indent, it)).getBytes()); int[] array = this.getIntegerArray(); for(int i=0;i 2147483648L || number <= -2147483648L) { + setLong(number); + } else if(number > 32768 || number <= -32768) { + setInteger((int)number); + } else if(number > 128 || number <= -128) { + setShort((short)number); + } else { + setByte((byte)number); + } + + return this; + } + + public long getAutoInt() + { + switch(type) + { + case BdfTypes.BYTE: + return getByte(); + case BdfTypes.SHORT: + return getShort(); + case BdfTypes.INTEGER: + return getInteger(); + case BdfTypes.LONG: + return getLong(); + default: + return 0; + } } // Primitives @@ -601,7 +1085,19 @@ public class BdfObject implements IBdfType } public BdfArray newArray() { - return new BdfArray(lookupTable); + return new BdfArray(lookupTable, 0); + } + + public BdfArray newArray(int size) { + return new BdfArray(lookupTable, size); + } + + public int getKeyLocation(String key) { + return lookupTable.getLocation(key.getBytes()); + } + + public String getKeyName(int key) { + return new String(lookupTable.getName(key)); } public BdfObject setBooleanArray(boolean[] value) { diff --git a/src/bdf/types/BdfReader.java b/src/bdf/types/BdfReader.java index 606f8ee..04f1666 100644 --- a/src/bdf/types/BdfReader.java +++ b/src/bdf/types/BdfReader.java @@ -2,9 +2,12 @@ package bdf.types; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import bdf.data.BdfDatabase; +import bdf.data.BdfStringPointer; import bdf.data.IBdfDatabase; import bdf.util.DataHelpers; @@ -28,34 +31,100 @@ public class BdfReader public BdfReader(IBdfDatabase database) { - if(database.size() < 4) { + if(database.size() == 0) { initNew(); return; } - // Get the lookup table - int lookupTable_size = DataHelpers.getByteBuffer(database.getPointer(0, 4)).getInt(); - lookupTable = new BdfLookupTable(this, database.getPointer(4, lookupTable_size)); + int upto = 0; + + IBdfDatabase flag_ptr = database.getPointer(upto); + byte lookupTable_size_tag = BdfObject.getParentFlags(flag_ptr); + byte lookupTable_size_bytes = 0; + + switch(lookupTable_size_tag) + { + case 0: + lookupTable_size_bytes = 4; + break; + case 1: + lookupTable_size_bytes = 2; + break; + case 2: + lookupTable_size_bytes = 1; + break; + } // Get the rest of the data - int upto = lookupTable_size + 4; - int bdf_size = BdfObject.getSize(database.getPointer(upto)); - bdf = new BdfObject(lookupTable, database.getPointer(upto, bdf_size)); + int bdf_size = BdfObject.getSize(flag_ptr); + IBdfDatabase database_bdf = database.getPointer(upto, bdf_size); + upto += bdf_size; + + // Get the lookup table + ByteBuffer lookupTable_size_buff = DataHelpers.getByteBuffer(database.getPointer(upto, lookupTable_size_bytes)); + int lookupTable_size = 0; + + switch(lookupTable_size_tag) + { + case 0: + lookupTable_size = lookupTable_size_buff.getInt(); + break; + case 1: + lookupTable_size = 0xffff & lookupTable_size_buff.getShort(); + break; + case 2: + lookupTable_size = 0xff & lookupTable_size_buff.get(); + break; + } + + lookupTable = new BdfLookupTable(this, database.getPointer(lookupTable_size_bytes + upto, lookupTable_size)); + bdf = new BdfObject(lookupTable, database_bdf); + } + + public static BdfReader readHumanReadable(String data) + { + BdfReader reader = new BdfReader(); + reader.bdf = new BdfObject(reader.lookupTable, new BdfStringPointer(data.toCharArray(), 0)); + + return reader; } public BdfDatabase serialize() { - int[] locations = lookupTable.serializeGetLocations(); + int[] locations = new int[lookupTable.size()]; + int[] map = lookupTable.serializeGetLocations(locations); int bdf_size = bdf.serializeSeeker(locations); int lookupTable_size = lookupTable.serializeSeeker(locations); - int database_size = bdf_size + lookupTable_size + 4; + + int lookupTable_size_bytes = 0; + byte lookupTable_size_tag = 0; + + if(lookupTable_size > 65535) { // >= 2 ^ 16 + lookupTable_size_tag = 0; + lookupTable_size_bytes = 4; + } else if(lookupTable_size > 255) { // >= 2 ^ 8 + lookupTable_size_tag = 1; + lookupTable_size_bytes = 2; + } else { // < 2 ^ 8 + lookupTable_size_tag = 2; + lookupTable_size_bytes = 1; + } + + int upto = 0; + int database_size = bdf_size + lookupTable_size + lookupTable_size_bytes; BdfDatabase database = new BdfDatabase(database_size); - database.setBytes(0, DataHelpers.serializeInt(lookupTable_size)); + bdf.serialize(database.getPointer(upto, bdf_size), locations, map, lookupTable_size_tag); + upto += bdf_size; - lookupTable.serialize(database.getPointer(4, lookupTable_size), locations); - bdf.serialize(database.getPointer(4 + lookupTable_size, database_size), locations); + byte[] bytes = DataHelpers.serializeInt(lookupTable_size); + + for(int i=0;i 0x7e && c < 0xa1) || c == 0xad) + { + // Will be in the format \u0000 + size += 6; + } + + else + { + size += 1; + } + } + + char[] chars = new char[size]; + int upto = 0; + + for(int i=0;i 0x7e && c < 0xa1) || c == 0xad) + { + // Will be in the format \u0000 + chars[upto] = '\\'; + chars[upto+1] = 'u'; + chars[upto+2] = HEX[(c & 0xf000) >> 12]; + chars[upto+3] = HEX[(c & 0x0f00) >> 8]; + chars[upto+4] = HEX[(c & 0x00f0) >> 4]; + chars[upto+5] = HEX[(c & 0x000f)]; + upto += 6; + } + + else + { + chars[upto] = string_chars[i]; + upto += 1; + } + } + + string_chars = chars; + } // Add quotes to the string - serialized = "\"" + serialized + "\""; + string = "\"" + new String(string_chars) + "\""; // Return the serialized string - return serialized; + return string; } } diff --git a/src/tests/Tests.java b/src/tests/Tests.java index 751e938..be7058f 100755 --- a/src/tests/Tests.java +++ b/src/tests/Tests.java @@ -1,19 +1,15 @@ package tests; -import java.io.IOException; - import bdf.data.IBdfDatabase; -import bdf.types.BdfIndent; -import bdf.types.BdfNamedList; -import bdf.types.BdfObject; -import bdf.types.BdfReader; public class Tests { - static void displayHex(IBdfDatabase db) + public static void displayHex(IBdfDatabase db) { char[] hex_chars = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + System.out.print("Size: " + db.size() + ", Hex: "); + for(int i=0;i