Improved efficiency of the bdf format by implementing a "pointer"

solution. Fixed the issue with very large files taking up too much
memory. Made the BdfDatabase object more useful in general.

.gitignore

@@ -1,2 +1,2 @@
 /bin/
-/db.bdf
+/*.bdf

src/bdf/data/BdfDatabase.java

@@ -1,62 +1,141 @@
 package bdf.data;
 
-import java.nio.charset.StandardCharsets;
+import java.io.IOException;
+import java.io.OutputStream;
 
-public class BdfDatabase
+public class BdfDatabase implements IBdfDatabase
 {
-	protected byte[] database = null;
+	private static final int STREAM_CHUNK_SIZE = 1024*1024;
 	
-	public BdfDatabase() {
+	private byte[] database;
-		this.database = new byte[0];
+	private int location;
+	private int size;
+	
+	public BdfDatabase(byte[] bytes) {
+		database = bytes;
+		size = bytes.length;
+		location = 0;
 	}
 	
-	public BdfDatabase(String database) {
+	public BdfDatabase(String str) {
-		this.database = database.getBytes();
+		this(str.getBytes());
 	}
 	
-	public BdfDatabase(byte ...database) {
+	public BdfDatabase(int size) {
-		this.database = database;
+		this.database = new byte[size];
+		this.location = 0;
+		this.size = size;
 	}
 	
-	public BdfDatabase getAt(int start, int end)
+	private BdfDatabase() {
+	}
+	
+	@Override
+	public IBdfDatabase getAt(int start, int end)
 	{
-		byte[] split = new byte[end - start];
+		byte[] database = new byte[end - start];
 		
-		for(int i=start;i<end;i++) {
+		for(int i=0;i<end-start;i++) {
-			split[i-start] = this.database[i];
+			database[i] = this.database[i + start + location];
 		}
 		
-		return new BdfDatabase(split);
+		return new BdfDatabase(database);
-	}
-	
-	public int length() {
-		return this.database.length;
-	}
-	
-	public byte[] getBytes() {
-		return this.database;
 	}
 	
+	@Override
 	public byte getByte(int i) {
-		return this.database[i];
+		return database[location + i];
 	}
 	
-	public String getString() {
+	@Override
-		return new String(this.database, StandardCharsets.UTF_8);
+	public byte[] getBytes(int start, int size)
-	}
-	
-	public static BdfDatabase add(BdfDatabase d1, BdfDatabase d2)
 	{
-		byte[] added = new byte[d1.length() + d2.length()];
+		byte[] database = new byte[size];
 		
-		for(int i=0;i<d1.length();i++) {
+		for(int i=0;i<size;i++) {
-			added[i] = d1.getByte(i);
+			database[i] = this.database[i + location + start];
 		}
 		
-		for(int i=0;i<d2.length();i++) {
+		return database;
-			added[d1.length()+i] = d2.getByte(i);
 	}
 	
-		return new BdfDatabase(added);
+	@Override
+	public byte[] getBytes() {
+		return getBytes(0, size);
+	}
+	
+	@Override
+	public IBdfDatabase getPointer(int location, int size)
+	{
+		BdfDatabase db = new BdfDatabase();
+		
+		db.database = database;
+		db.location = this.location + location;
+		db.size = size;
+		
+		return db;
+	}
+	
+	@Override
+	public IBdfDatabase getPointer(int location) {
+		return getPointer(location, size - location);
+	}
+	
+	@Override
+	public int size() {
+		return size;
+	}
+	
+	@Override
+	public String getString() {
+		return new String(database, location, size);
+	}
+	
+	@Override
+	public void writeToStream(OutputStream stream, int start, int size) throws IOException
+	{
+		for(int i=0;i<size;i+=STREAM_CHUNK_SIZE)
+		{
+			if(size - i < STREAM_CHUNK_SIZE) {
+				stream.write(getBytes(i + start, size - i));
+			} else {
+				stream.write(getBytes(i + start, STREAM_CHUNK_SIZE));
+			}
+		}
+	}
+	
+	@Override
+	public void writeToStream(OutputStream stream) throws IOException {
+		writeToStream(stream, 0, size);
+	}
+	
+	public static BdfDatabase add(IBdfDatabase b1, IBdfDatabase b2)
+	{
+		byte[] bytes = new byte[b1.size() + b2.size()];
+		int b1_size = b1.size();
+		
+		for(int i=0;i<bytes.length;i++)
+		{
+			if(i >= b1_size) {
+				bytes[i] = b2.getByte(i - b1_size);
+			} else {
+				bytes[i] = b1.getByte(i);
+			}
+		}
+		
+		return new BdfDatabase(bytes);
+	}
+	
+	@Override
+	public void setByte(int pos, byte b) {
+		database[pos + location] = b;
+	}
+	
+	@Override
+	public void setBytes(int pos, byte[] bytes)
+	{
+		for(int i=0;i<bytes.length;i++) {
+			database[pos + location + i] = bytes[i];
+		}
 	}
 }

src/bdf/data/IBdfDatabase.java

@@ -0,0 +1,25 @@
+package bdf.data;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public interface IBdfDatabase
+{
+	public IBdfDatabase getAt(int start, int end);
+	public IBdfDatabase getPointer(int location, int size);
+	public IBdfDatabase getPointer(int location);
+	
+	public void writeToStream(OutputStream stream, int start, int size) throws IOException;
+	public void writeToStream(OutputStream stream) throws IOException;
+	
+	public int size();
+	
+	public byte[] getBytes();
+	public byte[] getBytes(int start, int size);
+	
+	public byte getByte(int i);
+	public String getString();
+	
+	public void setBytes(int pos, byte[] bytes);
+	public void setByte(int pos, byte b);
+}

src/bdf/file/BdfCompressedFileManager.java

@@ -28,7 +28,7 @@ protected String path;
 		}
 		
 		// Return an empty database if there is no read access
-		return new BdfDatabase();
+		return new BdfDatabase(0);
 	}
 	
 	public BdfCompressedFileManager(String path) {

src/bdf/file/BdfFileManager.java

@@ -26,7 +26,7 @@ public class BdfFileManager extends BdfObject
 		}
 		
 		// Return an empty database if there is no read access
-		return new BdfDatabase();
+		return new BdfDatabase(0);
 	}
 	
 	public BdfFileManager(String path) {

src/bdf/types/BdfArray.java

@@ -3,7 +3,7 @@ package bdf.types;
 import java.util.ArrayList;
 import java.util.Iterator;
 
-import bdf.data.BdfDatabase;
+import bdf.data.IBdfDatabase;
 import bdf.util.DataHelpers;
 
 public class BdfArray implements IBdfType, Iterable<BdfObject>
@@ -13,47 +13,56 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
 	public BdfArray() {
 	}
 	
-	public BdfArray(BdfDatabase data)
+	public BdfArray(IBdfDatabase data)
 	{
 		// Create an iterator value to loop over the data
 		int i = 0;
 		
 		// Loop over the data
-		while(i < data.length())
+		while(i < data.size())
 		{
 			// Get the size of the object
-			int size = DataHelpers.getByteBuffer(data.getAt(i, (i+(Integer.SIZE/8)))).getInt();
+			int size = DataHelpers.getByteBuffer(data.getPointer(i, Integer.BYTES)).getInt();
 			
 			// Get the object
-			BdfObject object = new BdfObject(data.getAt((i+(Integer.SIZE/8)), (i+(Integer.SIZE/8)+size)));
+			BdfObject object = new BdfObject(data.getPointer((i+Integer.BYTES), size));
 			
 			// Add the object to the elements list
 			elements.add(object);
 			
 			// Increase the iterator by the amount of bytes
-			i += (Integer.SIZE/8)+size;
+			i += Integer.BYTES+size;
 		}
 	}
 	
 	@Override
-	public BdfDatabase serialize()
+	public int serializeSeeker()
 	{
-		// Create the serialized data string
+		int size = 0;
-		BdfDatabase serialized = new BdfDatabase();
 		
-		// Loop over the elements
+		for(BdfObject o : elements) {
-		for(BdfObject o : elements)
+			size += o.serializeSeeker();
-		{
+			size += 4;
-			// Convert the object to a string
-			BdfDatabase db = o.serialize();
-			
-			// Add the serialized object to the serialized data
-			serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(db.length()));
-			serialized = BdfDatabase.add(serialized, db);
 		}
 		
-		// Send back the serialized data
+		return size;
-		return serialized;
+	}
+	
+	@Override
+	public int serialize(IBdfDatabase database)
+	{
+		int pos = 0;
+		
+		for(BdfObject o : elements)
+		{
+			int size = o.serialize(database.getPointer(pos + 4));
+			database.setBytes(pos, DataHelpers.serializeInt(size));
+			
+			pos += size;
+			pos += 4;
+		}
+		
+		return pos;
 	}
 	
 	@Override
@@ -108,12 +117,16 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
 		return this;
 	}
 	
-	public BdfArray remove(int index)
+	public BdfArray remove(int index) {
-	{
 		elements.remove(index);
 		return this;
 	}
 	
+	public BdfArray remove(BdfObject bdf) {
+		elements.remove(bdf);
+		return this;
+	}
+	
 	public BdfObject get(int index) {
 		return elements.get(index);
 	}

src/bdf/types/BdfNamedList.java

@@ -3,7 +3,7 @@ package bdf.types;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 
-import bdf.data.BdfDatabase;
+import bdf.data.IBdfDatabase;
 import bdf.util.DataHelpers;
 
 public class BdfNamedList implements IBdfType
@@ -19,24 +19,24 @@ public class BdfNamedList implements IBdfType
 	public BdfNamedList() {
 	}
 
-	public BdfNamedList(BdfDatabase data)
+	public BdfNamedList(IBdfDatabase data)
 	{
 		// Create an iterator value to loop over the data
 		int i = 0;
 		
 		// Loop over the data
-		while(i < data.length())
+		while(i < data.size())
 		{
 			// Get the key
-			int key_size = DataHelpers.getByteBuffer(data.getAt(i, i+(Integer.SIZE/8))).getInt();
+			int key_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
-			i += (Integer.SIZE/8);
+			i += 4;
-			byte[] key = data.getAt(i, i+key_size).getBytes();
+			byte[] key = data.getPointer(i, key_size).getBytes();
 			
 			// Get the object
 			i += key_size;
-			int object_size = DataHelpers.getByteBuffer(data.getAt(i, i+(Integer.SIZE/8))).getInt();
+			int object_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
-			i += (Integer.SIZE/8);
+			i += 4;
-			BdfObject object = new BdfObject(data.getAt(i, i+object_size));
+			BdfObject object = new BdfObject(data.getPointer(i, object_size));
 			
 			// Create a new element and save some data to it
 			Element element = new Element();
@@ -52,24 +52,44 @@ public class BdfNamedList implements IBdfType
 	}
 	
 	@Override
-	public BdfDatabase serialize()
+	public int serialize(IBdfDatabase database)
 	{
-		// Create the serialized data string
+		int pos = 0;
-		BdfDatabase serialized = new BdfDatabase();
 		
-		// Loop over the elements
 		for(Element o : elements)
 		{
-			// Add the serialized data to the data string
+			database.setBytes(pos, DataHelpers.serializeInt(o.key.length));
-			BdfDatabase data = o.object.serialize();
+			
-			serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(o.key.length));
+			pos += 4;
-			serialized = BdfDatabase.add(serialized, new BdfDatabase(o.key));
+			
-			serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(data.length()));
+			database.setBytes(pos, o.key);
-			serialized = BdfDatabase.add(serialized, data);
+			
+			pos += o.key.length;
+			
+			int size = o.object.serialize(database.getPointer(pos + 4, database.size() - (pos + 4)));
+			
+			database.setBytes(pos, DataHelpers.serializeInt(size));
+			
+			pos += 4;
+			pos += size;
 		}
 		
-		// Send back the serialized data
+		return pos;
-		return serialized;
+	}
+	
+	@Override
+	public int serializeSeeker()
+	{
+		int size = 0;
+		
+		for(Element o : elements)
+		{
+			size += 8;
+			size += o.key.length;
+			size += o.object.serializeSeeker();
+		}
+		
+		return size;
 	}
 	
 	@Override
@@ -168,6 +188,18 @@ public class BdfNamedList implements IBdfType
 		return this;
 	}
 	
+	public BdfNamedList remove(BdfObject bdf)
+	{
+		for(int i=0;i<elements.size();i++) {
+			if(elements.get(i).object == bdf) {
+				elements.remove(i);
+				i -= 1;
+			}
+		}
+		
+		return this;
+	}
+	
 	public BdfNamedList set(String key, BdfObject object)
 	{
 		// Convert the key to bytes

src/bdf/types/BdfObject.java

@@ -3,11 +3,12 @@ package bdf.types;
 import java.nio.ByteBuffer;
 
 import bdf.data.BdfDatabase;
+import bdf.data.IBdfDatabase;
 import bdf.util.DataHelpers;
 
 public class BdfObject implements IBdfType
 {
-	protected BdfDatabase database = null;
+	protected IBdfDatabase database = null;
 	protected Object object = null;
 	protected byte type = BdfTypes.EMPTY;
 	
@@ -15,36 +16,93 @@ public class BdfObject implements IBdfType
 		return new BdfObject();
 	}
 	
-	public BdfObject(BdfDatabase data)
+	public BdfObject(byte[] data) {
+		this(new BdfDatabase(data));
+	}
+	
+	public BdfObject(IBdfDatabase data)
 	{
 		// Is the database length greater than 1
-		if(data.length() > 1)
+		if(data.size() > 1)
 		{
 			// Get the type and database values
-			type = data.getAt(0, 1).getByte(0);
+			type = data.getByte(0);
-			database = data.getAt(1, data.length());
+			database = data.getPointer(1, data.size() - 1);
 			
 			// Set the object variable if there is an object specified
 			if(type == BdfTypes.STRING) object = database.getString();
 			if(type == BdfTypes.ARRAY) object = new BdfArray(database);
 			if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(database);
+			
+			if(object != null) {
+				database = null;
+			}
 		}
 		
 		else
 		{
 			// Create a new database
-			database = new BdfDatabase();
+			database = new BdfDatabase(0);
 		}
 	}
 	
 	@Override
+	public int serialize(IBdfDatabase database)
+	{
+		int size;
+		
+		IBdfDatabase db = database.getPointer(1);
+		
+		// Objects
+		switch(type)
+		{
+		case BdfTypes.ARRAY:
+			size = ((BdfArray)object).serialize(db) + 1;
+			break;
+			
+		case BdfTypes.NAMED_LIST:
+			size = ((BdfNamedList)object).serialize(db) + 1;
+			break;
+			
+		case BdfTypes.STRING:
+			String str = (String)object;
+			size = str.length() + 1;
+			db.setBytes(0, str.getBytes());
+			break;
+			
+		default:
+			size = this.database.size() + 1;
+			db.setBytes(0, this.database.getBytes());
+			break;
+		}
+		
+		database.setByte(0, type);
+		
+		return size;
+	}
+
+	@Override
+	public int serializeSeeker()
+	{
+		// Objects
+		switch(type)
+		{
+		case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker() + 1;
+		case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker() + 1;
+		case BdfTypes.STRING: return ((String)object).length() + 1;
+		}
+		
+		// Anything else
+		return database.size() + 1;
+	}
+	
 	public BdfDatabase serialize()
 	{
-		if(type == BdfTypes.STRING) database = new BdfDatabase((String)object);
+		BdfDatabase database = new BdfDatabase(serializeSeeker());
-		if(type == BdfTypes.ARRAY) database = ((BdfArray)object).serialize();
-		if(type == BdfTypes.NAMED_LIST) database = ((BdfNamedList)object).serialize();
 		
-		return BdfDatabase.add(new BdfDatabase(type), database);
+		serialize(database);
+		
+		return database;
 	}
 	
 	private String calcIndent(BdfIndent indent, int it) {
@@ -152,7 +210,7 @@ public class BdfObject implements IBdfType
 	}
 	
 	public BdfObject() {
-		database = new BdfDatabase();
+		database = new BdfDatabase(0);
 	}
 	
 	public byte getType() {
@@ -378,7 +436,7 @@ public class BdfObject implements IBdfType
 	
 	public BdfObject setInteger(int value) {
 		this.type = BdfTypes.INTEGER;
-		ByteBuffer b = ByteBuffer.allocate(Integer.SIZE/8);
+		ByteBuffer b = ByteBuffer.allocate(Integer.BYTES);
 		b.putInt(0, value);
 		database = DataHelpers.getDatabase(b);
 		return this;

src/bdf/types/IBdfType.java

@@ -1,10 +1,12 @@
 package bdf.types;
 
-import bdf.data.BdfDatabase;
+import bdf.data.IBdfDatabase;
 
-public interface IBdfType
+interface IBdfType
 {
-	public BdfDatabase serialize();
+	int serialize(IBdfDatabase database);
+	int serializeSeeker();
+	
 	String serializeHumanReadable(BdfIndent indent, int it);
 
 	public default String serializeHumanReadable(BdfIndent indent) {

src/bdf/util/DataHelpers.java

@@ -3,10 +3,11 @@ package bdf.util;
 import java.nio.ByteBuffer;
 
 import bdf.data.BdfDatabase;
+import bdf.data.IBdfDatabase;
 
 public class DataHelpers
 {
-	public static ByteBuffer getByteBuffer(BdfDatabase db) {
+	public static ByteBuffer getByteBuffer(IBdfDatabase db) {
 		return ByteBuffer.wrap(db.getBytes());
 	}
 	
@@ -14,11 +15,11 @@ public class DataHelpers
 		return new BdfDatabase(buffer.array());
 	}
 	
-	public static BdfDatabase serializeInt(int value)
+	public static byte[] serializeInt(int value)
 	{
 		ByteBuffer buffer = ByteBuffer.allocate(Integer.SIZE/8);
 		buffer.putInt(value);
-		return getDatabase(buffer);
+		return buffer.array();
 	}
 	
 	public static boolean bytesAreEqual(byte[] b1, byte[] b2)
@@ -74,7 +75,10 @@ public class DataHelpers
 		String serialized = string;
 		
 		// Replace some parts of the string
+		serialized = replaceInString(serialized, '\\', "\\\\");
 		serialized = replaceInString(serialized, '"', "\\\"");
+		serialized = replaceInString(serialized, '\n', "\\n");
+		serialized = replaceInString(serialized, '\t', "\\t");
 		
 		// Add quotes to the string
 		serialized = "\"" + serialized + "\"";

src/tests/TestClass.java

@@ -1,32 +0,0 @@
-package tests;
-
-import bdf.classes.IBdfClassManager;
-import bdf.types.BdfNamedList;
-import bdf.types.BdfObject;
-
-public class TestClass implements IBdfClassManager
-{
-	int i = 0;
-	
-	@Override
-	public void BdfClassLoad(BdfObject bdf)
-	{
-		BdfNamedList nl = bdf.getNamedList();
-		this.i = nl.get("i").getInteger();
-	}
-
-	@Override
-	public void BdfClassSave(BdfObject bdf)
-	{
-		BdfNamedList nl = new BdfNamedList();
-		nl.set("i", BdfObject.withInteger(i));
-		bdf.setNamedList(nl);
-	}
-
-	public void tick()
-	{
-		System.out.println(i);
-		i++;
-	}
-	
-}

src/tests/Tests.java

@@ -1,6 +1,12 @@
 package tests;
 
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import bdf.data.BdfDatabase;
 import bdf.file.BdfCompressedFileManager;
+import bdf.file.BdfFileManager;
 import bdf.types.BdfArray;
 import bdf.types.BdfIndent;
 import bdf.types.BdfNamedList;
@@ -8,31 +14,55 @@ import bdf.types.BdfObject;
 
 public class Tests {
 
-	public static void main(String[] args)
+	public static void main(String[] args) throws InterruptedException, IOException
 	{
-		BdfCompressedFileManager bdf = new BdfCompressedFileManager("./db.bdf");
+		/*
-		
+		BdfObject bdf = new BdfObject();
 		BdfNamedList nl = bdf.getNamedList();
 		
-		int array[] = {1,2,3,6,7,0};
+		byte[] bytes = new byte[1024*1024*1024];
-		int array2[] = {1,2,3,6,7,0};
+		for(int i=0;i<bytes.length;i++) {
+			bytes[i] = (byte)0;
+		}
 		
-		BdfArray array_bdf = new BdfArray();
+		for(int i=0;i<1000;i++) {
+			nl = nl.get("next").getNamedList();
+		}
 		
-		array_bdf.add(BdfObject.withBoolean(true));
+		nl.get("next").setByteArray(bytes);
-		array_bdf.add(BdfObject.withBoolean(false));
-		array_bdf.add(BdfObject.withInteger(7));
-		array_bdf.add(BdfObject.withNamedList());
-		array_bdf.add(BdfObject.withArray());
-		array_bdf.add(BdfObject.withIntegerArray(array2));
 		
-		nl.set("it", BdfObject.withInteger(nl.get("it").getInteger() + 1));
+		BdfDatabase data = bdf.serialize();
-		nl.set("int_array", BdfObject.withIntegerArray(array));
-		nl.set("array", BdfObject.withArray(array_bdf));
 		
-		System.out.println(bdf.serializeHumanReadable(new BdfIndent("\t", "\n")));
+		FileOutputStream file = new FileOutputStream("./database.bdf");
+		data.writeToStream(file);
+		*/
 		
-		bdf.saveDatabase();
+		/*
+		BdfObject bdf = new BdfObject();
+		BdfArray a = bdf.getArray();
+		
+		byte[] bytes = new byte[1024*1024*1024];
+		for(int i=0;i<bytes.length;i++) {
+			bytes[i] = (byte)0;
+		}
+		
+		for(int i=0;i<1000;i++) {
+			BdfArray a2 = new BdfArray();
+			a.add(BdfObject.withArray(a2));
+			a = a2;
+		}
+		
+		a.add(BdfObject.withByteArray(bytes));
+		
+		BdfDatabase data = bdf.serialize();
+		
+		FileOutputStream file = new FileOutputStream("./database.bdf");
+		data.writeToStream(file);
+		 */
+		
+		BdfFileManager bdf = new BdfFileManager("./database.bdf");
+		System.out.println("Loaded bdf");
+		Thread.sleep(5000);
 	}
 
 }