Improved efficiency of the bdf format by implementing a "pointer"

solution. Fixed the issue with very large files taking up too much
memory. Made the BdfDatabase object more useful in general.
This commit is contained in:
josua 2020-06-18 22:54:27 +10:00
parent 03e4b583f4
commit 39de6a5df4
12 changed files with 359 additions and 148 deletions

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
/bin/
/db.bdf
/*.bdf

View File

@ -1,62 +1,141 @@
package bdf.data;
import java.nio.charset.StandardCharsets;
import java.io.IOException;
import java.io.OutputStream;
public class BdfDatabase
public class BdfDatabase implements IBdfDatabase
{
protected byte[] database = null;
private static final int STREAM_CHUNK_SIZE = 1024*1024;
public BdfDatabase() {
this.database = new byte[0];
private byte[] database;
private int location;
private int size;
public BdfDatabase(byte[] bytes) {
database = bytes;
size = bytes.length;
location = 0;
}
public BdfDatabase(String database) {
this.database = database.getBytes();
public BdfDatabase(String str) {
this(str.getBytes());
}
public BdfDatabase(byte ...database) {
this.database = database;
public BdfDatabase(int size) {
this.database = new byte[size];
this.location = 0;
this.size = size;
}
public BdfDatabase getAt(int start, int end)
private BdfDatabase() {
}
@Override
public IBdfDatabase getAt(int start, int end)
{
byte[] split = new byte[end - start];
byte[] database = new byte[end - start];
for(int i=start;i<end;i++) {
split[i-start] = this.database[i];
for(int i=0;i<end-start;i++) {
database[i] = this.database[i + start + location];
}
return new BdfDatabase(split);
}
public int length() {
return this.database.length;
}
public byte[] getBytes() {
return this.database;
return new BdfDatabase(database);
}
@Override
public byte getByte(int i) {
return this.database[i];
return database[location + i];
}
public String getString() {
return new String(this.database, StandardCharsets.UTF_8);
}
public static BdfDatabase add(BdfDatabase d1, BdfDatabase d2)
@Override
public byte[] getBytes(int start, int size)
{
byte[] added = new byte[d1.length() + d2.length()];
byte[] database = new byte[size];
for(int i=0;i<d1.length();i++) {
added[i] = d1.getByte(i);
for(int i=0;i<size;i++) {
database[i] = this.database[i + location + start];
}
for(int i=0;i<d2.length();i++) {
added[d1.length()+i] = d2.getByte(i);
return database;
}
@Override
public byte[] getBytes() {
return getBytes(0, size);
}
@Override
public IBdfDatabase getPointer(int location, int size)
{
BdfDatabase db = new BdfDatabase();
db.database = database;
db.location = this.location + location;
db.size = size;
return db;
}
@Override
public IBdfDatabase getPointer(int location) {
return getPointer(location, size - location);
}
@Override
public int size() {
return size;
}
@Override
public String getString() {
return new String(database, location, size);
}
@Override
public void writeToStream(OutputStream stream, int start, int size) throws IOException
{
for(int i=0;i<size;i+=STREAM_CHUNK_SIZE)
{
if(size - i < STREAM_CHUNK_SIZE) {
stream.write(getBytes(i + start, size - i));
} else {
stream.write(getBytes(i + start, STREAM_CHUNK_SIZE));
}
}
}
@Override
public void writeToStream(OutputStream stream) throws IOException {
writeToStream(stream, 0, size);
}
public static BdfDatabase add(IBdfDatabase b1, IBdfDatabase b2)
{
byte[] bytes = new byte[b1.size() + b2.size()];
int b1_size = b1.size();
for(int i=0;i<bytes.length;i++)
{
if(i >= b1_size) {
bytes[i] = b2.getByte(i - b1_size);
} else {
bytes[i] = b1.getByte(i);
}
}
return new BdfDatabase(added);
return new BdfDatabase(bytes);
}
@Override
public void setByte(int pos, byte b) {
database[pos + location] = b;
}
@Override
public void setBytes(int pos, byte[] bytes)
{
for(int i=0;i<bytes.length;i++) {
database[pos + location + i] = bytes[i];
}
}
}

View File

@ -0,0 +1,25 @@
package bdf.data;
import java.io.IOException;
import java.io.OutputStream;
public interface IBdfDatabase
{
public IBdfDatabase getAt(int start, int end);
public IBdfDatabase getPointer(int location, int size);
public IBdfDatabase getPointer(int location);
public void writeToStream(OutputStream stream, int start, int size) throws IOException;
public void writeToStream(OutputStream stream) throws IOException;
public int size();
public byte[] getBytes();
public byte[] getBytes(int start, int size);
public byte getByte(int i);
public String getString();
public void setBytes(int pos, byte[] bytes);
public void setByte(int pos, byte b);
}

View File

@ -28,7 +28,7 @@ protected String path;
}
// Return an empty database if there is no read access
return new BdfDatabase();
return new BdfDatabase(0);
}
public BdfCompressedFileManager(String path) {

View File

@ -26,7 +26,7 @@ public class BdfFileManager extends BdfObject
}
// Return an empty database if there is no read access
return new BdfDatabase();
return new BdfDatabase(0);
}
public BdfFileManager(String path) {

View File

@ -3,7 +3,7 @@ package bdf.types;
import java.util.ArrayList;
import java.util.Iterator;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
import bdf.util.DataHelpers;
public class BdfArray implements IBdfType, Iterable<BdfObject>
@ -13,47 +13,56 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
public BdfArray() {
}
public BdfArray(BdfDatabase data)
public BdfArray(IBdfDatabase data)
{
// Create an iterator value to loop over the data
int i = 0;
// Loop over the data
while(i < data.length())
while(i < data.size())
{
// Get the size of the object
int size = DataHelpers.getByteBuffer(data.getAt(i, (i+(Integer.SIZE/8)))).getInt();
int size = DataHelpers.getByteBuffer(data.getPointer(i, Integer.BYTES)).getInt();
// Get the object
BdfObject object = new BdfObject(data.getAt((i+(Integer.SIZE/8)), (i+(Integer.SIZE/8)+size)));
BdfObject object = new BdfObject(data.getPointer((i+Integer.BYTES), size));
// Add the object to the elements list
elements.add(object);
// Increase the iterator by the amount of bytes
i += (Integer.SIZE/8)+size;
i += Integer.BYTES+size;
}
}
@Override
public BdfDatabase serialize()
public int serializeSeeker()
{
// Create the serialized data string
BdfDatabase serialized = new BdfDatabase();
int size = 0;
// Loop over the elements
for(BdfObject o : elements)
{
// Convert the object to a string
BdfDatabase db = o.serialize();
// Add the serialized object to the serialized data
serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(db.length()));
serialized = BdfDatabase.add(serialized, db);
for(BdfObject o : elements) {
size += o.serializeSeeker();
size += 4;
}
// Send back the serialized data
return serialized;
return size;
}
@Override
public int serialize(IBdfDatabase database)
{
int pos = 0;
for(BdfObject o : elements)
{
int size = o.serialize(database.getPointer(pos + 4));
database.setBytes(pos, DataHelpers.serializeInt(size));
pos += size;
pos += 4;
}
return pos;
}
@Override
@ -108,12 +117,16 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
return this;
}
public BdfArray remove(int index)
{
public BdfArray remove(int index) {
elements.remove(index);
return this;
}
public BdfArray remove(BdfObject bdf) {
elements.remove(bdf);
return this;
}
public BdfObject get(int index) {
return elements.get(index);
}

View File

@ -3,7 +3,7 @@ package bdf.types;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
import bdf.util.DataHelpers;
public class BdfNamedList implements IBdfType
@ -19,24 +19,24 @@ public class BdfNamedList implements IBdfType
public BdfNamedList() {
}
public BdfNamedList(BdfDatabase data)
public BdfNamedList(IBdfDatabase data)
{
// Create an iterator value to loop over the data
int i = 0;
// Loop over the data
while(i < data.length())
while(i < data.size())
{
// Get the key
int key_size = DataHelpers.getByteBuffer(data.getAt(i, i+(Integer.SIZE/8))).getInt();
i += (Integer.SIZE/8);
byte[] key = data.getAt(i, i+key_size).getBytes();
int key_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
i += 4;
byte[] key = data.getPointer(i, key_size).getBytes();
// Get the object
i += key_size;
int object_size = DataHelpers.getByteBuffer(data.getAt(i, i+(Integer.SIZE/8))).getInt();
i += (Integer.SIZE/8);
BdfObject object = new BdfObject(data.getAt(i, i+object_size));
int object_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
i += 4;
BdfObject object = new BdfObject(data.getPointer(i, object_size));
// Create a new element and save some data to it
Element element = new Element();
@ -52,24 +52,44 @@ public class BdfNamedList implements IBdfType
}
@Override
public BdfDatabase serialize()
public int serialize(IBdfDatabase database)
{
// Create the serialized data string
BdfDatabase serialized = new BdfDatabase();
int pos = 0;
// Loop over the elements
for(Element o : elements)
{
// Add the serialized data to the data string
BdfDatabase data = o.object.serialize();
serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(o.key.length));
serialized = BdfDatabase.add(serialized, new BdfDatabase(o.key));
serialized = BdfDatabase.add(serialized, DataHelpers.serializeInt(data.length()));
serialized = BdfDatabase.add(serialized, data);
database.setBytes(pos, DataHelpers.serializeInt(o.key.length));
pos += 4;
database.setBytes(pos, o.key);
pos += o.key.length;
int size = o.object.serialize(database.getPointer(pos + 4, database.size() - (pos + 4)));
database.setBytes(pos, DataHelpers.serializeInt(size));
pos += 4;
pos += size;
}
// Send back the serialized data
return serialized;
return pos;
}
@Override
public int serializeSeeker()
{
int size = 0;
for(Element o : elements)
{
size += 8;
size += o.key.length;
size += o.object.serializeSeeker();
}
return size;
}
@Override
@ -168,6 +188,18 @@ public class BdfNamedList implements IBdfType
return this;
}
public BdfNamedList remove(BdfObject bdf)
{
for(int i=0;i<elements.size();i++) {
if(elements.get(i).object == bdf) {
elements.remove(i);
i -= 1;
}
}
return this;
}
public BdfNamedList set(String key, BdfObject object)
{
// Convert the key to bytes

View File

@ -3,11 +3,12 @@ package bdf.types;
import java.nio.ByteBuffer;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
import bdf.util.DataHelpers;
public class BdfObject implements IBdfType
{
protected BdfDatabase database = null;
protected IBdfDatabase database = null;
protected Object object = null;
protected byte type = BdfTypes.EMPTY;
@ -15,36 +16,93 @@ public class BdfObject implements IBdfType
return new BdfObject();
}
public BdfObject(BdfDatabase data)
public BdfObject(byte[] data) {
this(new BdfDatabase(data));
}
public BdfObject(IBdfDatabase data)
{
// Is the database length greater than 1
if(data.length() > 1)
if(data.size() > 1)
{
// Get the type and database values
type = data.getAt(0, 1).getByte(0);
database = data.getAt(1, data.length());
type = data.getByte(0);
database = data.getPointer(1, data.size() - 1);
// Set the object variable if there is an object specified
if(type == BdfTypes.STRING) object = database.getString();
if(type == BdfTypes.ARRAY) object = new BdfArray(database);
if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(database);
if(object != null) {
database = null;
}
}
else
{
// Create a new database
database = new BdfDatabase();
database = new BdfDatabase(0);
}
}
@Override
public int serialize(IBdfDatabase database)
{
int size;
IBdfDatabase db = database.getPointer(1);
// Objects
switch(type)
{
case BdfTypes.ARRAY:
size = ((BdfArray)object).serialize(db) + 1;
break;
case BdfTypes.NAMED_LIST:
size = ((BdfNamedList)object).serialize(db) + 1;
break;
case BdfTypes.STRING:
String str = (String)object;
size = str.length() + 1;
db.setBytes(0, str.getBytes());
break;
default:
size = this.database.size() + 1;
db.setBytes(0, this.database.getBytes());
break;
}
database.setByte(0, type);
return size;
}
@Override
public int serializeSeeker()
{
// Objects
switch(type)
{
case BdfTypes.ARRAY: return ((BdfArray)object).serializeSeeker() + 1;
case BdfTypes.NAMED_LIST: return ((BdfNamedList)object).serializeSeeker() + 1;
case BdfTypes.STRING: return ((String)object).length() + 1;
}
// Anything else
return database.size() + 1;
}
public BdfDatabase serialize()
{
if(type == BdfTypes.STRING) database = new BdfDatabase((String)object);
if(type == BdfTypes.ARRAY) database = ((BdfArray)object).serialize();
if(type == BdfTypes.NAMED_LIST) database = ((BdfNamedList)object).serialize();
BdfDatabase database = new BdfDatabase(serializeSeeker());
return BdfDatabase.add(new BdfDatabase(type), database);
serialize(database);
return database;
}
private String calcIndent(BdfIndent indent, int it) {
@ -152,7 +210,7 @@ public class BdfObject implements IBdfType
}
public BdfObject() {
database = new BdfDatabase();
database = new BdfDatabase(0);
}
public byte getType() {
@ -378,7 +436,7 @@ public class BdfObject implements IBdfType
public BdfObject setInteger(int value) {
this.type = BdfTypes.INTEGER;
ByteBuffer b = ByteBuffer.allocate(Integer.SIZE/8);
ByteBuffer b = ByteBuffer.allocate(Integer.BYTES);
b.putInt(0, value);
database = DataHelpers.getDatabase(b);
return this;

View File

@ -1,10 +1,12 @@
package bdf.types;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
public interface IBdfType
interface IBdfType
{
public BdfDatabase serialize();
int serialize(IBdfDatabase database);
int serializeSeeker();
String serializeHumanReadable(BdfIndent indent, int it);
public default String serializeHumanReadable(BdfIndent indent) {

View File

@ -3,10 +3,11 @@ package bdf.util;
import java.nio.ByteBuffer;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
public class DataHelpers
{
public static ByteBuffer getByteBuffer(BdfDatabase db) {
public static ByteBuffer getByteBuffer(IBdfDatabase db) {
return ByteBuffer.wrap(db.getBytes());
}
@ -14,11 +15,11 @@ public class DataHelpers
return new BdfDatabase(buffer.array());
}
public static BdfDatabase serializeInt(int value)
public static byte[] serializeInt(int value)
{
ByteBuffer buffer = ByteBuffer.allocate(Integer.SIZE/8);
buffer.putInt(value);
return getDatabase(buffer);
return buffer.array();
}
public static boolean bytesAreEqual(byte[] b1, byte[] b2)
@ -74,7 +75,10 @@ public class DataHelpers
String serialized = string;
// Replace some parts of the string
serialized = replaceInString(serialized, '\\', "\\\\");
serialized = replaceInString(serialized, '"', "\\\"");
serialized = replaceInString(serialized, '\n', "\\n");
serialized = replaceInString(serialized, '\t', "\\t");
// Add quotes to the string
serialized = "\"" + serialized + "\"";

View File

@ -1,32 +0,0 @@
package tests;
import bdf.classes.IBdfClassManager;
import bdf.types.BdfNamedList;
import bdf.types.BdfObject;
public class TestClass implements IBdfClassManager
{
int i = 0;
@Override
public void BdfClassLoad(BdfObject bdf)
{
BdfNamedList nl = bdf.getNamedList();
this.i = nl.get("i").getInteger();
}
@Override
public void BdfClassSave(BdfObject bdf)
{
BdfNamedList nl = new BdfNamedList();
nl.set("i", BdfObject.withInteger(i));
bdf.setNamedList(nl);
}
public void tick()
{
System.out.println(i);
i++;
}
}

View File

@ -1,6 +1,12 @@
package tests;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import bdf.data.BdfDatabase;
import bdf.file.BdfCompressedFileManager;
import bdf.file.BdfFileManager;
import bdf.types.BdfArray;
import bdf.types.BdfIndent;
import bdf.types.BdfNamedList;
@ -8,31 +14,55 @@ import bdf.types.BdfObject;
public class Tests {
public static void main(String[] args)
public static void main(String[] args) throws InterruptedException, IOException
{
BdfCompressedFileManager bdf = new BdfCompressedFileManager("./db.bdf");
/*
BdfObject bdf = new BdfObject();
BdfNamedList nl = bdf.getNamedList();
int array[] = {1,2,3,6,7,0};
int array2[] = {1,2,3,6,7,0};
byte[] bytes = new byte[1024*1024*1024];
for(int i=0;i<bytes.length;i++) {
bytes[i] = (byte)0;
}
BdfArray array_bdf = new BdfArray();
for(int i=0;i<1000;i++) {
nl = nl.get("next").getNamedList();
}
array_bdf.add(BdfObject.withBoolean(true));
array_bdf.add(BdfObject.withBoolean(false));
array_bdf.add(BdfObject.withInteger(7));
array_bdf.add(BdfObject.withNamedList());
array_bdf.add(BdfObject.withArray());
array_bdf.add(BdfObject.withIntegerArray(array2));
nl.get("next").setByteArray(bytes);
nl.set("it", BdfObject.withInteger(nl.get("it").getInteger() + 1));
nl.set("int_array", BdfObject.withIntegerArray(array));
nl.set("array", BdfObject.withArray(array_bdf));
BdfDatabase data = bdf.serialize();
System.out.println(bdf.serializeHumanReadable(new BdfIndent("\t", "\n")));
FileOutputStream file = new FileOutputStream("./database.bdf");
data.writeToStream(file);
*/
bdf.saveDatabase();
/*
BdfObject bdf = new BdfObject();
BdfArray a = bdf.getArray();
byte[] bytes = new byte[1024*1024*1024];
for(int i=0;i<bytes.length;i++) {
bytes[i] = (byte)0;
}
for(int i=0;i<1000;i++) {
BdfArray a2 = new BdfArray();
a.add(BdfObject.withArray(a2));
a = a2;
}
a.add(BdfObject.withByteArray(bytes));
BdfDatabase data = bdf.serialize();
FileOutputStream file = new FileOutputStream("./database.bdf");
data.writeToStream(file);
*/
BdfFileManager bdf = new BdfFileManager("./database.bdf");
System.out.println("Loaded bdf");
Thread.sleep(5000);
}
}