Started working on a lookup table system for BdfNamedList to save memory and space

This commit is contained in:
josua 2020-07-24 11:26:52 +10:00
parent 42374c79c5
commit b62c43e892
9 changed files with 288 additions and 273 deletions

View File

@ -1,39 +0,0 @@
package bdf.classes;
import bdf.types.BdfObject;
public class BdfClassManager
{
protected IBdfClassManager method;
protected BdfObject object = new BdfObject();
public BdfClassManager(IBdfClassManager method)
{
// Save some variables for later
this.method = method;
}
public void setBdf(BdfObject bdf) {
this.object = bdf;
}
public BdfObject getBdf() {
return this.object;
}
public void save(BdfObject bdf) {
method.BdfClassSave(bdf);
}
public void load(BdfObject bdf) {
method.BdfClassLoad(bdf);
}
public void save() {
this.save(this.object);
}
public void load() {
this.load(this.object);
}
}

View File

@ -8,9 +8,10 @@ import java.util.zip.DeflaterOutputStream;
import bdf.data.BdfDatabase;
import bdf.types.BdfObject;
import bdf.types.BdfReader;
import bdf.util.FileHelpers;
public class BdfFileManager extends BdfObject
public class BdfFileManager extends BdfReader
{
protected String path;
private boolean compressed;

View File

@ -11,12 +11,16 @@ import bdf.util.DataHelpers;
public class BdfArray implements IBdfType, Iterable<BdfObject>
{
protected ArrayList<BdfObject> elements = new ArrayList<BdfObject>();
protected BdfLookupTable lookupTable;
public BdfArray() {
BdfArray(BdfLookupTable lookupTable) {
this.lookupTable = lookupTable;
}
public BdfArray(IBdfDatabase data)
BdfArray(BdfLookupTable lookupTable, IBdfDatabase data)
{
this.lookupTable = lookupTable;
// Create an iterator value to loop over the data
int i = 0;
@ -27,7 +31,7 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
int size = DataHelpers.getByteBuffer(data.getPointer(i, Integer.BYTES)).getInt();
// Get the object
BdfObject object = new BdfObject(data.getPointer((i+Integer.BYTES), size));
BdfObject object = new BdfObject(lookupTable, data.getPointer((i+Integer.BYTES), size));
// Add the object to the elements list
elements.add(object);
@ -37,6 +41,18 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
}
}
public BdfObject createObject() {
return new BdfObject(lookupTable);
}
public BdfNamedList createNamedList() {
return new BdfNamedList(lookupTable);
}
public BdfArray createArray() {
return new BdfArray(lookupTable);
}
@Override
public int serializeSeeker()
{
@ -119,9 +135,10 @@ public class BdfArray implements IBdfType, Iterable<BdfObject>
return this;
}
public BdfArray remove(int index) {
public BdfObject remove(int index) {
BdfObject bdf = elements.get(index);
elements.remove(index);
return this;
return bdf;
}
public BdfArray remove(BdfObject bdf) {

View File

@ -0,0 +1,88 @@
package bdf.types;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import bdf.data.IBdfDatabase;
import bdf.util.DataHelpers;
class BdfLookupTable implements IBdfType
{
private ArrayList<String> keys;
BdfLookupTable() {
keys = new ArrayList<String>();
}
BdfLookupTable(IBdfDatabase database)
{
keys = new ArrayList<String>();
for(int i=0;i<database.size();)
{
int key_size = DataHelpers.getByteBuffer(database.getPointer(i, 4)).getInt();
i += 4;
String key = new String(database.getBytes(i, key_size), StandardCharsets.UTF_16);
keys.add(key);
i += key_size;
}
}
int getLocation(String name)
{
for(int i=0;i<keys.size();i++)
{
String key = keys.get(i);
if(key.contentEquals(name)) {
return i;
}
}
keys.add(name);
return keys.size() - 1;
}
String getName(int location) {
return keys.get(location);
}
@Override
public int serialize(IBdfDatabase database)
{
int upto = 0;
for(int i=0;i<keys.size();i++)
{
String key = keys.get(i);
database.setBytes(upto + 4, key.getBytes());
database.setBytes(upto, DataHelpers.serializeInt(key.length()));
upto += key.length();
upto += 4;
}
return upto;
}
@Override
public int serializeSeeker()
{
int size = 0;
for(int i=0;i<keys.size();i++) {
size += keys.get(i).length();
size += 4;
}
return size;
}
@Override
public void serializeHumanReadable(OutputStream stream, BdfIndent indent, int it) {
}
}

View File

@ -12,17 +12,21 @@ public class BdfNamedList implements IBdfType
{
protected class Element
{
public byte[] key;
public int key;
public BdfObject object;
}
protected ArrayList<Element> elements = new ArrayList<Element>();
protected BdfLookupTable lookupTable;
public BdfNamedList() {
BdfNamedList(BdfLookupTable lookupTable) {
this.lookupTable = lookupTable;
}
public BdfNamedList(IBdfDatabase data)
BdfNamedList(BdfLookupTable lookupTable, IBdfDatabase data)
{
this.lookupTable = lookupTable;
// Create an iterator value to loop over the data
int i = 0;
@ -30,15 +34,13 @@ public class BdfNamedList implements IBdfType
while(i < data.size())
{
// Get the key
int key_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
int key = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
i += 4;
byte[] key = data.getPointer(i, key_size).getBytes();
// Get the object
i += key_size;
int object_size = DataHelpers.getByteBuffer(data.getPointer(i, 4)).getInt();
i += 4;
BdfObject object = new BdfObject(data.getPointer(i, object_size));
BdfObject object = new BdfObject(lookupTable, data.getPointer(i, object_size));
// Create a new element and save some data to it
Element element = new Element();
@ -60,14 +62,10 @@ public class BdfNamedList implements IBdfType
for(Element o : elements)
{
database.setBytes(pos, DataHelpers.serializeInt(o.key.length));
database.setBytes(pos, DataHelpers.serializeInt(o.key));
pos += 4;
database.setBytes(pos, o.key);
pos += o.key.length;
int size = o.object.serialize(database.getPointer(pos + 4, database.size() - (pos + 4)));
database.setBytes(pos, DataHelpers.serializeInt(size));
@ -87,7 +85,6 @@ public class BdfNamedList implements IBdfType
for(Element o : elements)
{
size += 8;
size += o.key.length;
size += o.object.serializeSeeker();
}
@ -114,7 +111,7 @@ public class BdfNamedList implements IBdfType
stream.write(indent.indent.getBytes());
}
stream.write((DataHelpers.serializeString(new String(e.key, StandardCharsets.UTF_8)) + ": ").getBytes());
stream.write((DataHelpers.serializeString(lookupTable.getName(e.key)) + ": ").getBytes());
e.object.serializeHumanReadable(stream, indent, it + 1);
if(elements.size() > i+1) {
@ -143,7 +140,7 @@ public class BdfNamedList implements IBdfType
for(Element e : elements)
{
// Is this the element key
if(DataHelpers.bytesAreEqual(e.key, key_bytes))
if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes))
{
// Set the object
object = e.object;
@ -154,7 +151,7 @@ public class BdfNamedList implements IBdfType
}
// Get a bdf object
BdfObject o = new BdfObject();
BdfObject o = new BdfObject(lookupTable);
// Set the bdf object
this.set(key, o);
@ -163,7 +160,7 @@ public class BdfNamedList implements IBdfType
return o;
}
public BdfNamedList remove(String key)
public BdfObject remove(String key)
{
// Convert the key to bytes
byte[] key_bytes = key.getBytes();
@ -175,18 +172,13 @@ public class BdfNamedList implements IBdfType
Element e = elements.get(i);
// Is the specified key the same as the elements key
if(DataHelpers.bytesAreEqual(e.key, key_bytes))
{
// Delete this element
elements.remove(i);
// Exit out of the function, prevent NullPointException
return this;
if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes)) {
return elements.remove(i).object;
}
}
// Send back nothing
return this;
return null;
}
public BdfNamedList remove(BdfObject bdf)
@ -201,6 +193,18 @@ public class BdfNamedList implements IBdfType
return this;
}
public BdfObject createObject() {
return new BdfObject(lookupTable);
}
public BdfNamedList createNamedList() {
return new BdfNamedList(lookupTable);
}
public BdfArray createArray() {
return new BdfArray(lookupTable);
}
public BdfNamedList set(String key, BdfObject object)
{
// Convert the key to bytes
@ -210,7 +214,7 @@ public class BdfNamedList implements IBdfType
for(Element e : elements)
{
// Is the key here the same as the specified key
if(DataHelpers.bytesAreEqual(e.key, key_bytes))
if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes))
{
// Set the new object
e.object = object;
@ -222,7 +226,7 @@ public class BdfNamedList implements IBdfType
// Create a new element object
Element e = new Element();
e.key = key_bytes;
e.key = lookupTable.getLocation(key);
e.object = object;
// Add the new element object to the elements list
@ -241,7 +245,7 @@ public class BdfNamedList implements IBdfType
for(Element e : elements)
{
// Is the elements key the same as the specified key
if(DataHelpers.bytesAreEqual(e.key, key_bytes))
if(DataHelpers.bytesAreEqual(lookupTable.getName(e.key).getBytes(), key_bytes))
{
// Send back true to say the element was found
return true;
@ -262,7 +266,7 @@ public class BdfNamedList implements IBdfType
{
// Get the element
Element e = elements.get(i);
keys[i] = new String(e.key, StandardCharsets.UTF_8);
keys[i] = lookupTable.getName(e.key);
}
// Return the list of keys as strings

View File

@ -13,18 +13,17 @@ public class BdfObject implements IBdfType
{
protected IBdfDatabase database = null;
protected Object object = null;
protected byte type = BdfTypes.EMPTY;
protected byte type = BdfTypes.UNDEFINED;
protected BdfLookupTable lookupTable;
public static BdfObject getNew() {
return new BdfObject();
BdfObject(BdfLookupTable lookupTable) {
this.lookupTable = lookupTable;
}
public BdfObject(byte[] data) {
this(new BdfDatabase(data));
}
public BdfObject(IBdfDatabase data)
BdfObject(BdfLookupTable lookupTable, IBdfDatabase data)
{
this.lookupTable = lookupTable;
// Is the database length greater than 1
if(data.size() > 1)
{
@ -34,8 +33,8 @@ public class BdfObject implements IBdfType
// Set the object variable if there is an object specified
if(type == BdfTypes.STRING) object = database.getString();
if(type == BdfTypes.ARRAY) object = new BdfArray(database);
if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(database);
if(type == BdfTypes.ARRAY) object = new BdfArray(lookupTable, database);
if(type == BdfTypes.NAMED_LIST) object = new BdfNamedList(lookupTable, database);
if(object != null) {
database = null;
@ -99,15 +98,6 @@ public class BdfObject implements IBdfType
return database.size() + 1;
}
public BdfDatabase serialize()
{
BdfDatabase database = new BdfDatabase(serializeSeeker());
serialize(database);
return database;
}
private String calcIndent(BdfIndent indent, int it) {
String t = "";
for(int i=0;i<=it;i++) {
@ -116,36 +106,6 @@ public class BdfObject implements IBdfType
return t;
}
public String serializeHumanReadable(BdfIndent indent) {
return serializeHumanReadable(indent, 0);
}
public String serializeHumanReadable() {
return serializeHumanReadable(new BdfIndent("", ""), 0);
}
public String serializeHumanReadable(BdfIndent indent, int it)
{
ByteArrayOutputStream stream = new ByteArrayOutputStream();
try {
serializeHumanReadable(stream, indent, it);
return stream.toString();
}
catch(IOException e) {
return "undefined";
}
}
public void serializeHumanReadable(OutputStream stream, BdfIndent indent) throws IOException {
serializeHumanReadable(stream, indent, 0);
}
public void serializeHumanReadable(OutputStream stream) throws IOException {
serializeHumanReadable(stream, new BdfIndent("", ""), 0);
}
public void serializeHumanReadable(OutputStream stream, BdfIndent indent, int it) throws IOException
{
String str = null;
@ -280,10 +240,6 @@ public class BdfObject implements IBdfType
}
}
public BdfObject() {
database = new BdfDatabase(0);
}
public byte getType() {
return this.type;
}
@ -492,7 +448,7 @@ public class BdfObject implements IBdfType
public BdfArray getArray()
{
if(this.type != BdfTypes.ARRAY)
this.setArray();
setArray(createArray());
return (BdfArray)object;
}
@ -500,7 +456,7 @@ public class BdfObject implements IBdfType
public BdfNamedList getNamedList()
{
if(this.type != BdfTypes.NAMED_LIST)
this.setNamedList();
setNamedList(createNamedList());
return (BdfNamedList)object;
}
@ -577,12 +533,16 @@ public class BdfObject implements IBdfType
return this;
}
public BdfObject setArray() {
return this.setArray(new BdfArray());
public BdfObject createObject() {
return new BdfObject(lookupTable);
}
public BdfObject setNamedList() {
return this.setNamedList(new BdfNamedList());
public BdfNamedList createNamedList() {
return new BdfNamedList(lookupTable);
}
public BdfArray createArray() {
return new BdfArray(lookupTable);
}
public BdfObject setBooleanArray(boolean[] value) {
@ -654,84 +614,5 @@ public class BdfObject implements IBdfType
database = DataHelpers.getDatabase(b);
return this;
}
// Primitives
public static BdfObject withInteger(int v) {
return (new BdfObject()).setInteger(v);
}
public static BdfObject withByte(byte v) {
return (new BdfObject()).setByte(v);
}
public static BdfObject withBoolean(boolean v) {
return (new BdfObject()).setBoolean(v);
}
public static BdfObject withFloat(float v) {
return (new BdfObject()).setFloat(v);
}
public static BdfObject withDouble(double v) {
return (new BdfObject()).setDouble(v);
}
public static BdfObject withLong(long v) {
return (new BdfObject()).setLong(v);
}
public static BdfObject withShort(short v) {
return (new BdfObject()).setShort(v);
}
// Arrays
public static BdfObject withIntegerArray(int[] v) {
return (new BdfObject()).setIntegerArray(v);
}
public static BdfObject withByteArray(byte[] v) {
return (new BdfObject()).setByteArray(v);
}
public static BdfObject withBooleanArray(boolean[] v) {
return (new BdfObject()).setBooleanArray(v);
}
public static BdfObject withFloatArray(float[] v) {
return (new BdfObject()).setFloatArray(v);
}
public static BdfObject withDoubleArray(double[] v) {
return (new BdfObject()).setDoubleArray(v);
}
public static BdfObject withLongArray(long[] v) {
return (new BdfObject()).setLongArray(v);
}
public static BdfObject withShortArray(short[] v) {
return (new BdfObject()).setShortArray(v);
}
// Objects
public static BdfObject withString(String v) {
return (new BdfObject()).setString(v);
}
public static BdfObject withArray(BdfArray v) {
return (new BdfObject()).setArray(v);
}
public static BdfObject withNamedList(BdfNamedList v) {
return (new BdfObject()).setNamedList(v);
}
public static BdfObject withArray() {
return (new BdfObject()).setArray(new BdfArray());
}
public static BdfObject withNamedList() {
return (new BdfObject()).setNamedList(new BdfNamedList());
}
}

View File

@ -0,0 +1,106 @@
package bdf.types;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import bdf.data.BdfDatabase;
import bdf.data.IBdfDatabase;
import bdf.util.DataHelpers;
public class BdfReader
{
protected BdfLookupTable lookupTable;
protected BdfObject bdf;
public BdfReader() {
lookupTable = new BdfLookupTable();
bdf = new BdfObject(lookupTable);
}
public BdfReader(byte[] database) {
this(new BdfDatabase(database));
}
public BdfReader(IBdfDatabase database)
{
// Check the version of the BDF file
if(!"BDF3".contentEquals(new String(database.getBytes(0, 4)))) {
lookupTable = new BdfLookupTable();
bdf = new BdfObject(lookupTable);
return;
}
// Get the lookup table
int lookupTable_size = DataHelpers.getByteBuffer(database.getPointer(4, 4)).getInt();
lookupTable = new BdfLookupTable(database.getPointer(8, lookupTable_size));
// Get the rest of the data
int upto = lookupTable_size + 8;
int bdf_size = DataHelpers.getByteBuffer(database.getPointer(upto, 4)).getInt();
bdf = new BdfObject(lookupTable, database.getPointer(upto + 4, bdf_size));
}
public BdfDatabase serialize()
{
int bdf_size = bdf.serializeSeeker();
int lookupTable_size = lookupTable.serializeSeeker();
int database_size = bdf_size + lookupTable_size + 12;
BdfDatabase database = new BdfDatabase(database_size);
database.setBytes(0, "BDF3".getBytes());
database.setBytes(4, DataHelpers.serializeInt(lookupTable_size));
database.setBytes(8 + lookupTable_size, DataHelpers.serializeInt(bdf_size));
lookupTable.serialize(database.getPointer(8));
bdf.serialize(database.getPointer(12 + lookupTable_size));
return database;
}
public BdfObject getBDF() {
return bdf;
}
public String serializeHumanReadable(BdfIndent indent) {
return serializeHumanReadable(indent, 0);
}
public String serializeHumanReadable() {
return serializeHumanReadable(new BdfIndent("", ""), 0);
}
public String serializeHumanReadable(BdfIndent indent, int it)
{
ByteArrayOutputStream stream = new ByteArrayOutputStream();
try {
bdf.serializeHumanReadable(stream, indent, it);
return stream.toString();
}
catch(IOException e) {
return "undefined";
}
}
public void serializeHumanReadable(OutputStream stream, BdfIndent indent) throws IOException {
bdf.serializeHumanReadable(stream, indent, 0);
}
public void serializeHumanReadable(OutputStream stream) throws IOException {
bdf.serializeHumanReadable(stream, new BdfIndent("", ""), 0);
}
public BdfObject createObject() {
return new BdfObject(lookupTable);
}
public BdfNamedList createNamedList() {
return new BdfNamedList(lookupTable);
}
public BdfArray createArray() {
return new BdfArray(lookupTable);
}
}

View File

@ -2,18 +2,19 @@ package bdf.types;
public class BdfTypes
{
public static final byte BOOLEAN = 0;
public static final byte INTEGER = 1;
public static final byte LONG = 2;
public static final byte SHORT = 3;
public static final byte BYTE = 4;
public static final byte DOUBLE = 5;
public static final byte FLOAT = 6;
public static final byte UNDEFINED = 0;
public static final byte STRING = 7;
public static final byte ARRAY = 8;
public static final byte NAMED_LIST = 9;
public static final byte EMPTY = 10;
public static final byte BOOLEAN = 1;
public static final byte INTEGER = 2;
public static final byte LONG = 3;
public static final byte SHORT = 4;
public static final byte BYTE = 5;
public static final byte DOUBLE = 6;
public static final byte FLOAT = 7;
public static final byte STRING = 8;
public static final byte ARRAY = 9;
public static final byte NAMED_LIST = 10;
public static final byte ARRAY_BOOLEAN = 11;
public static final byte ARRAY_INTEGER = 12;

View File

@ -1,67 +1,23 @@
package tests;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import bdf.data.BdfDatabase;
import bdf.file.BdfFileManager;
import bdf.types.BdfArray;
import bdf.types.BdfIndent;
import bdf.types.BdfNamedList;
import bdf.types.BdfObject;
import bdf.types.BdfReader;
public class Tests {
public static void main(String[] args) throws InterruptedException, IOException
{
/*
BdfObject bdf = new BdfObject();
BdfReader reader = new BdfReader();
BdfObject bdf = reader.getBDF();
BdfNamedList nl = bdf.getNamedList();
nl.set("hello", nl.createObject().setInteger(69));
nl.set("world", nl.createObject().setInteger(420));
byte[] bytes = new byte[1024*1024*1024];
for(int i=0;i<bytes.length;i++) {
bytes[i] = (byte)0;
}
for(int i=0;i<1000;i++) {
nl = nl.get("next").getNamedList();
}
nl.get("next").setByteArray(bytes);
BdfDatabase data = bdf.serialize();
FileOutputStream file = new FileOutputStream("./database.bdf");
data.writeToStream(file);
*/
BdfObject bdf = new BdfObject();
BdfArray a = bdf.getArray();
byte[] bytes = new byte[1024*1024*1024/2];
for(int i=0;i<bytes.length;i++) {
bytes[i] = (byte)0;
}
for(int i=0;i<10;i++) {
BdfArray a2 = new BdfArray();
a.add(BdfObject.withArray(a2));
a = a2;
}
a.add(BdfObject.withByteArray(bytes));
BdfDatabase data = bdf.serialize();
FileOutputStream file = new FileOutputStream("./database.bdf");
data.writeToStream(file);
//BdfFileManager bdf = new BdfFileManager("./database.bdf");
//System.out.println("Loaded bdf");
//Thread.sleep(5000);
reader.serializeHumanReadable(System.out, new BdfIndent(" ", "\n"));
}
}