Skip to content

Commit 7dcd677

Browse files
muditsinmuditsin
committed
Add support for Array-Type in JSON
Fixes: #35
1 parent 260efe6 commit 7dcd677

File tree

3 files changed

+99
-25
lines changed

3 files changed

+99
-25
lines changed

‎serde/src/main/java/edu/berkeley/cs/succinct/DataType.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ public enum DataType {
99
LONG(4),
1010
FLOAT(5),
1111
DOUBLE(6),
12-
STRING(7);
12+
STRING(7),
13+
STRINGARRAY(8),
14+
LONGARRAY(9),
15+
BYTEARRAY(10),
16+
BOOLARRAY(11);
1317

1418
private final int order;
1519

‎serde/src/main/java/edu/berkeley/cs/succinct/block/json/JsonBlockSerializer.java‎

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.fasterxml.jackson.databind.JsonNode;
44
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import com.fasterxml.jackson.databind.node.ArrayNode;
56
import com.fasterxml.jackson.databind.node.ObjectNode;
67
import com.fasterxml.jackson.databind.node.ValueNode;
78
import edu.berkeley.cs.succinct.DataType;
@@ -67,28 +68,48 @@ private void flattenJsonTree(String currentPath, JsonNode jsonNode, ByteArrayOut
6768
flattenJsonTree(pathPrefix + entry.getKey(), entry.getValue(), out);
6869
}
6970
} else if (jsonNode.isArray()) {
70-
throw new SerializationException("Arrays in JSON are not supported yet.");
71+
ArrayNode arrayNode = (ArrayNode) jsonNode;
72+
DataType primitiveArrayType = getNodeType(arrayNode.get(0));
73+
DataType jsonArrayType = getArrayNodeType(primitiveArrayType);
74+
for (int i = 0; i < arrayNode.size(); i++) {
75+
if (getNodeType(arrayNode.get(i)) != primitiveArrayType)
76+
throw new SerializationException("Multi Type Arrays in JSON are not supported yet.");
77+
else if (!arrayNode.get(i).isValueNode())
78+
throw new SerializationException("Non primitive types in Arrays in JSON are not supported yet.");
79+
}
80+
for (int i = 0; i < arrayNode.size(); i++) {
81+
ValueNode valueNode = (ValueNode) arrayNode.get(i);
82+
writeJsonTree(currentPath, valueNode, jsonArrayType, out, true);
83+
}
7184
} else if (jsonNode.isValueNode()) {
7285
ValueNode valueNode = (ValueNode) jsonNode;
73-
if (!fieldMapping.containsField(currentPath)) {
74-
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], getNodeType(jsonNode));
75-
} else {
76-
DataType existingType = fieldMapping.getDataType(currentPath);
77-
DataType newType = getNodeType(valueNode);
78-
if (existingType != newType) {
79-
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
86+
writeJsonTree(currentPath, valueNode, getNodeType(jsonNode), out, false);
87+
}
88+
}
89+
90+
private void writeJsonTree(String currentPath, ValueNode valueNode, DataType fieldMappingType, ByteArrayOutputStream out,
91+
boolean isArray) throws SerializationException {
92+
if (!fieldMapping.containsField(currentPath)) {
93+
fieldMapping.put(currentPath, delimiters[currentDelimiterIdx++], fieldMappingType);
94+
} else {
95+
DataType existingType = fieldMapping.getDataType(currentPath);
96+
DataType newType = getNodeType(valueNode);
97+
if (existingType != newType) {
98+
DataType encapsulatingType = DataType.encapsulatingType(existingType, newType);
99+
if(isArray)
100+
fieldMapping.updateType(currentPath, getArrayNodeType(encapsulatingType));
101+
else
80102
fieldMapping.updateType(currentPath, encapsulatingType);
81-
}
82-
}
83-
try {
84-
byte fieldByte = fieldMapping.getDelimiter(currentPath);
85-
out.write(fieldByte);
86-
out.write(valueNode.asText().getBytes());
87-
out.write(fieldByte);
88-
} catch (IOException e) {
89-
throw new SerializationException(e.getMessage());
90103
}
91104
}
105+
try {
106+
byte fieldByte = fieldMapping.getDelimiter(currentPath);
107+
out.write(fieldByte);
108+
out.write(valueNode.asText().getBytes());
109+
out.write(fieldByte);
110+
} catch (IOException e) {
111+
throw new SerializationException(e.getMessage());
112+
}
92113
}
93114

94115
private DataType getNodeType(JsonNode node) {
@@ -108,4 +129,12 @@ private DataType getNodeType(JsonNode node) {
108129
throw new UnsupportedOperationException("JSON DataType not supported.");
109130
}
110131
}
132+
133+
private DataType getArrayNodeType(DataType fieldType) {
134+
if (fieldType == DataType.STRING) return DataType.STRINGARRAY;
135+
else if (fieldType == DataType.LONG) return DataType.LONGARRAY;
136+
else if (fieldType == DataType.BYTE) return DataType.BYTEARRAY;
137+
else if (fieldType == DataType.BOOLEAN) return DataType.BOOLARRAY;
138+
else throw new UnsupportedOperationException("JSON DataType not supported.");
139+
}
111140
}

‎serde/src/main/java/edu/berkeley/cs/succinct/object/deserializer/JsonDeserializer.java‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@
77
import edu.berkeley.cs.succinct.PrimitiveDeserializer;
88
import edu.berkeley.cs.succinct.block.json.FieldMapping;
99

10-
import java.util.Arrays;
11-
import java.util.HashMap;
12-
import java.util.Map;
10+
import java.util.*;
1311

1412
public class JsonDeserializer implements ObjectDeserializer<String> {
1513

@@ -42,23 +40,53 @@ private String mapToJsonString(Map<String, Object> jsonMap) throws Deserializati
4240
return jsonString;
4341
}
4442

45-
private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException {
43+
private Map<String, Object> deserializeToMap(byte[] data) throws DeserializationException, IllegalArgumentException {
4644
Map<String, Object> jsonMap = new HashMap<String, Object>();
4745

4846
int curOffset = 0;
4947
while (curOffset < data.length) {
5048
byte curDelimiter = data[curOffset++];
5149
String fieldKey = fieldMapping.getField(curDelimiter);
5250
DataType fieldType = fieldMapping.getDataType(fieldKey);
53-
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
54-
Object fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
51+
Object fieldValue;
52+
int totalLength;
53+
if (isArray(fieldType)) {
54+
ArrayList<byte []> fieldValuesBytes = extractArrayField(data, curOffset, curDelimiter);
55+
final DataType typeinArray = getPrimitiveFromArray(fieldType);
56+
ArrayList<Object> resultSet = new ArrayList<>();
57+
totalLength = 0;
58+
for (byte [] fieldValueBytes : fieldValuesBytes) {
59+
resultSet.add(PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, typeinArray));
60+
totalLength += fieldValueBytes.length + 2; // Skip the field data, and the delimiter at end and start of next field
61+
}
62+
fieldValue = resultSet.toArray();
63+
totalLength -= 1; // Remove last additional append of delimiter
64+
}
65+
else {
66+
byte[] fieldValueBytes = extractField(data, curOffset, curDelimiter);
67+
fieldValue = PrimitiveDeserializer.deserializePrimitive(fieldValueBytes, fieldType);
68+
totalLength = fieldValueBytes.length + 1; // Skip the field data, and the end delimiter
69+
}
5570
add(jsonMap, fieldKey, fieldValue);
56-
curOffset += (fieldValueBytes.length + 1); // Skip the field data, and the end delimiter
71+
curOffset += totalLength;
5772
}
5873

5974
return jsonMap;
6075
}
6176

77+
private DataType getPrimitiveFromArray(DataType fieldType) throws IllegalArgumentException {
78+
if (fieldType == DataType.STRINGARRAY) return DataType.STRING;
79+
else if (fieldType == DataType.LONGARRAY) return DataType.LONG;
80+
else if (fieldType == DataType.BYTEARRAY) return DataType.BYTE;
81+
else if (fieldType == DataType.BOOLARRAY) return DataType.BOOLEAN;
82+
else throw new IllegalArgumentException("Called getPrimitiveFromArray on a non Array DataType");
83+
}
84+
85+
private Boolean isArray(DataType fieldType) {
86+
return fieldType == DataType.STRINGARRAY || fieldType == DataType.LONGARRAY ||
87+
fieldType == DataType.BYTEARRAY || fieldType == DataType.BOOLARRAY;
88+
}
89+
6290
private void add(Map<String, Object> map, String key, Object value) {
6391
if (key.contains(".")) {
6492
String[] keySplits = key.split("\\.", 2);
@@ -84,4 +112,17 @@ private byte[] extractField(byte[] data, int startOffset, byte delimiter) {
84112
return Arrays.copyOfRange(data, startOffset, i);
85113
}
86114

115+
private ArrayList<byte []> extractArrayField(byte[] data, int startOffset, byte delimiter) {
116+
int i = startOffset;
117+
ArrayList<byte []> res = new ArrayList<>();
118+
while (true) {
119+
byte[] result = extractField(data, i, delimiter);
120+
res.add(result);
121+
i += result.length + 1;
122+
if(data.length <= i || data[i] != delimiter) break;
123+
else i++;
124+
}
125+
return res;
126+
}
127+
87128
}

0 commit comments

Comments
 (0)