Binary list support.

pascaldekloe · Oct 23, 2016 · 7ec3694 · 7ec3694
1 parent c8e147d
commit 7ec3694
Show file tree

Hide file tree

Showing 75 changed files with 849 additions and 99 deletions.
diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ The format is inspired by Proto**col** Buf**fer**.
 * RMI (WIP
 [![GoDoc](https://godoc.org/github.com/pascaldekloe/colfer/rpc?status.svg)](https://godoc.org/github.com/pascaldekloe/colfer/rpc)
 )
-* Lists for numbers, timestamps and binaries
+* Lists for numbers and timestamps
 
 
 

diff --git a/colfer.go b/colfer.go
@@ -103,6 +103,26 @@ func (s *Struct) HasText() bool {
 	return false
 }
 
+// HasBinary returns whether s has one or more binary fields.
+func (s *Struct) HasBinary() bool {
+	for _, f := range s.Fields {
+		if f.Type == "binary" {
+			return true
+		}
+	}
+	return false
+}
+
+// HasBinaryList returns whether s has one or more binary list fields.
+func (s *Struct) HasBinaryList() bool {
+	for _, f := range s.Fields {
+		if f.Type == "binary" && f.TypeList {
+			return true
+		}
+	}
+	return false
+}
+
 // HasTimestamp returns whether s has one or more timestamp fields.
 func (s *Struct) HasTimestamp() bool {
 	for _, f := range s.Fields {
@@ -210,7 +230,7 @@ func ReadDefs(files []string) ([]*Package, error) {
 				t := f.Type
 				_, ok := datatypes[t]
 				if ok {
-					if f.TypeList && t != "text" {
+					if f.TypeList && t != "text" && t != "binary" {
 						return nil, fmt.Errorf("colfer: unsupported lists type %s for field %q", t, f)
 					}
 					continue

diff --git a/ecma.go b/ecma.go
@@ -135,7 +135,7 @@ var <:.NameNative:> = new function() {
 const ecmaMarshal = `
 	// Serializes the object into an Uint8Array.
 <:- range .Fields:><:if .TypeList:>
-	// All null (and undefined) entries in property <:.Name:> will be replaced with <:if eq .Type "text":>an empty String<:else:>a new <:.TypeRef.Pkg.NameNative:>.<:.TypeRef.NameTitle:><:end:>.
+	// All null entries in property <:.Name:> will be replaced with <:if eq .Type "text":>an empty String<:else if eq .Type "binary":>an empty Array<:else:>a new <:.TypeRef.Pkg.NameNative:>.<:.TypeRef.NameTitle:><:end:>.
 <:- end:><:end:>
 	this.<:.NameTitle:>.prototype.marshal = function() {
 		var segs = [];
@@ -275,7 +275,7 @@ const ecmaMarshal = `
 		if (this.<:.Name:> && this.<:.Name:>.length) {
 			var a = this.<:.Name:>;
 			if (a.length > colferListMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length exceeds colferListMax';
+				throw 'colfer: <:.String:> length exceeds colferListMax';
 			var seg = [<:.Index:>];
 			encodeVarint(seg, a.length);
 			segs.push(seg);
@@ -302,17 +302,39 @@ const ecmaMarshal = `
 		}
  <:- end:>
 <:else if eq .Type "binary":>
+ <:- if .TypeList:>
+		if (this.<:.Name:> && this.<:.Name:>.length) {
+			var a = this.<:.Name:>;
+			if (a.length > colferListMax)
+				throw 'colfer: <:.String:> length exceeds colferListMax';
+			var seg = [<:.Index:>];
+			encodeVarint(seg, a.length);
+			segs.push(seg);
+			for (var i = 0; i < a.length; i++) {
+				var b = a[i];
+				if (b == null) {
+					b = new Uint8Array(0);
+					a[i] = b;
+				}
+				seg = [];
+				encodeVarint(seg, b.length);
+				segs.push(seg);
+				segs.push(b)
+			}
+		}
+ <:- else:>
 		if (this.<:.Name:> && this.<:.Name:>.length) {
 			var seg = [<:.Index:>];
 			encodeVarint(seg, this.<:.Name:>.length);
 			segs.push(seg);
 			segs.push(this.<:.Name:>);
 		}
+ <:- end:>
 <:else if .TypeList:>
 		if (this.<:.Name:> && this.<:.Name:>.length) {
 			var a = this.<:.Name:>;
 			if (a.length > colferListMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length exceeds colferListMax';
+				throw 'colfer: <:.String:> length exceeds colferListMax';
 			var seg = [<:.Index:>];
 			encodeVarint(seg, a.length);
 			segs.push(seg);
@@ -336,7 +358,7 @@ const ecmaMarshal = `
 			size += seg.length;
 		});
 		if (size > colferSizeMax)
-			throw 'colfer: <:.Pkg.NameNative:>/<:.NameTitle:> serial size ' + size + ' exceeds ' + colferListMax + ' bytes';
+			throw 'colfer: <:.String:> serial size ' + size + ' exceeds ' + colferListMax + ' bytes';
 
 		var bytes = new Uint8Array(size);
 		var i = 0;
@@ -500,15 +522,15 @@ const ecmaUnmarshal = `
  <:- if .TypeList:>
 			var length = readVarint();
 			if (length < 0)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length exceeds Number.MAX_SAFE_INTEGER';
+				throw 'colfer: <:.String:> length exceeds Number.MAX_SAFE_INTEGER';
 			if (length > colferListMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length ' + length + ' exceeds ' + colferListMax + ' elements';
+				throw 'colfer: <:.String:> length ' + length + ' exceeds ' + colferListMax + ' elements';
 			while (--length >= 0) {
 				var size = readVarint();
 				if (size < 0)
-					throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> element ' + this.<:.Name:>.length + ' size exceeds Number.MAX_SAFE_INTEGER';
+					throw 'colfer: <:.String:> element ' + this.<:.Name:>.length + ' size exceeds Number.MAX_SAFE_INTEGER';
 				else if (size > colferSizeMax)
-					throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> element ' + this.<:.Name:>.length + ' size ' + size + ' exceeds ' + colferSizeMax + ' UTF-8 bytes';
+					throw 'colfer: <:.String:> element ' + this.<:.Name:>.length + ' size ' + size + ' exceeds ' + colferSizeMax + ' UTF-8 bytes';
 				var to = i + size;
 				if (to > data.length) throw EOF;
 				this.<:.Name:>.push(decodeUTF8(data.subarray(i, to)));
@@ -517,9 +539,9 @@ const ecmaUnmarshal = `
  <:- else:>
 			var size = readVarint();
 			if (size < 0)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> size exceeds Number.MAX_SAFE_INTEGER';
+				throw 'colfer: <:.String:> size exceeds Number.MAX_SAFE_INTEGER';
 			else if (size > colferSizeMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> size ' + size + ' exceeds ' + colferSizeMax + ' UTF-8 bytes';
+				throw 'colfer: <:.String:> size ' + size + ' exceeds ' + colferSizeMax + ' UTF-8 bytes';
 			var to = i + size;
 			if (to > data.length) throw EOF;
 			this.<:.Name:> = decodeUTF8(data.subarray(i, to));
@@ -529,24 +551,43 @@ const ecmaUnmarshal = `
 		}
 <:else if eq .Type "binary":>
 		if (header == <:.Index:>) {
+ <:- if .TypeList:>
+			var length = readVarint();
+			if (length < 0)
+				throw 'colfer: <:.String:> length exceeds Number.MAX_SAFE_INTEGER';
+			if (length > colferListMax)
+				throw 'colfer: <:.String:> length ' + length + ' exceeds ' + colferListMax + ' elements';
+			while (--length >= 0) {
+				var size = readVarint();
+				if (size < 0)
+					throw 'colfer: <:.String:> element ' + this.<:.Name:>.length + ' size exceeds Number.MAX_SAFE_INTEGER';
+				else if (size > colferSizeMax)
+					throw 'colfer: <:.String:> element ' + this.<:.Name:>.length + ' size ' + size + ' exceeds ' + colferSizeMax + ' UTF-8 bytes';
+				var to = i + size;
+				if (to > data.length) throw EOF;
+				this.<:.Name:>.push(data.subarray(i, to));
+				i = to;
+			}
+ <:- else:>
 			var size = readVarint();
 			if (size < 0)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> size exceeds Number.MAX_SAFE_INTEGER';
+				throw 'colfer: <:.String:> size exceeds Number.MAX_SAFE_INTEGER';
 			else if (size > colferSizeMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> size ' + size + ' exceeds ' + colferSizeMax + ' bytes';
+				throw 'colfer: <:.String:> size ' + size + ' exceeds ' + colferSizeMax + ' bytes';
 			var to = i + size;
 			if (to > data.length) throw EOF;
 			this.<:.Name:> = data.subarray(i, to);
 			i = to;
+ <:- end:>
 			readHeader();
 		}
 <:else if .TypeList:>
 		if (header == <:.Index:>) {
 			var length = readVarint();
 			if (length < 0)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length exceeds Number.MAX_SAFE_INTEGER';
+				throw 'colfer: <:.String:> length exceeds Number.MAX_SAFE_INTEGER';
 			if (length > colferListMax)
-				throw 'colfer: <:.Struct.Pkg.NameNative:>/<:.Struct.NameTitle:> field <:.Name:> length ' + length + ' exceeds ' + colferListMax + ' elements';
+				throw 'colfer: <:.String:> length ' + length + ' exceeds ' + colferListMax + ' elements';
 			while (--length >= 0) {
 				var o = new <:.TypeRef.Pkg.NameNative:>.<:.TypeRef.NameTitle:>();
 				i += o.unmarshal(data.subarray(i));
@@ -564,6 +605,6 @@ const ecmaUnmarshal = `
 <:end:><:end:>
 		if (header != 127) throw 'colfer: unknown header at byte ' + (i - 1);
 		if (i > colferSizeMax)
-			throw 'colfer: <:.Pkg.NameNative:>/<:.NameTitle:> serial size ' + size + ' exceeds ' + colferSizeMax + ' bytes';
+			throw 'colfer: <:.String:> serial size ' + size + ' exceeds ' + colferSizeMax + ' bytes';
 		return i;
 	}`
diff --git a/gen.go b/gen.go
@@ -801,7 +801,7 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 		}
  <:- if .TypeList:>
 		if x > uint(ColferListMax) {
-			return 0, ColferMax(fmt.Sprintf("colfer: field <:.String:> length %d exceeds %d elements", x, ColferListMax))
+			return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> length %d exceeds %d elements", x, ColferListMax))
 		}
 		a := make([]string, int(x))
 		o.<:.NameTitle:> = a
@@ -835,7 +835,7 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 				}
 			}
 			if x > uint(ColferSizeMax) {
-				return 0, ColferMax(fmt.Sprintf("colfer: field <:.String:> element %d size %d exceeds %d bytes", ai, x, ColferSizeMax))
+				return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> element %d size %d exceeds %d bytes", ai, x, ColferSizeMax))
 			}
 			to := i + int(x)
 			if to >= len(data) {
@@ -856,7 +856,7 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 	}
  <:- else:>
 		if x > uint(ColferSizeMax) {
-			return 0, ColferMax(fmt.Sprintf("colfer: field <:.String:> size %d exceeds %d bytes", x, ColferSizeMax))
+			return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> size %d exceeds %d bytes", x, ColferSizeMax))
 		}
 		to := i + int(x)
 		if to >= len(data) {
@@ -901,9 +901,9 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 				x |= (b & 0x7f) << shift
 			}
 		}
-
+ <:- if not .TypeList:>
 		if x > uint(ColferSizeMax) {
-			return 0, ColferMax(fmt.Sprintf("colfer: field <:.String:> size %d exceeds %d bytes", x, ColferSizeMax))
+			return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> size %d exceeds %d bytes", x, ColferSizeMax))
 		}
 		l := int(x)
 		if i + l >= len(data) {
@@ -918,6 +918,66 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 
 		header = data[i]
 		i++
+ <:- else:>
+		if x > uint(ColferListMax) {
+			return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> length %d exceeds %d elements", x, ColferListMax))
+		}
+		a := make([][]byte, int(x))
+		o.<:.NameTitle:> = a
+		for ai := range a {
+			if i >= len(data) {
+				if i >= ColferSizeMax {
+					return 0, ColferMax(fmt.Sprintf("colfer: <:.Struct.String:> exceeds %d bytes", i, ColferSizeMax))
+				}
+				return 0, io.EOF
+			}
+			x := uint(data[i])
+			i++
+
+			if x >= 0x80 {
+				x &= 0x7f
+				for shift := uint(7); ; shift += 7 {
+					if i >= len(data) {
+						if i >= ColferSizeMax {
+							return 0, ColferMax(fmt.Sprintf("colfer: <:.Struct.String:> exceeds %d bytes", i, ColferSizeMax))
+						}
+						return 0, io.EOF
+					}
+					b := uint(data[i])
+					i++
+
+					if b < 0x80 {
+						x |= b << shift
+						break
+					}
+					x |= (b & 0x7f) << shift
+				}
+			}
+			if x > uint(ColferSizeMax) {
+				return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> element %d size %d exceeds %d bytes", ai, x, ColferSizeMax))
+			}
+			v := make([]byte, int(x))
+
+			if i + len(v) >= len(data) {
+				if i >= ColferSizeMax {
+					return 0, ColferMax(fmt.Sprintf("colfer: <:.Struct.String:> exceeds %d bytes", i, ColferSizeMax))
+				}
+				return 0, io.EOF
+			}
+
+			i += copy(v, data[i:])
+			a[ai] = v
+		}
+
+		if i >= len(data) {
+			if i >= ColferSizeMax {
+				return 0, ColferMax(fmt.Sprintf("colfer: <:.Struct.String:> exceeds %d bytes", i, ColferSizeMax))
+			}
+			return 0, io.EOF
+		}
+		header = data[i]
+		i++
+ <:- end:>
 	}
 <:else if .TypeList:>
 	if header == <:.Index:> {
@@ -947,7 +1007,7 @@ const goUnmarshalField = `<:if eq .Type "bool":>
 			}
 		}
 		if x > uint(ColferListMax) {
-			return 0, ColferMax(fmt.Sprintf("colfer: field <:.String:> length %d exceeds %d elements", x, ColferListMax))
+			return 0, ColferMax(fmt.Sprintf("colfer: <:.String:> length %d exceeds %d elements", x, ColferListMax))
 		}
 
 		l := int(x)

diff --git a/gen_test.go b/gen_test.go
@@ -74,6 +74,7 @@ func newGoldenCases() []*golden {
 		{"0b01007f7f", testdata.O{Os: []*testdata.O{{B: true}}}},
 		{"0b027f7f7f", testdata.O{Os: []*testdata.O{{}, {}}}},
 		{"0c0300016101627f", testdata.O{Ss: []string{"", "a", "b"}}},
+		{"0d0201000201027f", testdata.O{As: [][]byte{[]byte{0}, []byte{1, 2}}}},
 	}
 }