Skip to content

Commit

Permalink
Floating point list support.
Browse files Browse the repository at this point in the history
  • Loading branch information
@pascaldekloe
pascaldekloe committed Nov 30, 2016
1 parent 0313b0f commit 12d89b5
Show file tree
Hide file tree
Showing 98 changed files with 1,235 additions and 29 deletions.
8 changes: 4 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ The format is inspired by Proto**col** Buf**fer**.
* RMI (WIP
[![GoDoc](https://godoc.org/github.com/pascaldekloe/colfer/rpc?status.svg)](https://godoc.org/github.com/pascaldekloe/colfer/rpc)
)
* Lists for numbers and timestamps
* Lists for integers and timestamps
* Please [share](https://github.com/pascaldekloe/colfer/wiki/Users#production-use) your experiences


Expand Down Expand Up @@ -143,7 +143,7 @@ The following table shows how Colfer data types are applied per language.
* †† timezone not preserved
* ‡‡ characters limited by UTF-16 (`U+0000`, `U+10FFFF`)

Lists may contain text, binaries or data structures.
Lists may contain floating points, text, binaries or data structures.


## Compatibility
Expand Down Expand Up @@ -217,5 +217,5 @@ as a 64-bit two's complement integer. In both cases the value is followed with
The data for text and binaries is prefixed with a varint byte size declaration.
Text is encoded as UTF-8.

Lists of text, binaries and data structures are prefixed with a varint element
size declaration.
Lists of floating points, text, binaries and data structures are prefixed with a
varint element size declaration.
8 changes: 6 additions & 2 deletions colfer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,12 @@ func ReadDefs(files []string) ([]*Package, error) {
t := f.Type
_, ok := datatypes[t]
if ok {
if f.TypeList && t != "text" && t != "binary" {
return nil, fmt.Errorf("colfer: unsupported lists type %q for field %s", t, f.String())
if f.TypeList {
switch t {
case "float32", "float64", "text", "binary":
default:
return nil, fmt.Errorf("colfer: unsupported lists type %q for field %s", t, f.String())
}
}
continue
}
Expand Down
72 changes: 70 additions & 2 deletions ecma.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,9 @@ const ecmaCode = `// This file was generated by colf(1); DO NOT EDIT

const ecmaMarshal = `
// Serializes the object into an Uint8Array.
{{- range .Fields}}{{if .TypeList}}
{{- range .Fields}}{{if .TypeList}}{{if eq .Type "float32" "float64"}}{{else}}
// All null entries in property {{.NameNative}} will be replaced with {{if eq .Type "text"}}an empty String{{else if eq .Type "binary"}}an empty Array{{else}}a new {{.TypeRef.Pkg.NameNative}}.{{.TypeRef.NameTitle}}{{end}}.
{{- end}}{{end}}
{{- end}}{{end}}{{end}}
this.{{.NameTitle}}.prototype.marshal = function() {
var segs = [];
{{range .Fields}}{{if eq .Type "bool"}}
Expand Down Expand Up @@ -248,6 +248,25 @@ const ecmaMarshal = `
segs.push(seg);
}
{{else if eq .Type "float32"}}
{{- if .TypeList}}
if (this.{{.NameNative}} && this.{{.NameNative}}.length) {
if (this.{{.NameNative}}.length > colferListMax)
throw 'colfer: {{.String}} length exceeds colferListMax';
var seg = [{{.Index}}];
encodeVarint(seg, this.{{.NameNative}}.length);
segs.push(seg);
var bytes = new Uint8Array(this.{{.NameNative}}.length * 4);
segs.push(bytes);
var view = new DataView(bytes.buffer);
this.{{.NameNative}}.forEach(function(f, i) {
if (f > 3.4028234663852886E38 || f < -3.4028234663852886E38)
throw 'colfer: {{.String}}[' + i + '] exceeds 32-bit range';
view.setFloat32(i * 4, f);
});
}
{{- else}}
if (this.{{.NameNative}} || Number.isNaN(this.{{.NameNative}})) {
if (this.{{.NameNative}} > 3.4028234663852886E38 || this.{{.NameNative}} < -3.4028234663852886E38)
throw 'colfer: {{.Struct.Pkg.NameNative}}/{{.Struct.NameTitle}} field {{.NameNative}} exceeds 32-bit range';
Expand All @@ -256,13 +275,32 @@ const ecmaMarshal = `
new DataView(bytes.buffer).setFloat32(1, this.{{.NameNative}});
segs.push(bytes);
}
{{- end}}
{{else if eq .Type "float64"}}
{{- if .TypeList}}
if (this.{{.NameNative}} && this.{{.NameNative}}.length) {
if (this.{{.NameNative}}.length > colferListMax)
throw 'colfer: {{.String}} length exceeds colferListMax';
var seg = [{{.Index}}];
encodeVarint(seg, this.{{.NameNative}}.length);
segs.push(seg);
var bytes = new Uint8Array(this.{{.NameNative}}.length * 8);
segs.push(bytes);
var view = new DataView(bytes.buffer);
this.{{.NameNative}}.forEach(function(f, i) {
view.setFloat64(i * 8, f);
});
}
{{- else}}
if (this.{{.NameNative}} || Number.isNaN(this.{{.NameNative}})) {
var bytes = new Uint8Array(9);
bytes[0] = {{.Index}};
new DataView(bytes.buffer).setFloat64(1, this.{{.NameNative}});
segs.push(bytes);
}
{{- end}}
{{else if eq .Type "timestamp"}}
if ((this.{{.NameNative}} && this.{{.NameNative}}.getTime()) || this.{{.NameNative}}_ns) {
var ms = this.{{.NameNative}} ? this.{{.NameNative}}.getTime() : 0;
Expand Down Expand Up @@ -511,16 +549,46 @@ const ecmaUnmarshal = `
}
{{else if eq .Type "float32"}}
if (header == {{.Index}}) {
{{- if .TypeList}}
var length = readVarint();
if (length < 0)
throw 'colfer: {{.String}} length exceeds Number.MAX_SAFE_INTEGER';
if (length > colferListMax)
throw 'colfer: {{.String}} length ' + length + ' exceeds ' + colferListMax + ' elements';
if (i + length * 4 > data.length) throw EOF;
var view = new DataView(data.buffer);
while (--length >= 0) {
this.{{.NameNative}}.push(view.getFloat32(i));
i += 4;
}
{{- else}}
if (i + 4 > data.length) throw EOF;
this.{{.NameNative}} = new DataView(data.buffer).getFloat32(i);
i += 4;
{{- end}}
readHeader();
}
{{else if eq .Type "float64"}}
if (header == {{.Index}}) {
{{- if .TypeList}}
var length = readVarint();
if (length < 0)
throw 'colfer: {{.String}} length exceeds Number.MAX_SAFE_INTEGER';
if (length > colferListMax)
throw 'colfer: {{.String}} length ' + length + ' exceeds ' + colferListMax + ' elements';
if (i + length * 8 > data.length) throw EOF;
var view = new DataView(data.buffer);
while (--length >= 0) {
this.{{.NameNative}}.push(view.getFloat64(i));
i += 8;
}
{{- else}}
if (i + 8 > data.length) throw EOF;
this.{{.NameNative}} = new DataView(data.buffer).getFloat64(i);
i += 8;
{{- end}}
readHeader();
}
{{else if eq .Type "timestamp"}}
Expand Down
182 changes: 178 additions & 4 deletions go.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,20 +285,61 @@ const goMarshalField = `{{if eq .Type "bool"}}
i++
}
{{else if eq .Type "float32"}}
if v := o.{{.NameTitle}}; v != 0.0 {
{{- if .TypeList}}
if l := len(o.{{.NameTitle}}); l != 0 {
buf[i] = {{.Index}}
i++
x := uint(l)
for x >= 0x80 {
buf[i] = byte(x | 0x80)
x >>= 7
i++
}
buf[i] = byte(x)
i++
for _, v := range o.{{.NameTitle}} {
x := math.Float32bits(v)
buf[i], buf[i+1], buf[i+2], buf[i+3] = byte(x>>24), byte(x>>16), byte(x>>8), byte(x)
i += 4
}
}
{{- else}}
if v := o.{{.NameTitle}}; v != 0 {
buf[i] = {{.Index}}
x := math.Float32bits(v)
buf[i+1], buf[i+2], buf[i+3], buf[i+4] = byte(x>>24), byte(x>>16), byte(x>>8), byte(x)
i += 5
}
{{- end}}
{{else if eq .Type "float64"}}
if v := o.{{.NameTitle}}; v != 0.0 {
{{- if .TypeList}}
if l := len(o.{{.NameTitle}}); l != 0 {
buf[i] = {{.Index}}
i++
x := uint(l)
for x >= 0x80 {
buf[i] = byte(x | 0x80)
x >>= 7
i++
}
buf[i] = byte(x)
i++
for _, v := range o.{{.NameTitle}} {
x := math.Float64bits(v)
buf[i], buf[i+1], buf[i+2], buf[i+3] = byte(x>>56), byte(x>>48), byte(x>>40), byte(x>>32)
buf[i+4], buf[i+5], buf[i+6], buf[i+7] = byte(x>>24), byte(x>>16), byte(x>>8), byte(x)
i += 8
}
}
{{- else}}
if v := o.{{.NameTitle}}; v != 0 {
buf[i] = {{.Index}}
x := math.Float64bits(v)
buf[i+1], buf[i+2], buf[i+3], buf[i+4] = byte(x>>56), byte(x>>48), byte(x>>40), byte(x>>32)
buf[i+5], buf[i+6], buf[i+7], buf[i+8] = byte(x>>24), byte(x>>16), byte(x>>8), byte(x)
i += 9
}
{{- end}}
{{else if eq .Type "timestamp"}}
if v := o.{{.NameTitle}}; !v.IsZero() {
s, ns := uint64(v.Unix()), uint(v.Nanosecond())
Expand Down Expand Up @@ -430,13 +471,33 @@ const goMarshalFieldLen = `{{if eq .Type "bool"}}
}
}
{{else if eq .Type "float32"}}
if o.{{.NameTitle}} != 0.0 {
{{- if .TypeList}}
if x := len(o.{{.NameTitle}}); x != 0 {
l += 2 + x*4
for x >= 0x80 {
x >>= 7
l++
}
}
{{- else}}
if o.{{.NameTitle}} != 0 {
l += 5
}
{{- end}}
{{else if eq .Type "float64"}}
if o.{{.NameTitle}} != 0.0 {
{{- if .TypeList}}
if x := len(o.{{.NameTitle}}); x != 0 {
l += 2 + x*8
for x >= 0x80 {
x >>= 7
l++
}
}
{{- else}}
if o.{{.NameTitle}} != 0 {
l += 9
}
{{- end}}
{{else if eq .Type "timestamp"}}
if v := o.{{.NameTitle}}; !v.IsZero() {
if s := uint64(v.Unix()); s < 1<<32 {
Expand Down Expand Up @@ -781,6 +842,61 @@ const goUnmarshalField = `{{if eq .Type "bool"}}
i++
}
{{else if eq .Type "float32"}}
{{- if .TypeList}}
if header == {{.Index}} {
if i >= len(data) {
if i >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", i, ColferSizeMax))
}
return 0, io.EOF
}
x := uint(data[i])
i++
if x >= 0x80 {
x &= 0x7f
for shift := uint(7); ; shift += 7 {
if i >= len(data) {
if i >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", i, ColferSizeMax))
}
return 0, io.EOF
}
b := uint(data[i])
i++
if b < 0x80 {
x |= b << shift
break
}
x |= (b & 0x7f) << shift
}
}
if x > uint(ColferListMax) {
return 0, ColferMax(fmt.Sprintf("colfer: {{.String}} length %d exceeds %d elements", x, ColferListMax))
}
l := int(x)
if end := i + l*4; end >= len(data) {
if end >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", end, ColferSizeMax))
}
return 0, io.EOF
}
a := make([]float32, l)
o.{{.NameTitle}} = a
for ai := range a {
x := uint32(data[i])<<24 | uint32(data[i+1])<<16 | uint32(data[i+2])<<8 | uint32(data[i+3])
i += 4
a[ai] = math.Float32frombits(x)
}
header = data[i]
i++
}
{{- else}}
if header == {{.Index}} {
if i+4 >= len(data) {
if i+4 >= ColferSizeMax {
Expand All @@ -794,7 +910,64 @@ const goUnmarshalField = `{{if eq .Type "bool"}}
header = data[i+4]
i += 5
}
{{- end}}
{{else if eq .Type "float64"}}
{{- if .TypeList}}
if header == {{.Index}} {
if i >= len(data) {
if i >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", i, ColferSizeMax))
}
return 0, io.EOF
}
x := uint(data[i])
i++
if x >= 0x80 {
x &= 0x7f
for shift := uint(7); ; shift += 7 {
if i >= len(data) {
if i >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", i, ColferSizeMax))
}
return 0, io.EOF
}
b := uint(data[i])
i++
if b < 0x80 {
x |= b << shift
break
}
x |= (b & 0x7f) << shift
}
}
if x > uint(ColferListMax) {
return 0, ColferMax(fmt.Sprintf("colfer: {{.String}} length %d exceeds %d elements", x, ColferListMax))
}
l := int(x)
if end := i + l*8; end >= len(data) {
if end >= ColferSizeMax {
return 0, ColferMax(fmt.Sprintf("colfer: {{.Struct.String}} size %d exceeds %d bytes", end, ColferSizeMax))
}
return 0, io.EOF
}
a := make([]float64, l)
o.{{.NameTitle}} = a
for ai := range a {
x := uint64(data[i])<<56 | uint64(data[i+1])<<48 | uint64(data[i+2])<<40 | uint64(data[i+3])<<32
x |= uint64(data[i+4])<<24 | uint64(data[i+5])<<16 | uint64(data[i+6])<<8 | uint64(data[i+7])
i += 8
a[ai] = math.Float64frombits(x)
}
header = data[i]
i++
}
{{- else}}
if header == {{.Index}} {
if i+8 >= len(data) {
if i+8 >= ColferSizeMax {
Expand All @@ -809,6 +982,7 @@ const goUnmarshalField = `{{if eq .Type "bool"}}
header = data[i+8]
i += 9
}
{{- end}}
{{else if eq .Type "timestamp"}}
if header == {{.Index}} {
if i+8 >= len(data) {
Expand Down
Loading

0 comments on commit 12d89b5

Please sign in to comment.