Skip to content

Commit

Permalink
bigquery: convert nested objects to their native types (#1648)
Browse files Browse the repository at this point in the history
  • Loading branch information
@stephenplusplus @callmehiphop
stephenplusplus authored and callmehiphop committed Oct 14, 2016
1 parent c30952a commit a7a831c
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 53 deletions.
110 changes: 82 additions & 28 deletions src/table.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,39 @@ Table.createSchemaFromString_ = function(str) {
});
};

/**
* Convert a row entry from native types to their encoded types that the API
* expects.
*
* @static
* @private
*
* @param {*} value - The value to be converted.
* @return {*} The converted value.
*/
Table.encodeValue_ = function(value) {
if (value instanceof Buffer) {
return value.toString('base64');
}

if (is.date(value)) {
return value.toJSON();
}

if (is.array(value)) {
return value.map(Table.encodeValue_);
}

if (is.object(value)) {
return Object.keys(value).reduce(function(acc, key) {
acc[key] = Table.encodeValue_(value[key]);
return acc;
}, {});
}

return value;
};

/**
* Merge a rowset returned from the API with a table schema.
*
Expand All @@ -188,34 +221,19 @@ Table.createSchemaFromString_ = function(str) {
* @return {array} Fields using their matching names from the table's schema.
*/
Table.mergeSchemaWithRows_ = function(schema, rows) {
return rows.map(mergeSchema).map(flattenRows);
return arrify(rows).map(mergeSchema).map(flattenRows);

function mergeSchema(row) {
return row.f.map(function(field, index) {
var schemaField = schema.fields[index];
var value = field.v;

switch (schemaField.type) {
case 'BOOLEAN': {
value = value === 'true';
break;
}
case 'FLOAT': {
if (!is.nil(value)) {
value = parseFloat(value);
}
break;
}
case 'INTEGER': {
if (!is.nil(value)) {
value = parseInt(value, 10);
}
break;
}
case 'TIMESTAMP': {
value = new Date(value * 1000);
break;
}
if (schemaField.mode === 'REPEATED') {
value = value.map(function(val) {
return convert(schemaField, val.v);
});
} else {
value = convert(schemaField, value);
}

var fieldObject = {};
Expand All @@ -224,6 +242,41 @@ Table.mergeSchemaWithRows_ = function(schema, rows) {
});
}

function convert(schemaField, value) {
if (is.nil(value)) {
return value;
}

switch (schemaField.type) {
case 'BOOLEAN': {
value = value === 'true';
break;
}
case 'BYTES': {
value = new Buffer(value, 'base64');
break;
}
case 'FLOAT': {
value = parseFloat(value);
break;
}
case 'INTEGER': {
value = parseInt(value, 10);
break;
}
case 'RECORD': {
value = Table.mergeSchemaWithRows_(schemaField, value).pop();
break;
}
case 'TIMESTAMP': {
value = new Date(value * 1000);
break;
}
}

return value;
}

function flattenRows(rows) {
return rows.reduce(function(acc, row) {
var key = Object.keys(row)[0];
Expand Down Expand Up @@ -925,7 +978,7 @@ Table.prototype.insert = function(rows, options, callback) {
if (!options.raw) {
json.rows = arrify(rows).map(function(row) {
return {
json: row
json: Table.encodeValue_(row)
};
});
}
Expand Down Expand Up @@ -977,11 +1030,12 @@ Table.prototype.query = function(query, callback) {
* table.
* @param {string} metadata.name - A descriptive name for the table.
* @param {string|object} metadata.schema - A comma-separated list of name:type
* pairs. Valid types are "string", "integer", "float", "boolean", and
* "timestamp". If the type is omitted, it is assumed to be "string".
* Example: "name:string, age:integer". Schemas can also be specified as a
* JSON array of fields, which allows for nested and repeated fields. See
* a [Table resource](http://goo.gl/sl8Dmg) for more detailed information.
* pairs. Valid types are "string", "integer", "float", "boolean", "bytes",
* "record", and "timestamp". If the type is omitted, it is assumed to be
* "string". Example: "name:string, age:integer". Schemas can also be
* specified as a JSON array of fields, which allows for nested and repeated
* fields. See a [Table resource](http://goo.gl/sl8Dmg) for more detailed
* information.
* @param {function} callback - The callback function.
* @param {?error} callback.err - An error returned while making this request.
* @param {object} callback.apiResponse - The full API response.
Expand Down
87 changes: 68 additions & 19 deletions system-test/bigquery.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,55 @@ describe('BigQuery', function() {

var query = 'SELECT url FROM [publicdata:samples.github_nested] LIMIT 100';

var SCHEMA = [
{
name: 'id',
type: 'INTEGER'
},
{
name: 'breed',
type: 'STRING'
},
{
name: 'name',
type: 'STRING'
},
{
name: 'dob',
type: 'TIMESTAMP'
},
{
name: 'around',
type: 'BOOLEAN'
},
{
name: 'buffer',
type: 'BYTES'
},
{
name: 'arrayOfInts',
type: 'INTEGER',
mode: 'REPEATED'
},
{
name: 'recordOfRecords',
type: 'RECORD',
fields: [
{
name: 'records',
type: 'RECORD',
mode: 'REPEATED',
fields: [
{
name: 'record',
type: 'BOOLEAN'
}
]
}
]
}
];

before(function(done) {
async.series([
// Remove buckets created for the tests.
Expand All @@ -51,7 +100,7 @@ describe('BigQuery', function() {

// Create the test table.
table.create.bind(table, {
schema: 'id:integer,breed,name,dob:timestamp,around:boolean'
schema: SCHEMA
}),

// Create a Bucket.
Expand Down Expand Up @@ -312,15 +361,7 @@ describe('BigQuery', function() {
var TEST_DATA_JSON_PATH = require.resolve('./data/kitten-test-data.json');

it('should have created the correct schema', function() {
assert.deepEqual(table.metadata.schema, {
fields: [
{ name: 'id', type: 'INTEGER' },
{ name: 'breed', type: 'STRING' },
{ name: 'name', type: 'STRING' },
{ name: 'dob', type: 'TIMESTAMP' },
{ name: 'around', type: 'BOOLEAN' }
]
});
assert.deepEqual(table.metadata.schema.fields, SCHEMA);
});

it('should get the rows in a table', function(done) {
Expand Down Expand Up @@ -388,14 +429,21 @@ describe('BigQuery', function() {
});

it('should convert values to their schema types', function(done) {
var now = new Date();

var data = {
name: 'dave',
breed: 'british shorthair',
id: 99,
dob: now.toJSON(),
around: true
dob: new Date(),
around: true,
buffer: new Buffer('test'),
arrayOfInts: [1, 3, 5],
recordOfRecords: {
records: [
{
record: true
}
]
}
};

table.insert(data, function(err, insertErrors) {
Expand All @@ -409,7 +457,11 @@ describe('BigQuery', function() {
function query(callback) {
var row;

table.query('SELECT * FROM ' + table.id + ' WHERE id = ' + data.id)
table
.query({
query: 'SELECT * FROM ' + table.id + ' WHERE id = ' + data.id,
useLegacySql: false
})
.on('error', callback)
.once('data', function(row_) { row = row_; })
.on('end', function() {
Expand All @@ -418,10 +470,7 @@ describe('BigQuery', function() {
return;
}

assert.strictEqual(row.name, data.name);
assert.strictEqual(row.breed, data.breed);
assert.strictEqual(row.id, data.id);
assert.deepEqual(row.dob, now);
assert.deepEqual(row, data);
callback();
});
}
Expand Down
6 changes: 3 additions & 3 deletions system-test/data/kitten-test-data.json
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false }
{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false }
{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true }
{ "name": "silvano", "breed": "the cat kind", "id": 1, "dob": 1414634759011, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
{ "name": "ryan", "breed": "golden retriever?", "id": 2, "dob": 1414634759012, "around": false, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
{ "name": "stephen", "breed": "idkanycatbreeds", "id": 3, "dob": 1414634759013, "around": true, "buffer": "dGVzdA==", "arrayOfInts": [1, 3, 5], "recordOfRecords": { "records": [{ "record": true }] } }
Loading

0 comments on commit a7a831c

Please sign in to comment.