diff --git a/src/index.js b/src/index.js index 8d668db5..76806b9c 100644 --- a/src/index.js +++ b/src/index.js @@ -166,6 +166,8 @@ BigQuery.prototype.datetime = function(value) { d: value.day, time: time ? ' ' + time : '' }); + } else { + value = value.replace(/^(.*)T(.*)Z$/, '$1 $2'); } this.value = value; @@ -228,11 +230,8 @@ BigQuery.prototype.timestamp = function(value) { return new BigQuery.timestamp(value); } - value = value || new Date(); - - if (is.date(value)) { - value = value.toJSON().replace(/^(.*)T(.*)Z$/, '$1 $2'); - } + value = new Date(value); + value = value.toJSON().replace(/^(.*)T(.*)Z$/, '$1 $2'); this.value = value; }; @@ -841,7 +840,7 @@ BigQuery.prototype.query = function(options, callback) { var rows = []; if (resp.schema && resp.rows) { - rows = Table.mergeSchemaWithRows_(resp.schema, resp.rows); + rows = Table.mergeSchemaWithRows_(BigQuery, resp.schema, resp.rows); } var nextQuery = null; diff --git a/src/table.js b/src/table.js index 09e49d93..2e6e4dc6 100644 --- a/src/table.js +++ b/src/table.js @@ -258,7 +258,7 @@ Table.encodeValue_ = function(value) { * @param {array} rows * @return {array} Fields using their matching names from the table's schema. */ -Table.mergeSchemaWithRows_ = function(schema, rows) { +Table.mergeSchemaWithRows_ = function(BigQuery, schema, rows) { return arrify(rows).map(mergeSchema).map(flattenRows); function mergeSchema(row) { @@ -303,11 +303,23 @@ Table.mergeSchemaWithRows_ = function(schema, rows) { break; } case 'RECORD': { - value = Table.mergeSchemaWithRows_(schemaField, value).pop(); + value = Table.mergeSchemaWithRows_(BigQuery, schemaField, value).pop(); + break; + } + case 'DATE': { + value = BigQuery.date(value); + break; + } + case 'DATETIME': { + value = BigQuery.datetime(value); + break; + } + case 'TIME': { + value = BigQuery.time(value); break; } case 'TIMESTAMP': { - value = new Date(value * 1000); + value = BigQuery.timestamp(new Date(value * 1000)); break; } } @@ -780,7 +792,11 @@ Table.prototype.getRows = function(options, callback) { return; } - rows = Table.mergeSchemaWithRows_(self.metadata.schema, rows || []); + rows = Table.mergeSchemaWithRows_( + self.bigQuery, + self.metadata.schema, + rows || [] + ); callback(null, rows, nextQuery, resp); } diff --git a/system-test/bigquery.js b/system-test/bigquery.js index b68df3da..12a54d0d 100644 --- a/system-test/bigquery.js +++ b/system-test/bigquery.js @@ -501,68 +501,9 @@ describe('BigQuery', function() { }); }); - it('should convert values to their schema types', function(done) { - var data = { - name: 'dave', - breed: 'husky', - id: 99, - dob: new Date(), - around: true, - buffer: new Buffer('test'), - arrayOfInts: [1, 3, 5], - recordOfRecords: { - records: [ - { - record: true - } - ] - } - }; - - table.insert(data, function(err) { - assert.ifError(err); - - function query(callback) { - var query = { - query: 'SELECT * FROM ' + table.id + ' WHERE id = ' + data.id, - useLegacySql: false - }; - var row; - - table.createQueryStream(query) - .on('error', callback) - .once('data', function(row_) { row = row_; }) - .on('end', function() { - if (!row) { - callback(new Error('Query returned 0 results.')); - return; - } - - assert.deepEqual(row, data); - callback(); - }); - } - - async.retry({ times: 3, interval: 2000 }, query, done); - }); - }); - it('should return partial errors', function(done) { var data = { - name: 'dave', - breed: 'british husky', - id: 99, - dob: new Date(), - around: true, - buffer: new Buffer('test'), - arrayOfInts: [1, 3, 5], - recordOfRecords: { - records: [ - { - record: true - } - ] - } + name: 'dave' }; var improperData = { @@ -988,6 +929,121 @@ describe('BigQuery', function() { }); }); + describe('Provided Tests', function() { + var table = dataset.table(generateName('table')); + var schema = require('./data/schema.json'); + var testData = require('./data/schema-test-data.json'); + + var EXPECTED_ROWS = { + Bilbo: { + Name: 'Bilbo', + Age: 111, + Weight: 67.2, + IsMagic: false, + Spells: [], + TeaTime: bigquery.time('10:00:00'), + NextVacation: bigquery.date('2017-09-22'), + FavoriteTime: bigquery.datetime('2031-04-01T05:09:27') + }, + + Gandalf: { + Name: 'Gandalf', + Age: 1000, + Weight: 198.6, + IsMagic: true, + Spells: [ + { + Name: 'Skydragon', + LastUsed: bigquery.timestamp('2015-10-31T23:59:56.000Z'), + DiscoveredBy: 'Firebreather', + Properties: [ + { + Name: 'Flying', + Power: 1 + }, + { + Name: 'Creature', + Power: 1 + }, + { + Name: 'Explodey', + Power: 11 + } + ], + Icon: new Buffer(testData[1].Spells[0].Icon, 'base64') + } + ], + TeaTime: bigquery.time('15:00:00'), + NextVacation: bigquery.date('2666-06-06'), + FavoriteTime: bigquery.datetime('2001-12-19T23:59:59') + }, + + Sabrina: { + Name: 'Sabrina', + Age: 17, + Weight: 128.3, + IsMagic: true, + Spells: [ + { + Name: 'Talking cats', + LastUsed: bigquery.timestamp('2017-02-14T12:07:23.000Z'), + DiscoveredBy: 'Salem', + Properties: [ + { + Name: 'Makes you look crazy', + Power: 1 + } + ], + Icon: new Buffer(testData[2].Spells[0].Icon, 'base64') + } + ], + TeaTime: bigquery.time('12:00:00'), + NextVacation: bigquery.date('2017-03-14'), + FavoriteTime: bigquery.datetime('2000-10-31T23:27:46') + } + }; + + before(function(done) { + async.series([ + function(next) { + table.create({ + schema: schema + }, next); + }, + + function(next) { + table.insert(testData, next); + }, + + function(next) { + setTimeout(next, 15000); + } + ], done); + }); + + after(function(done) { + table.delete(done); + }); + + it('should convert rows back correctly', function(done) { + table.getRows(function(err, rows) { + assert.ifError(err); + + if (rows.length === 0) { + done(new Error('Rows not returned from the API.')); + return; + } + + rows.forEach(function(row) { + var expectedRow = EXPECTED_ROWS[row.Name]; + assert.deepEqual(row, expectedRow); + }); + + done(); + }); + }); + }); + function generateName(resourceType) { return (GCLOUD_TESTS_PREFIX + resourceType + '_' + uuid.v1()) .replace(/-/g, '_'); diff --git a/system-test/data/schema-test-data.json b/system-test/data/schema-test-data.json new file mode 100644 index 00000000..83d045e7 --- /dev/null +++ b/system-test/data/schema-test-data.json @@ -0,0 +1,66 @@ +[ + { + "Age": "111", + "FavoriteTime": "2031-04-01T05:09:27", + "IsMagic": false, + "Name": "Bilbo", + "NextVacation": "2017-09-22", + "Spells": [], + "TeaTime": "10:00:00", + "Weight": 67.2 + }, + { + "Age": "1000", + "FavoriteTime": "2001-12-19T23:59:59", + "IsMagic": true, + "Name": "Gandalf", + "NextVacation": "2666-06-06", + "Spells": [ + { + "DiscoveredBy": "Firebreather", + "Icon": "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAgCAYAAAAFQMh/AAAAAXNSR0IArs4c6QAAA9lJREFUSA21lk9OVEEQxvsRDImoiMG9mLjjCG5mEg7gEfQGsIcF7p0EDsBBSJiNO7ZsFRZqosb/QkSj7fer7ur33sw8GDFUUq+7q6vqq6qu7pkQzqG4EeI521e7FePVgM9cGPYwhCi6UO8qFOK+YY+Br66ujsmmxb84Yzwp6zCsxjJfWVkxnMsEMGuWHZ9Wcz11cM48hkq0vLwc1tbW4mAwqDpcdIqnMmgF0JMv2CiGnZ2dcHR0FA4PD8Pe3t5U/tx6bCSlb+JT8XfxT3HsUek0Li0tRdjWl+z6iRF+FNA1hXPDQ/IMNyRg3s8bD/OaZS+VP+9cOLSa64cA34oXZWagDkRzAaJxXaE+ufc4rCN7LrazZ2+8+STtpAL8WYDvpTaHKlkB2iQARMvb2+H27m4YaL7zaDtUw1BZAASi6T8T2UZnPZV2pvnJfCH5p8bewcGB6TrIfz8wBZgHQ83kjpuj6RBYQpuo09Tvmpd7TPe+ktZN8cKwS92KWXGuaqWowlYEwthtMcWOZUNJc8at+zuF/Xkqo69baS7P+AvWjYwJ4jyHXXsEnd74ZO/Pq+uXUuv6WNlso6cvnDsZB1V/unJab3D1/KrJDw9NCM9wHf2FK2ejTKMejnBHfGtfH7LGGCdQDqaqJgfgzWjXK1nYV4jRbPGnxUT7cqUaZfJrVZeOm9QmB21L6xXgbu/ScsYusJFMoU0x2fsamRJOd6kOYDRLUxv94ENZe8+0gM+0dyz+KgU7X8rLHHCIOZyrna4y6ykIu0YCs02TBXmk3PZssmEgaTxTo83xjCIjoE21h0Yah3MrV4+9kR8MaabGze+9NEILGAFE5nMOiiA32KnAr/sb7tED3nzlzC4dB38WMC+EjaqHfqvUKHi2gJPdWQ6AbH8hgyQ7QY6jvjj3QZWvX6pUAtduTX5Dss96Q7NI9RQRJeeKvRFbt0v2gb1Gx/PooJsztn1c1DqpAU3Hde2dB2aEHBhjgOFjMeDvxLafjQ3YZQSgOcHJZX611H45sGLHWvYTz9hiURlpNoBZvxb/Ft9lAQ1DmBfUiR+j1hAPkMBTE9L9+zLva1QvGFHurRBaZ5xLVitoBviiRkD/sIMDztKA5FA0b9/0OclzO2/XAQymJ0TcghZwEo9/AX8gMeAJMOvIsWWt5bwCoiFhVSllrdH0t5Q1JHAFlKJNkvTVdn2GHb9KdmacMT+d/Os05imJUccRX2YuZ93Sxf0Ilc4DPDeAq5SAvFEAY94cQc6BA26dzb4HWAJI4DPmQE5KCVUyvb2FcDZem7JdT2ggKUP3xX6n9XNq1DpzSf4Cy4ZqSlmM8d8AAAAASUVORK5CYII=", + "LastUsed": "2015-10-31 23:59:56 UTC", + "Name": "Skydragon", + "Properties": [ + { + "Name": "Flying", + "Power": 1 + }, + { + "Name": "Creature", + "Power": 1 + }, + { + "Name": "Explodey", + "Power": 11 + } + ] + } + ], + "TeaTime": "15:00:00", + "Weight": 198.6 + }, + { + "Age": "17", + "FavoriteTime": "2000-10-31T23:27:46", + "IsMagic": true, + "Name": "Sabrina", + "NextVacation": "2017-03-14", + "Spells": [ + { + "DiscoveredBy": "Salem", + "Icon": "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAABxpRE9UAAAAAgAAAAAAAAAgAAAAKAAAACAAAAAgAAABxj2CfowAAAGSSURBVHgB7Jc9TsNAEIX3JDkCPUV6KlpKFHEGlD4nyA04ACUXQKTgCEipUnKGNEbP0otentayicZ24SlWs7tjO/N9u/5J2b2+NUtuZcnwYE8BuQPyGZAPwXwLLPk5kG+BJa9+fgfkh1B+CeancL4F8i2Q/wWm/S/w+XFoTseftn0dvhu0OXfhpM+AGvzcEiYVAFisPqE9zrETJhHAlXfg2lglMK9z0f3RBfB+ZyRUV3x+erzsEIjjOBqc1xtNAIrvguybV3A9lkVHxlEE6GrrPb/ZvAySwlUnfCmlPQ+R8JCExvGtcRQBLFwj4FGkznX1VYDKPG/f2/MjwCksXACgdNUxJjwK9xwl4JihOwTFR0kIF+CABEPRnvsvPFctMoYKqAFSAFaMwB4pp3Y+bodIYL9WmIAaIOHxo7W8wiHvAjTvhUeNwwSgeAeAABbqOewC5hBdwFD4+9+7puzXV9fS6/b1wwT4tsaYAhwOOQdUQch5vgZCeAhAv3ZM31yYAAUgvApQQQ6n5w6FB/RVe1jdJOAPAAD//1eMQwoAAAGQSURBVO1UMU4DQQy8X9AgWopIUINEkS4VlJQo4gvwAV7AD3gEH4iSgidESpWSXyyZExP5lr0c7K5PsXBhec/2+jzjuWtent9CLdtu1mG5+gjz+WNr7IsY7eH+tvO+xfuqk4vz7CH91edFaF5v9nb6dBKm13edvrL+0Lk5lMzJkQDeJSkkgHF6mR8CHwMHCQR/NAQQGD0BAlwK4FCefQiefq+A2Vn29tG7igLAfmwcnJu/nJy3BMQkMN9HEPr8AL3bfBv7Bp+7/SoExMDjZwKEJwmyhnnmQIQEBIlz2x0iKoAvJkAC6TsTIH6MqRrEWUMSZF2zAwqT4Eu/e6pzFAIkmNSZ4OFT+VYBIIF//UqbJwnF/4DU0GwOn8r/JQYCpPGufEfJuZiA37ycQw/5uFeqPq4pfR6FADmkBCXjfWdZj3NfXW58dAJyB9W65wRoMWulryvAyqa05nQFaDFrpa8rwMqmtOZ0BWgxa6WvK8DKprTmdAVoMWulryvAyqa05nQFaDFrpa8rwMqmtOb89wr4AtQ4aPoL6yVpAAAAAElFTkSuQmCC", + "LastUsed": "2017-02-14 12:07:23 UTC", + "Name": "Talking cats", + "Properties": [ + { + "Name": "Makes you look crazy", + "Power": 1 + } + ] + } + ], + "TeaTime": "12:00:00", + "Weight": 128.3 + } +] \ No newline at end of file diff --git a/system-test/data/schema.json b/system-test/data/schema.json new file mode 100644 index 00000000..165859b1 --- /dev/null +++ b/system-test/data/schema.json @@ -0,0 +1,81 @@ +[ + { + "mode": "NULLABLE", + "name": "Name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "Age", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "Weight", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "IsMagic", + "type": "BOOLEAN" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "Name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "LastUsed", + "type": "TIMESTAMP" + }, + { + "mode": "NULLABLE", + "name": "DiscoveredBy", + "type": "STRING" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "Name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "Power", + "type": "FLOAT" + } + ], + "mode": "REPEATED", + "name": "Properties", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "Icon", + "type": "BYTES" + } + ], + "mode": "REPEATED", + "name": "Spells", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "TeaTime", + "type": "TIME" + }, + { + "mode": "NULLABLE", + "name": "NextVacation", + "type": "DATE" + }, + { + "mode": "NULLABLE", + "name": "FavoriteTime", + "type": "DATETIME" + } +] \ No newline at end of file diff --git a/test/index.js b/test/index.js index 7cd6a706..ed405f4d 100644 --- a/test/index.js +++ b/test/index.js @@ -183,7 +183,8 @@ describe('BigQuery', function() { }); describe('datetime', function() { - var INPUT_STRING = '2017-1-1 14:2:38.883388'; + var INPUT_STRING = '2017-1-1T14:2:38.883388Z'; + var INPUT_OBJ = { year: 2017, month: 1, @@ -194,24 +195,21 @@ describe('BigQuery', function() { fractional: 883388 }; + var EXPECTED_VALUE = '2017-1-1 14:2:38.883388'; + it('should expose static and instance constructors', function() { - var staticDt = BigQuery.datetime(); + var staticDt = BigQuery.datetime(INPUT_OBJ); assert(staticDt instanceof BigQuery.datetime); assert(staticDt instanceof bq.datetime); - var instanceDt = bq.datetime(); + var instanceDt = bq.datetime(INPUT_OBJ); assert(instanceDt instanceof BigQuery.datetime); assert(instanceDt instanceof bq.datetime); }); - it('should accept a string', function() { - var datetime = bq.datetime(INPUT_STRING); - assert.strictEqual(datetime.value, INPUT_STRING); - }); - it('should accept an object', function() { var datetime = bq.datetime(INPUT_OBJ); - assert.strictEqual(datetime.value, INPUT_STRING); + assert.strictEqual(datetime.value, EXPECTED_VALUE); }); it('should not include time if hours not provided', function() { @@ -223,6 +221,11 @@ describe('BigQuery', function() { assert.strictEqual(datetime.value, '2016-1-1'); }); + + it('should accept a string', function() { + var datetime = bq.datetime(INPUT_STRING); + assert.strictEqual(datetime.value, EXPECTED_VALUE); + }); }); describe('time', function() { @@ -271,49 +274,37 @@ describe('BigQuery', function() { }); describe('timestamp', function() { - var INPUT_STRING = '2016-12-06 12:00:00.000+0'; + var INPUT_STRING = '2016-12-06T12:00:00.000Z'; var INPUT_DATE = new Date(INPUT_STRING); + var EXPECTED_VALUE = '2016-12-06 12:00:00.000'; it('should expose static and instance constructors', function() { - var staticT = BigQuery.timestamp(); + var staticT = BigQuery.timestamp(INPUT_DATE); assert(staticT instanceof BigQuery.timestamp); assert(staticT instanceof bq.timestamp); - var instanceT = bq.timestamp(); + var instanceT = bq.timestamp(INPUT_DATE); assert(instanceT instanceof BigQuery.timestamp); assert(instanceT instanceof bq.timestamp); }); it('should accept a string', function() { var timestamp = bq.timestamp(INPUT_STRING); - assert.strictEqual(timestamp.value, INPUT_STRING); + assert.strictEqual(timestamp.value, EXPECTED_VALUE); }); it('should accept a Date object', function() { var timestamp = bq.timestamp(INPUT_DATE); - assert.strictEqual(timestamp.value, INPUT_STRING.replace('+0', '')); - }); - - it('should default to now', function() { - var now = new Date(); - var timestamp = new Date(bq.timestamp().value + '+0'); - - var expectedTimestampBoundaries = { - start: new Date(now.getTime() - 1000), - end: new Date(now.getTime() + 1000) - }; - - assert(timestamp >= expectedTimestampBoundaries.start); - assert(timestamp <= expectedTimestampBoundaries.end); + assert.strictEqual(timestamp.value, EXPECTED_VALUE); }); }); describe('getType_', function() { it('should return correct types', function() { assert.strictEqual(BigQuery.getType_(bq.date()).type, 'DATE'); - assert.strictEqual(BigQuery.getType_(bq.datetime()).type, 'DATETIME'); + assert.strictEqual(BigQuery.getType_(bq.datetime('')).type, 'DATETIME'); assert.strictEqual(BigQuery.getType_(bq.time()).type, 'TIME'); - assert.strictEqual(BigQuery.getType_(bq.timestamp()).type, 'TIMESTAMP'); + assert.strictEqual(BigQuery.getType_(bq.timestamp(0)).type, 'TIMESTAMP'); assert.strictEqual(BigQuery.getType_(new Buffer(2)).type, 'BYTES'); assert.strictEqual(BigQuery.getType_(true).type, 'BOOL'); assert.strictEqual(BigQuery.getType_(8).type, 'INT64'); @@ -1034,8 +1025,9 @@ describe('BigQuery', function() { var rows = [{ row: 'a' }, { row: 'b' }, { row: 'c' }]; var schema = [{ fields: [] }]; - mergeSchemaWithRowsOverride = function(s, r) { + mergeSchemaWithRowsOverride = function(bq, s, r) { mergeSchemaWithRowsOverride = null; + assert.strictEqual(bq, BigQuery); assert.deepEqual(s, schema); assert.deepEqual(r, rows); done(); diff --git a/test/table.js b/test/table.js index dd949268..44538608 100644 --- a/test/table.js +++ b/test/table.js @@ -69,11 +69,43 @@ function FakeServiceObject() { nodeutil.inherits(FakeServiceObject, ServiceObject); +function fakeDate(input) { + return { + type: 'fakeDate', + input: input + }; +} + +function fakeDatetime(input) { + return { + type: 'fakeDatetime', + input: input + }; +} + +function fakeTime(input) { + return { + type: 'fakeTime', + input: input + }; +} + +function fakeTimestamp(input) { + return { + type: 'fakeTimestamp', + input: input + }; +} + describe('BigQuery/Table', function() { var DATASET = { id: 'dataset-id', createTable: util.noop, bigQuery: { + date: fakeDate, + datetime: fakeDatetime, + time: fakeTime, + timestamp: fakeTimestamp, projectId: 'project-id', job: function(id) { return { id: id }; @@ -274,13 +306,19 @@ describe('BigQuery/Table', function() { } } ] - } + }, + { v: 'date-input' }, + { v: 'datetime-input' }, + { v: 'time-input' } ] }, expected: { id: 3, name: 'Milo', - dob: now, + dob: { + input: now, + type: 'fakeTimestamp' + }, has_claws: false, hair_count: 5.222330009847, arr: [10], @@ -292,7 +330,19 @@ describe('BigQuery/Table', function() { nested_property: 'nested_value' } } - ] + ], + date: { + input: 'date-input', + type: 'fakeDate' + }, + datetime: { + input: 'datetime-input', + type: 'fakeDatetime' + }, + time: { + input: 'time-input', + type: 'fakeTime' + } } } ]; @@ -334,8 +384,27 @@ describe('BigQuery/Table', function() { ] }); + schemaObject.fields.push({ + name: 'date', + type: 'DATE' + }); + + schemaObject.fields.push({ + name: 'datetime', + type: 'DATETIME' + }); + + schemaObject.fields.push({ + name: 'time', + type: 'TIME' + }); + var rawRows = rows.map(prop('raw')); - var mergedRows = Table.mergeSchemaWithRows_(schemaObject, rawRows); + var mergedRows = Table.mergeSchemaWithRows_( + DATASET.bigQuery, + schemaObject, + rawRows + ); mergedRows.forEach(function(mergedRow, index) { assert.deepEqual(mergedRow, rows[index].expected);