Skip to content

Commit

Permalink
feat(job): regionalization support (#81)
Browse files Browse the repository at this point in the history
  • Loading branch information
@callmehiphop @stephenplusplus
callmehiphop authored and stephenplusplus committed Apr 6, 2018
1 parent f64f949 commit 1269b8c
Show file tree
Hide file tree
Showing 10 changed files with 809 additions and 139 deletions.
53 changes: 46 additions & 7 deletions src/dataset.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,22 @@ var Table = require('./table.js');
* @class
* @param {BigQuery} bigQuery {@link BigQuery} instance.
* @param {string} id The ID of the Dataset.
* @param {object} [options] Dataset options.
* @param {string} [options.location] The geographic location of the dataset.
* Defaults to US.
*
* @example
* const BigQuery = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('institutions');
*/
function Dataset(bigQuery, id) {
function Dataset(bigQuery, id, options) {
var self = this;

if (options && options.location) {
this.location = options.location;
}

var methods = {
/**
* Create a dataset.
Expand Down Expand Up @@ -199,8 +208,16 @@ function Dataset(bigQuery, id) {
parent: bigQuery,
baseUrl: '/datasets',
id: id,
createMethod: bigQuery.createDataset.bind(bigQuery),
methods: methods,
createMethod: function(id, options, callback) {
if (is.fn(options)) {
callback = options;
options = {};
}

options = extend({}, options, {location: self.location});
return bigQuery.createDataset(id, options, callback);
},
});

this.bigQuery = bigQuery;
Expand Down Expand Up @@ -244,6 +261,7 @@ Dataset.prototype.createQueryJob = function(options, callback) {
defaultDataset: {
datasetId: this.id,
},
location: this.location,
});

return this.bigQuery.createQueryJob(options, callback);
Expand All @@ -270,6 +288,7 @@ Dataset.prototype.createQueryStream = function(options) {
defaultDataset: {
datasetId: this.id,
},
location: this.location,
});

return this.bigQuery.createQueryStream(options);
Expand Down Expand Up @@ -345,9 +364,11 @@ Dataset.prototype.createTable = function(id, options, callback) {
return;
}

var table = self.table(resp.tableReference.tableId);
table.metadata = resp;
var table = self.table(resp.tableReference.tableId, {
location: resp.location,
});

table.metadata = resp;
callback(null, table, resp);
}
);
Expand Down Expand Up @@ -484,7 +505,10 @@ Dataset.prototype.getTables = function(options, callback) {
}

var tables = (resp.tables || []).map(function(tableObject) {
var table = that.table(tableObject.tableReference.tableId);
var table = that.table(tableObject.tableReference.tableId, {
location: tableObject.location,
});

table.metadata = tableObject;
return table;
});
Expand Down Expand Up @@ -545,6 +569,7 @@ Dataset.prototype.query = function(options, callback) {
defaultDataset: {
datasetId: this.id,
},
location: this.location,
});

return this.bigQuery.query(options, callback);
Expand All @@ -554,6 +579,13 @@ Dataset.prototype.query = function(options, callback) {
* Create a Table object.
*
* @param {string} id The ID of the table.
* @param {object} [options] Table options.
* @param {string} [options.location] The geographic location of the table, by
* default this value is inherited from the dataset. This can be used to
* configure the location of all jobs created through a table instance. It
* cannot be used to set the actual location of the table. This value will
* be superseded by any API responses containing location data for the
* table.
* @return {Table}
*
* @example
Expand All @@ -563,8 +595,15 @@ Dataset.prototype.query = function(options, callback) {
*
* const institutions = dataset.table('institution_data');
*/
Dataset.prototype.table = function(id) {
return new Table(this, id);
Dataset.prototype.table = function(id, options) {
options = extend(
{
location: this.location,
},
options
);

return new Table(this, id, options);
};

/*! Developer Documentation
Expand Down
87 changes: 74 additions & 13 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ var Table = require('./table.js');
* attempted before returning the error.
* @property {Constructor} [promise] Custom promise module to use instead of
* native Promises.
* @property {string} [location] The geographic location of all datasets and
* jobs referenced and created through the client.
*/

/**
Expand Down Expand Up @@ -102,6 +104,12 @@ function BigQuery(options) {
};

common.Service.call(this, config, options);

/**
* @name BigQuery#location
* @type {string}
*/
this.location = options.location;
}

util.inherits(BigQuery, common.Service);
Expand Down Expand Up @@ -617,11 +625,18 @@ BigQuery.prototype.createDataset = function(id, options, callback) {
{
method: 'POST',
uri: '/datasets',
json: extend(true, {}, options, {
datasetReference: {
datasetId: id,
json: extend(
true,
{
location: this.location,
},
}),
options,
{
datasetReference: {
datasetId: id,
},
}
),
},
function(err, resp) {
if (err) {
Expand Down Expand Up @@ -654,6 +669,8 @@ BigQuery.prototype.createDataset = function(id, options, callback) {
* @param {boolean} [options.dryRun] If set, don't actually run this job. A
* valid query will update the job with processing statistics. These can be
* accessed via `job.metadata`.
* @param {string} [options.location] The geographic location of the job.
* Required except for US and EU.
* @param {string} [options.jobId] Custom job id.
* @param {string} [options.jobPrefix] Prefix to apply to the job id.
* @param {string} options.query A query string, following the BigQuery query
Expand Down Expand Up @@ -779,6 +796,11 @@ BigQuery.prototype.createQueryJob = function(options, callback) {
delete query.jobPrefix;
}

if (query.location) {
reqOpts.location = query.location;
delete query.location;
}

if (query.jobId) {
reqOpts.jobId = query.jobId;
delete query.jobId;
Expand Down Expand Up @@ -840,6 +862,8 @@ BigQuery.prototype.createQueryStream = common.paginator.streamify('query');
* @param {object} options Object in the form of a [Job resource](https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs);
* @param {string} [options.jobId] Custom job id.
* @param {string} [options.jobPrefix] Prefix to apply to the job id.
* @param {string} [options.location] The geographic location of the job.
* Required except for US and EU.
* @param {function} [callback] The callback function.
* @param {?error} callback.err An error returned while making this request.
* @param {Job} callback.job The newly created job.
Expand Down Expand Up @@ -893,8 +917,14 @@ BigQuery.prototype.createJob = function(options, callback) {
reqOpts.jobReference = {
projectId: this.projectId,
jobId: jobId,
location: this.location,
};

if (options.location) {
reqOpts.jobReference.location = options.location;
delete reqOpts.location;
}

this.request(
{
method: 'POST',
Expand All @@ -907,10 +937,19 @@ BigQuery.prototype.createJob = function(options, callback) {
return;
}

var job = self.job(jobId);
job.metadata = resp;
if (resp.status.errors) {
err = new common.util.ApiError({
errors: resp.status.errors,
response: resp,
});
}

var job = self.job(jobId, {
location: resp.jobReference.location,
});

callback(null, job, resp);
job.metadata = resp;
callback(err, job, resp);
}
);
};
Expand All @@ -919,15 +958,22 @@ BigQuery.prototype.createJob = function(options, callback) {
* Create a reference to a dataset.
*
* @param {string} id ID of the dataset.
* @param {object} [options] Dataset options.
* @param {string} [options.location] The geographic location of the dataset.
* Required except for US and EU.
* @returns {Dataset}
*
* @example
* const BigQuery = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const dataset = bigquery.dataset('higher_education');
*/
BigQuery.prototype.dataset = function(id) {
return new Dataset(this, id);
BigQuery.prototype.dataset = function(id, options) {
if (this.location) {
options = extend({location: this.location}, options);
}

return new Dataset(this, id, options);
};

/**
Expand Down Expand Up @@ -1008,7 +1054,10 @@ BigQuery.prototype.getDatasets = function(options, callback) {
}

var datasets = (resp.datasets || []).map(function(dataset) {
var ds = that.dataset(dataset.datasetReference.datasetId);
var ds = that.dataset(dataset.datasetReference.datasetId, {
location: dataset.location,
});

ds.metadata = dataset;
return ds;
});
Expand Down Expand Up @@ -1141,7 +1190,10 @@ BigQuery.prototype.getJobs = function(options, callback) {
}

var jobs = (resp.jobs || []).map(function(jobObject) {
var job = that.job(jobObject.jobReference.jobId);
var job = that.job(jobObject.jobReference.jobId, {
location: jobObject.jobReference.location,
});

job.metadata = jobObject;
return job;
});
Expand Down Expand Up @@ -1187,6 +1239,9 @@ BigQuery.prototype.getJobsStream = common.paginator.streamify('getJobs');
* Create a reference to an existing job.
*
* @param {string} id ID of the job.
* @param {object} [options] Configuration object.
* @param {string} [options.location] The geographic location of the job.
* Required except for US and EU.
* @returns {Job}
*
* @example
Expand All @@ -1195,8 +1250,12 @@ BigQuery.prototype.getJobsStream = common.paginator.streamify('getJobs');
*
* const myExistingJob = bigquery.job('job-id');
*/
BigQuery.prototype.job = function(id) {
return new Job(this, id);
BigQuery.prototype.job = function(id, options) {
if (this.location) {
options = extend({location: this.location}, options);
}

return new Job(this, id, options);
};

/**
Expand All @@ -1208,6 +1267,8 @@ BigQuery.prototype.job = function(id) {
* @param {string|object} query A string SQL query or configuration object.
* For all available options, see
* [Jobs: query request body](https://cloud.google.com/bigquery/docs/reference/v2/jobs/query#request-body).
* @param {string} [query.location] The geographic location of the job.
* Required except for US and EU.
* @param {string} [query.jobId] Custom id for the underlying job.
* @param {string} [query.jobPrefix] Prefix to apply to the underlying job id.
* @param {object|Array<*>} query.params For positional SQL parameters, provide
Expand Down
Loading

0 comments on commit 1269b8c

Please sign in to comment.