Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add copyTable sample #198

Merged
merged 2 commits into from
Sep 1, 2016
Merged

Add copyTable sample #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e6cf48
First draft of copyTable sample
Aug 30, 2016
c7f8a7d
Address comments
Sep 1, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 51 additions & 17 deletions bigquery/system-test/tables.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,37 +36,47 @@ var options = {
schema: 'Name:string, Age:integer, Weight:float, IsMagic:boolean',
rows: rows
};
var srcDataset = options.dataset;
var srcTable = options.table;
var destDataset = generateUuid();
var destTable = generateUuid();

describe('bigquery:tables', function () {
before(function (done) {
// Create bucket
storage.createBucket(options.bucket, function (err, bucket) {
assert.ifError(err, 'bucket creation succeeded');

// Upload data.csv
bucket.upload(options.localFilePath, function (err) {
assert.ifError(err, 'file upload succeeded');

// Create dataset
bigquery.createDataset(options.dataset, function (err, dataset) {
assert.ifError(err, 'dataset creation succeeded');
done();
// Create srcDataset
bigquery.createDataset(srcDataset, function (err) {
assert.ifError(err, 'srcDataset creation succeeded');
// Create destDataset
bigquery.createDataset(destDataset, function (err) {
assert.ifError(err, 'destDataset creation succeeded');
done();
});
});
});
});
});

after(function (done) {
// Delete testing dataset/table
bigquery.dataset(options.dataset).delete({ force: true }, function () {
// Delete files
storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) {
if (err) {
return done(err);
}
// Delete bucket
setTimeout(function () {
storage.bucket(options.bucket).delete(done);
}, 2000);
// Delete srcDataset
bigquery.dataset(srcDataset).delete({ force: true }, function () {
// Delete destDataset
bigquery.dataset(destDataset).delete({ force: true }, function () {
// Delete files
storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) {
if (err) {
return done(err);
}
// Delete bucket
setTimeout(function () {
storage.bucket(options.bucket).delete(done);
}, 2000);
});
});
});
});
Expand Down Expand Up @@ -157,6 +167,30 @@ describe('bigquery:tables', function () {
});
});

describe('copyTable', function () {
it('should copy a table between datasets', function (done) {
program.copyTable(srcDataset, srcTable, destDataset, destTable, function (err, metadata) {
assert.equal(err, null);
assert.deepEqual(metadata.status, { state: 'DONE' });

bigquery.dataset(srcDataset).table(srcTable).exists(
function (err, exists) {
assert.equal(err, null);
assert.equal(exists, true, 'srcTable exists');

bigquery.dataset(destDataset).table(destTable).exists(
function (err, exists) {
assert.equal(err, null);
assert.equal(exists, true, 'destTable exists');
done();
}
);
}
);
});
});
});

describe('deleteTable', function () {
it('should delete table', function (done) {
program.deleteTable(options, function (err) {
Expand Down
38 changes: 38 additions & 0 deletions bigquery/tables.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,27 @@ function deleteTable (options, callback) {
}
// [END delete_table]

function copyTable (srcDataset, srcTable, destDataset, destTable, callback) {
var bigquery = BigQuery();

var srcTableObj = bigquery.dataset(srcDataset).table(srcTable);
var destTableObj = bigquery.dataset(destDataset).table(destTable);

srcTableObj.copy(destTableObj, function (err, job) {
if (err) {
return callback(err);
}

console.log('Started job: %s', job.id);
job
.on('error', callback)
.on('complete', function (metadata) {
console.log('Completed job: %s', job.id);
return callback(null, metadata);
});
});
}

// [START import_file]
/**
* Load a csv file into a BigQuery table.
Expand Down Expand Up @@ -219,6 +240,7 @@ var program = module.exports = {
importFile: importFile,
exportTableToGCS: exportTableToGCS,
insertRowsAsStream: insertRowsAsStream,
copyTable: copyTable,
main: function (args) {
// Run the command-line program
cli.help().strict().parse(args).argv;
Expand All @@ -236,6 +258,18 @@ cli
.command('delete <dataset> <table>', 'Delete a table in the specified dataset.', {}, function (options) {
program.deleteTable(utils.pick(options, ['dataset', 'table']), utils.makeHandler());
})
.command('copy <srcDataset> <srcTable> <destDataset> <destTable>',
'Make a copy of an existing table.', {},
function (options) {
program.copyTable(
options.srcDataset,
options.srcTable,
options.destDataset,
options.destTable,
utils.makeHandler()
);
}
)
.command('import <dataset> <table> <file>', 'Import data from a local file or a Google Cloud Storage file into BigQuery.', {
bucket: {
alias: 'b',
Expand Down Expand Up @@ -325,6 +359,10 @@ cli
'node $0 insert my_dataset my_table json_file',
'Insert the JSON objects contained in json_file (one per line) into my_dataset:my_table.'
)
.example(
'node $0 copy src_dataset src_table dest_dataset dest_table',
'Copy src_dataset:src_table to dest_dataset:dest_table.'
)
.wrap(100)
.recommendCommands()
.epilogue('For more information, see https://cloud.google.com/bigquery/docs');
Expand Down
56 changes: 51 additions & 5 deletions bigquery/test/tables.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,13 @@
var proxyquire = require('proxyquire').noCallThru();
var bucket = 'bucket';
var file = 'file';
var job = 'job';
var jobId = 'job';
var dataset = 'dataset';
var table = 'table';
var srcDataset = dataset;
var srcTable = table;
var destDataset = dataset + '_dest';
var destTable = table + '_dest';
var format = 'JSON';
var schema = 'schema';
var jsonArray = [
Expand Down Expand Up @@ -46,12 +50,14 @@ function getSample () {
var fileMock = {};
var metadataMock = { status: { state: 'DONE' } };
var jobMock = {
id: job,
id: jobId,
getMetadata: sinon.stub().yields(null, metadataMock),
on: sinon.stub().returnsThis()
};
jobMock.on.withArgs('complete').yields(metadataMock);
var tableMock = {
export: sinon.stub().yields(null, jobMock),
copy: sinon.stub().yields(null, jobMock),
delete: sinon.stub().yields(null),
import: sinon.stub().yields(null, jobMock),
insert: sinon.stub().yields(null, errorList)
Expand Down Expand Up @@ -220,7 +226,6 @@ describe('bigquery:tables', function () {
table: table,
file: file
};
sample.mocks.job.on.withArgs('complete').yields(sample.mocks.metadata);

sample.program.importFile(options, callback);

Expand All @@ -243,7 +248,6 @@ describe('bigquery:tables', function () {
bucket: bucket,
format: format
};
sample.mocks.job.on.withArgs('complete').yields(sample.mocks.metadata);

sample.program.importFile(options, callback);

Expand All @@ -269,6 +273,38 @@ describe('bigquery:tables', function () {
});
});

describe('copyTable', function () {
it('should copy a table', function () {
var sample = getSample();
var callback = sinon.stub();

sample.program.copyTable(srcDataset, srcTable, destDataset, destTable, callback);

assert.equal(sample.mocks.table.copy.calledOnce, true);
assert.deepEqual(
sample.mocks.table.copy.firstCall.args.slice(0, -1),
[sample.mocks.table]
);
assert.equal(callback.calledOnce, true);
assert.deepEqual(callback.firstCall.args, [null, sample.mocks.metadata]);
assert.equal(console.log.calledTwice, true);
assert.equal(console.log.calledWith('Started job: %s', sample.mocks.job.id), true);
assert.equal(console.log.calledWith('Completed job: %s', sample.mocks.job.id), true);
});

it('should handle error', function () {
var error = new Error('error');
var sample = getSample();
var callback = sinon.stub();
sample.mocks.table.copy.yields(error);

sample.program.copyTable(srcDataset, srcTable, destDataset, destTable, callback);

assert.equal(callback.calledOnce, true);
assert.deepEqual(callback.firstCall.args, [error]);
});
});

describe('exportTableToGCS', function () {
it('should export to a table', function () {
var sample = getSample();
Expand All @@ -281,7 +317,6 @@ describe('bigquery:tables', function () {
gzip: true
};
var callback = sinon.stub();
sample.mocks.job.on.withArgs('complete').yields(sample.mocks.metadata);

sample.program.exportTableToGCS(options, callback);

Expand Down Expand Up @@ -389,6 +424,17 @@ describe('bigquery:tables', function () {
}]);
});

it('should call copyTable', function () {
var program = getSample().program;
program.copyTable = sinon.stub();

program.main(['copy', srcDataset, srcTable, destDataset, destTable]);
assert.equal(program.copyTable.calledOnce, true);
assert.deepEqual(program.copyTable.firstCall.args.slice(0, -1),
[srcDataset, srcTable, destDataset, destTable]
);
});

it('should call exportTableToGCS', function () {
var program = getSample().program;
program.exportTableToGCS = sinon.stub();
Expand Down