From 147e1a8087057fcd2c3a1d0b1cbf998ba85f4d7d Mon Sep 17 00:00:00 2001 From: Boris Peterbarg Date: Sun, 9 Jul 2017 18:56:35 +0300 Subject: [PATCH 1/5] CSV import: use existing users&rooms if no files for them were supplied --- packages/rocketchat-importer-csv/server.js | 45 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/rocketchat-importer-csv/server.js b/packages/rocketchat-importer-csv/server.js index f701075e6a6c..12ff1cc79396 100644 --- a/packages/rocketchat-importer-csv/server.js +++ b/packages/rocketchat-importer-csv/server.js @@ -232,11 +232,54 @@ Importer.CSV = class ImporterCSV extends Importer.Base { } this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + //If no channels file, collect channel map from DB for message-only import + if (this.channels.channels.length === 0) { + for (const cname of this.messages.keys()) { + Meteor.runAsUser(startedByUserId, () => { + const existantRoom = RocketChat.models.Rooms.findOneByName(cname); + if (existantRoom || cname.toUpperCase() === 'GENERAL') { + this.channels.channels.push({ + id: cname.replace('.', '_'), + name: cname, + rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id), + do_import: true + }); + } + }); + } + } + + //If no users file, collect user map from DB for message-only import + if (this.users.users.length === 0) { + for (const [ch, messagesMap] of this.messages.entries()) { + const csvChannel = this.getChannelFromName(ch); + if (!csvChannel || !csvChannel.do_import) { + continue; + } + Meteor.runAsUser(startedByUserId, () => { + for (const msgs of messagesMap.values()) { + for (const msg of msgs.messages) { + if (!this.getUserFromUsername(msg.username)) { + const user = RocketChat.models.Users.findOneByUsername(msg.username); + if (user) { + this.users.users.push({ + rocketId: user._id, + username: user.username + }); + } + } + } + } + }); + } + } + + //Import the Messages super.updateProgress(Importer.ProgressStep.IMPORTING_MESSAGES); for (const [ch, messagesMap] of this.messages.entries()) { const csvChannel = this.getChannelFromName(ch); - if (!csvChannel.do_import) { + if (!csvChannel || !csvChannel.do_import) { continue; } From b67e19900a0034d4cd761d0a6cf5f5e7141ac3d0 Mon Sep 17 00:00:00 2001 From: Boris Peterbarg Date: Sun, 9 Jul 2017 18:57:32 +0300 Subject: [PATCH 2/5] CSV import: bump included csv-parse module to fix import with quotes --- .../.npm/package/npm-shrinkwrap.json | 6 +++--- packages/rocketchat-importer-csv/package.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json b/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json index 7a0f451ebb1c..324c8efc5f33 100644 --- a/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json +++ b/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json @@ -1,9 +1,9 @@ { "dependencies": { "csv-parse": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.1.7.tgz", - "from": "csv-parse@1.1.7" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.2.0.tgz", + "from": "csv-parse@1.2.0" } } } diff --git a/packages/rocketchat-importer-csv/package.js b/packages/rocketchat-importer-csv/package.js index 33976fbeb100..fb11d108fb55 100644 --- a/packages/rocketchat-importer-csv/package.js +++ b/packages/rocketchat-importer-csv/package.js @@ -17,5 +17,5 @@ Package.onUse(function(api) { }); Npm.depends({ - 'csv-parse': '1.1.7' + 'csv-parse': '1.2.0' }); From 8caa0a2bf9490b5ab900e78e20b76dc1cca146ee Mon Sep 17 00:00:00 2001 From: Boris Peterbarg Date: Sun, 9 Jul 2017 19:02:21 +0300 Subject: [PATCH 3/5] CSV import: unique _id should be fine with identical timestamps Otherwise two messages that received the same timestamp in the same room will break the import. --- packages/rocketchat-importer-csv/server.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/rocketchat-importer-csv/server.js b/packages/rocketchat-importer-csv/server.js index 12ff1cc79396..68bb231e495d 100644 --- a/packages/rocketchat-importer-csv/server.js +++ b/packages/rocketchat-importer-csv/server.js @@ -285,6 +285,7 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); Meteor.runAsUser(startedByUserId, () => { + let roomCounter = 0; for (const [msgGroupData, msgs] of messagesMap.entries()) { super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { @@ -297,7 +298,7 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const creator = this.getUserFromUsername(msg.username); if (creator) { const msgObj = { - _id: `csv-${ csvChannel.id }-${ msg.ts }`, + _id: `csv-${ csvChannel.id }-${ roomCounter }-${ msg.ts }`, ts: new Date(parseInt(msg.ts)), msg: msg.text, rid: room._id, @@ -308,6 +309,7 @@ Importer.CSV = class ImporterCSV extends Importer.Base { }; RocketChat.sendMessage(creator, msgObj, room, true); + roomCounter += 1; } super.addCountCompleted(1); From 93610714bc298dd259d36878436c359eaa5147fb Mon Sep 17 00:00:00 2001 From: Boris Peterbarg Date: Thu, 20 Jul 2017 12:41:15 +0300 Subject: [PATCH 4/5] Import: add message count to UI and hide empty blocks Interface clean up, with users&channels being allowed to be empty for CSV import. --- packages/rocketchat-importer-csv/server.js | 6 +- .../server.js | 6 +- .../rocketchat-importer-hipchat/server.js | 6 +- packages/rocketchat-importer-slack/server.js | 6 +- .../client/admin/adminImportPrepare.html | 56 +++++++++++-------- .../client/admin/adminImportPrepare.js | 6 ++ .../server/classes/ImporterSelection.js | 4 +- 7 files changed, 57 insertions(+), 33 deletions(-) diff --git a/packages/rocketchat-importer-csv/server.js b/packages/rocketchat-importer-csv/server.js index 68bb231e495d..32b647d2b955 100644 --- a/packages/rocketchat-importer-csv/server.js +++ b/packages/rocketchat-importer-csv/server.js @@ -131,9 +131,10 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = tempChannels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; super.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { @@ -330,8 +331,9 @@ Importer.CSV = class ImporterCSV extends Importer.Base { getSelection() { const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } getChannelFromName(channelName) { diff --git a/packages/rocketchat-importer-hipchat-enterprise/server.js b/packages/rocketchat-importer-hipchat-enterprise/server.js index 40f3762df621..961cdd26b222 100644 --- a/packages/rocketchat-importer-hipchat-enterprise/server.js +++ b/packages/rocketchat-importer-hipchat-enterprise/server.js @@ -190,10 +190,11 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true)); const selectionChannels = tempRooms.map((r) => new Importer.SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate)); + const selectionMessages = this.importRecord.count.messages; super.updateProgress(Importer.ProgressStep.USER_SELECTION); - resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels)); + resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages)); })); //Wish I could make this cleaner :( @@ -431,8 +432,9 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba getSelection() { const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } getChannelFromRoomIdentifier(roomIdentifier) { diff --git a/packages/rocketchat-importer-hipchat/server.js b/packages/rocketchat-importer-hipchat/server.js index 89713ce84c11..007e90efa078 100644 --- a/packages/rocketchat-importer-hipchat/server.js +++ b/packages/rocketchat-importer-hipchat/server.js @@ -131,8 +131,9 @@ Importer.HipChat = Importer.HipChat = (function() { const selectionChannels = tempRooms.map(function(room) { return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false); }); + const selectionMessages = this.importRecord.count.messages; this.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { @@ -331,7 +332,8 @@ Importer.HipChat = Importer.HipChat = (function() { const selectionChannels = this.channels.channels.map(function(room) { return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false); }); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + const selectionMessages = this.importRecord.count.messages; + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } } diff --git a/packages/rocketchat-importer-slack/server.js b/packages/rocketchat-importer-slack/server.js index 94b719786faf..53a28f69572f 100644 --- a/packages/rocketchat-importer-slack/server.js +++ b/packages/rocketchat-importer-slack/server.js @@ -92,8 +92,9 @@ Importer.Slack = class extends Importer.Base { } const selectionUsers = tempUsers.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); const selectionChannels = tempChannels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); + const selectionMessages = this.importRecord.count.messages; this.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { super.startImport(importSelection); @@ -429,6 +430,7 @@ Importer.Slack = class extends Importer.Base { getSelection() { const selectionUsers = this.users.users.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); const selectionChannels = this.channels.channels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + const selectionMessages = this.importRecord.count.messages; + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } }; diff --git a/packages/rocketchat-importer/client/admin/adminImportPrepare.html b/packages/rocketchat-importer/client/admin/adminImportPrepare.html index cac461eb89e7..82efec25c976 100644 --- a/packages/rocketchat-importer/client/admin/adminImportPrepare.html +++ b/packages/rocketchat-importer/client/admin/adminImportPrepare.html @@ -33,35 +33,43 @@

{{_ "Actions"}}

-
-

{{_ "Users"}}

-
-
    - {{#each users}} - {{#unless is_bot}} + {{#if users.length}} +
    +

    {{_ "Users"}}

    +
    +
      + {{#each users}} + {{#unless is_bot}} +
    • + {{username}} - {{email}} + {{#if is_deleted }} ({{_ "Deleted"}}){{/if}} +
      • + {{/unless}} + {{/each}} +
    +
    +
    + {{/if}} + + {{#if channels.length}} +
    +

    {{_ "Channels"}}

    +
    +
      + {{#each channels}}
    • - {{username}} - {{email}} - {{#if is_deleted }} ({{_ "Deleted"}}){{/if}} + {{name}} + {{#if is_archived}} ({{_ "Importer_Archived"}}){{/if}} + {{#if is_private}} ({{_ "Private_Group"}}){{/if}}
      • - {{/unless}} - {{/each}} -
    + {{/each}} +
+
- + {{/if}}
-

{{_ "Channels"}}

-
-
    - {{#each channels}} -
  • - {{name}} - {{#if is_archived}} ({{_ "Importer_Archived"}}){{/if}} - {{#if is_private}} ({{_ "Private_Group"}}){{/if}} -
    • - {{/each}} -
-
+

{{_ "Messages"}}: {{message_count}}

{{else}} {{#if isPreparing}} diff --git a/packages/rocketchat-importer/client/admin/adminImportPrepare.js b/packages/rocketchat-importer/client/admin/adminImportPrepare.js index fe7e00ee045f..bdc11eb442f7 100644 --- a/packages/rocketchat-importer/client/admin/adminImportPrepare.js +++ b/packages/rocketchat-importer/client/admin/adminImportPrepare.js @@ -27,6 +27,9 @@ Template.adminImportPrepare.helpers({ }, channels() { return Template.instance().channels.get(); + }, + message_count() { + return Template.instance().message_count.get(); } }); @@ -70,6 +73,7 @@ Template.adminImportPrepare.events({ template.users.set(data.users); template.channels.set(data.channels); + template.message_count.set(data.message_count); template.loaded.set(true); template.preparing.set(false); }); @@ -131,6 +135,7 @@ Template.adminImportPrepare.onCreated(function() { this.loaded = new ReactiveVar(false); this.users = new ReactiveVar([]); this.channels = new ReactiveVar([]); + this.message_count = new ReactiveVar(0); function loadSelection(progress) { if ((progress != null ? progress.step : undefined)) { @@ -146,6 +151,7 @@ Template.adminImportPrepare.onCreated(function() { } instance.users.set(data.users); instance.channels.set(data.channels); + instance.message_count.set(data.message_count); instance.loaded.set(true); return instance.preparing.set(false); }); diff --git a/packages/rocketchat-importer/server/classes/ImporterSelection.js b/packages/rocketchat-importer/server/classes/ImporterSelection.js index 070af9a70be6..d1a45a56a972 100644 --- a/packages/rocketchat-importer/server/classes/ImporterSelection.js +++ b/packages/rocketchat-importer/server/classes/ImporterSelection.js @@ -6,10 +6,12 @@ Importer.Selection = (Importer.Selection = class Selection { // @param [String] name the name of the Importer // @param [Array] users the array of users // @param [Array] channels the array of channels + // @param [Integer] number of collected messages // - constructor(name, users, channels) { + constructor(name, users, channels, message_count) { this.name = name; this.users = users; this.channels = channels; + this.message_count = message_count; } }); From 1440fe38d396946285b2cd1486b9c2d793704d47 Mon Sep 17 00:00:00 2001 From: Boris Peterbarg Date: Wed, 2 Aug 2017 18:49:04 +0300 Subject: [PATCH 5/5] CSV Import: Redo the logic for repeating timestamp This way, it will re-import old imports just fine. For new imports, the order of timestamps in the CSV files will have to be the same, always. So, either using the same dumps or enforcing order on the CSV dump is required. --- packages/rocketchat-importer-csv/server.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/rocketchat-importer-csv/server.js b/packages/rocketchat-importer-csv/server.js index 32b647d2b955..3597f1df6616 100644 --- a/packages/rocketchat-importer-csv/server.js +++ b/packages/rocketchat-importer-csv/server.js @@ -286,7 +286,7 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); Meteor.runAsUser(startedByUserId, () => { - let roomCounter = 0; + const timestamps = {}; for (const [msgGroupData, msgs] of messagesMap.entries()) { super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { @@ -298,8 +298,15 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const creator = this.getUserFromUsername(msg.username); if (creator) { + let suffix = ''; + if (timestamps[msg.ts] === undefined) { + timestamps[msg.ts] = 1; + } else { + suffix = `-${ timestamps[msg.ts] }`; + timestamps[msg.ts] += 1; + } const msgObj = { - _id: `csv-${ csvChannel.id }-${ roomCounter }-${ msg.ts }`, + _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, ts: new Date(parseInt(msg.ts)), msg: msg.text, rid: room._id, @@ -310,7 +317,6 @@ Importer.CSV = class ImporterCSV extends Importer.Base { }; RocketChat.sendMessage(creator, msgObj, room, true); - roomCounter += 1; } super.addCountCompleted(1);