Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX] Csv importer: work with more problematic data #7456

Merged
merged 5 commits into from
Aug 8, 2017
Merged

[FIX] Csv importer: work with more problematic data #7456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
147e1a8
CSV import: use existing users&rooms if no files for them were supplied
reist Jul 9, 2017
b67e199
CSV import: bump included csv-parse module to fix import with quotes
reist Jul 9, 2017
8caa0a2
CSV import: unique _id should be fine with identical timestamps
reist Jul 9, 2017
9361071
Import: add message count to UI and hide empty blocks
reist Jul 20, 2017
1440fe3
CSV Import: Redo the logic for repeating timestamp
reist Aug 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/rocketchat-importer-csv/package.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ Package.onUse(function(api) {
});

Npm.depends({
'csv-parse': '1.1.7'
'csv-parse': '1.2.0'
});
61 changes: 57 additions & 4 deletions packages/rocketchat-importer-csv/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,10 @@ Importer.CSV = class ImporterCSV extends Importer.Base {

const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = tempChannels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

super.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

startImport(importSelection) {
Expand Down Expand Up @@ -232,16 +233,60 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
}
this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }});

//If no channels file, collect channel map from DB for message-only import
if (this.channels.channels.length === 0) {
for (const cname of this.messages.keys()) {
Meteor.runAsUser(startedByUserId, () => {
const existantRoom = RocketChat.models.Rooms.findOneByName(cname);
if (existantRoom || cname.toUpperCase() === 'GENERAL') {
this.channels.channels.push({
id: cname.replace('.', '_'),
name: cname,
rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id),
do_import: true
});
}
});
}
}

//If no users file, collect user map from DB for message-only import
if (this.users.users.length === 0) {
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel || !csvChannel.do_import) {
continue;
}
Meteor.runAsUser(startedByUserId, () => {
for (const msgs of messagesMap.values()) {
for (const msg of msgs.messages) {
if (!this.getUserFromUsername(msg.username)) {
const user = RocketChat.models.Users.findOneByUsername(msg.username);
if (user) {
this.users.users.push({
rocketId: user._id,
username: user.username
});
}
}
}
}
});
}
}


//Import the Messages
super.updateProgress(Importer.ProgressStep.IMPORTING_MESSAGES);
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel.do_import) {
if (!csvChannel || !csvChannel.do_import) {
continue;
}

const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } });
Meteor.runAsUser(startedByUserId, () => {
const timestamps = {};
for (const [msgGroupData, msgs] of messagesMap.entries()) {
super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` });
for (const msg of msgs.messages) {
Expand All @@ -253,8 +298,15 @@ Importer.CSV = class ImporterCSV extends Importer.Base {

const creator = this.getUserFromUsername(msg.username);
if (creator) {
let suffix = '';
if (timestamps[msg.ts] === undefined) {
timestamps[msg.ts] = 1;
} else {
suffix = `-${ timestamps[msg.ts] }`;
timestamps[msg.ts] += 1;
}
const msgObj = {
_id: `csv-${ csvChannel.id }-${ msg.ts }`,
_id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`,
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
rid: room._id,
Expand Down Expand Up @@ -285,8 +337,9 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

getChannelFromName(channelName) {
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-hipchat-enterprise/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,10 +190,11 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba

const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true));
const selectionChannels = tempRooms.map((r) => new Importer.SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate));
const selectionMessages = this.importRecord.count.messages;

super.updateProgress(Importer.ProgressStep.USER_SELECTION);

resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels));
resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages));
}));

//Wish I could make this cleaner :(
Expand Down Expand Up @@ -431,8 +432,9 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

getChannelFromRoomIdentifier(roomIdentifier) {
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-hipchat/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,9 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = tempRooms.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

startImport(importSelection) {
Expand Down Expand Up @@ -331,7 +332,8 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = this.channels.channels.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

}
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-slack/server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,9 @@ Importer.Slack = class extends Importer.Base {
}
const selectionUsers = tempUsers.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = tempChannels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
startImport(importSelection) {
super.startImport(importSelection);
Expand Down Expand Up @@ -429,6 +430,7 @@ Importer.Slack = class extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = this.channels.channels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
};
56 changes: 32 additions & 24 deletions packages/rocketchat-importer/client/admin/adminImportPrepare.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,35 +33,43 @@ <h1>{{_ "Actions"}}</h1>
</div>
</div>

<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
{{#if users.length}}
<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
</div>
</div>
{{/if}}

{{#if channels.length}}
<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
{{/each}}
</ul>
</div>
</div>
</div>
{{/if}}

<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/each}}
</ul>
</div>
<h1>{{_ "Messages"}}: {{message_count}}</h1>
</div>
{{else}}
{{#if isPreparing}}
Expand Down
6 changes: 6 additions & 0 deletions packages/rocketchat-importer/client/admin/adminImportPrepare.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ Template.adminImportPrepare.helpers({
},
channels() {
return Template.instance().channels.get();
},
message_count() {
return Template.instance().message_count.get();
}
});

Expand Down Expand Up @@ -70,6 +73,7 @@ Template.adminImportPrepare.events({

template.users.set(data.users);
template.channels.set(data.channels);
template.message_count.set(data.message_count);
template.loaded.set(true);
template.preparing.set(false);
});
Expand Down Expand Up @@ -131,6 +135,7 @@ Template.adminImportPrepare.onCreated(function() {
this.loaded = new ReactiveVar(false);
this.users = new ReactiveVar([]);
this.channels = new ReactiveVar([]);
this.message_count = new ReactiveVar(0);

function loadSelection(progress) {
if ((progress != null ? progress.step : undefined)) {
Expand All @@ -146,6 +151,7 @@ Template.adminImportPrepare.onCreated(function() {
}
instance.users.set(data.users);
instance.channels.set(data.channels);
instance.message_count.set(data.message_count);
instance.loaded.set(true);
return instance.preparing.set(false);
});
Expand Down
4 changes: 3 additions & 1 deletion packages/rocketchat-importer/server/classes/ImporterSelection.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ Importer.Selection = (Importer.Selection = class Selection {
// @param [String] name the name of the Importer
// @param [Array<Importer.User>] users the array of users
// @param [Array<Importer.Channel>] channels the array of channels
// @param [Integer] number of collected messages
//
constructor(name, users, channels) {
constructor(name, users, channels, message_count) {
this.name = name;
this.users = users;
this.channels = channels;
this.message_count = message_count;
}
});