[NEW] GDPR - Right to access and Data Portability (#9906)

* Implemented basic JSON generation

* Added new data to message json

* Changed export operation structure

* Changed code to use forEach instead of fetch

* Split the operation into two independent commands

* File download, zip generation, admin settings

* Use syncedcron to process data downloads

* Added download URL

* Sending emails when the download file is ready

* Allow usage of GridFS as storage for the finished file.

* Lint

* Added support for Google and Amazon as storage types

* Split the options to download and export data

* Removed commented code

.meteor/packages

@@ -146,6 +146,7 @@ rocketchat:ui-master
 rocketchat:ui-message
 rocketchat:ui-sidenav
 rocketchat:ui-vrecord
+rocketchat:user-data-download
 rocketchat:version
 rocketchat:videobridge
 rocketchat:webrtc

.meteor/versions

@@ -237,6 +237,7 @@ rocketchat:[email protected]
 rocketchat:[email protected]
 rocketchat:[email protected]
 rocketchat:[email protected]
+rocketchat:[email protected]
 rocketchat:[email protected]
 rocketchat:[email protected]
 rocketchat:[email protected]

package.json

@@ -114,6 +114,7 @@
 		"@google-cloud/storage": "^1.6.0",
 		"@google-cloud/vision": "^0.15.2",
 		"adm-zip": "^0.4.7",
+		"archiver": "^2.1.1",
 		"atlassian-crowd": "^0.5.0",
 		"autolinker": "^1.6.2",
 		"aws-sdk": "^2.199.0",

packages/rocketchat-file-upload/client/lib/fileUploadHandler.js

@@ -17,7 +17,6 @@ new UploadFS.Store({
 	})
 });
 
-
 fileUploadHandler = (directive, meta, file) => {
 	const store = UploadFS.getStore(directive);
 

packages/rocketchat-file-upload/lib/FileUploadBase.js

@@ -4,7 +4,21 @@ import _ from 'underscore';
 
 UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({
 	insert(userId, doc) {
-		return userId || (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0); // allow inserts from slackbridge (message_id = slack-timestamp-milli)
+		if (userId) {
+			return true;
+		}
+
+		// allow inserts from slackbridge (message_id = slack-timestamp-milli)
+		if (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0) {
+			return true;
+		}
+
+		// allow inserts to the UserDataFiles store
+		if (doc && doc.store && doc.store.split(':').pop() === 'UserDataFiles') {
+			return true;
+		}
+
+		return false;
 	},
 	update(userId, doc) {
 		return RocketChat.authz.hasPermission(Meteor.userId(), 'delete-message', doc.rid) || (RocketChat.settings.get('Message_AllowDeleting') && userId === doc.userId);

packages/rocketchat-file-upload/server/config/AmazonS3.js

@@ -25,15 +25,35 @@ const get = function(file, req, res) {
 	}
 };
 
+const copy = function(file, out) {
+	const fileUrl = this.store.getRedirectURL(file);
+
+	if (fileUrl) {
+		const request = /^https:/.test(fileUrl) ? https : http;
+		request.get(fileUrl, fileRes => fileRes.pipe(out));
+	} else {
+		out.end();
+	}
+};
+
 const AmazonS3Uploads = new FileUploadClass({
 	name: 'AmazonS3:Uploads',
-	get
+	get,
+	copy
 	// store setted bellow
 });
 
 const AmazonS3Avatars = new FileUploadClass({
 	name: 'AmazonS3:Avatars',
-	get
+	get,
+	copy
+	// store setted bellow
+});
+
+const AmazonS3UserDataFiles = new FileUploadClass({
+	name: 'AmazonS3:UserDataFiles',
+	get,
+	copy
 	// store setted bellow
 });
 
@@ -74,6 +94,7 @@ const configure = _.debounce(function() {
 
 	AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config);
 	AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config);
+	AmazonS3UserDataFiles.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3UserDataFiles.name, config);
 }, 500);
 
 RocketChat.settings.get(/^FileUpload_S3_/, configure);

packages/rocketchat-file-upload/server/config/FileSystem.js

@@ -28,6 +28,22 @@ const FileSystemUploads = new FileUploadClass({
 			res.end();
 			return;
 		}
+	},
+
+	copy(file, out) {
+		const filePath = this.store.getFilePath(file._id, file);
+		try {
+			const stat = Meteor.wrapAsync(fs.stat)(filePath);
+
+			if (stat && stat.isFile()) {
+				file = FileUpload.addExtensionTo(file);
+
+				this.store.getReadStream(file._id, file).pipe(out);
+			}
+		} catch (e) {
+			out.end();
+			return;
+		}
 	}
 });
 
@@ -54,6 +70,31 @@ const FileSystemAvatars = new FileUploadClass({
 	}
 });
 
+const FileSystemUserDataFiles = new FileUploadClass({
+	name: 'FileSystem:UserDataFiles',
+
+	get(file, req, res) {
+		const filePath = this.store.getFilePath(file._id, file);
+
+		try {
+			const stat = Meteor.wrapAsync(fs.stat)(filePath);
+
+			if (stat && stat.isFile()) {
+				file = FileUpload.addExtensionTo(file);
+				res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
+				res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
+				res.setHeader('Content-Type', file.type);
+				res.setHeader('Content-Length', file.size);
+
+				this.store.getReadStream(file._id, file).pipe(res);
+			}
+		} catch (e) {
+			res.writeHead(404);
+			res.end();
+			return;
+		}
+	}
+});
 
 const createFileSystemStore = _.debounce(function() {
 	const options = {
@@ -62,6 +103,7 @@ const createFileSystemStore = _.debounce(function() {
 
 	FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options);
 	FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options);
+	FileSystemUserDataFiles.store = FileUpload.configureUploadsStore('Local', FileSystemUserDataFiles.name, options);
 
 	// DEPRECATED backwards compatibililty (remove)
 	UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name];

packages/rocketchat-file-upload/server/config/GoogleStorage.js

@@ -29,15 +29,39 @@ const get = function(file, req, res) {
 	});
 };
 
+const copy = function(file, out) {
+	this.store.getRedirectURL(file, (err, fileUrl) => {
+		if (err) {
+			console.error(err);
+		}
+
+		if (fileUrl) {
+			const request = /^https:/.test(fileUrl) ? https : http;
+			request.get(fileUrl, fileRes => fileRes.pipe(out));
+		} else {
+			out.end();
+		}
+	});
+};
+
 const GoogleCloudStorageUploads = new FileUploadClass({
 	name: 'GoogleCloudStorage:Uploads',
-	get
+	get,
+	copy
 	// store setted bellow
 });
 
 const GoogleCloudStorageAvatars = new FileUploadClass({
 	name: 'GoogleCloudStorage:Avatars',
-	get
+	get,
+	copy
+	// store setted bellow
+});
+
+const GoogleCloudStorageUserDataFiles = new FileUploadClass({
+	name: 'GoogleCloudStorage:UserDataFiles',
+	get,
+	copy
 	// store setted bellow
 });
 
@@ -64,6 +88,7 @@ const configure = _.debounce(function() {
 
 	GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config);
 	GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config);
+	GoogleCloudStorageUserDataFiles.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUserDataFiles.name, config);
 }, 500);
 
 RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure);

packages/rocketchat-file-upload/server/config/GridFS.js

@@ -62,7 +62,6 @@ const getByteRange = function(header) {
 	return null;
 };
 
-
 // code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310
 const readFromGridFS = function(storeName, fileId, file, req, res) {
 	const store = UploadFS.getStore(storeName);
@@ -123,10 +122,26 @@ const readFromGridFS = function(storeName, fileId, file, req, res) {
 	}
 };
 
+const copyFromGridFS = function(storeName, fileId, file, out) {
+	const store = UploadFS.getStore(storeName);
+	const rs = store.getReadStream(fileId, file);
+
+	[rs, out].forEach(stream => stream.on('error', function(err) {
+		store.onReadError.call(store, err, fileId, file);
+		out.end();
+	}));
+
+	rs.pipe(out);
+};
+
 FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', {
 	collectionName: 'rocketchat_uploads'
 });
 
+FileUpload.configureUploadsStore('GridFS', 'GridFS:UserDataFiles', {
+	collectionName: 'rocketchat_userDataFiles'
+});
+
 // DEPRECATED: backwards compatibility (remove)
 UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads'];
 
@@ -147,6 +162,29 @@ new FileUploadClass({
 		res.setHeader('Content-Length', file.size);
 
 		return readFromGridFS(file.store, file._id, file, req, res);
+	},
+
+	copy(file, out) {
+		copyFromGridFS(file.store, file._id, file, out);
+	}
+});
+
+new FileUploadClass({
+	name: 'GridFS:UserDataFiles',
+
+	get(file, req, res) {
+		file = FileUpload.addExtensionTo(file);
+
+		res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
+		res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
+		res.setHeader('Content-Type', file.type);
+		res.setHeader('Content-Length', file.size);
+
+		return readFromGridFS(file.store, file._id, file, req, res);
+	},
+
+	copy(file, out) {
+		copyFromGridFS(file.store, file._id, file, out);
 	}
 });
 

packages/rocketchat-file-upload/server/config/_configUploadStorage.js

@@ -14,6 +14,7 @@ const configStore = _.debounce(() => {
 		console.log('Setting default file store to', store);
 		UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`);
 		UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`);
+		UploadFS.getStores().UserDataFiles = UploadFS.getStore(`${ store }:UserDataFiles`);
 	}
 }, 1000);
 

packages/rocketchat-file-upload/server/lib/FileUpload.js

@@ -58,6 +58,25 @@ Object.assign(FileUpload, {
 		};
 	},
 
+	defaultUserDataFiles() {
+		return {
+			collection: RocketChat.models.UserDataFiles.model,
+			getPath(file) {
+				return `${ RocketChat.settings.get('uniqueID') }/uploads/userData/${ file.userId }`;
+			},
+			onValidate: FileUpload.uploadsOnValidate,
+			onRead(fileId, file, req, res) {
+				if (!FileUpload.requestCanAccessFiles(req)) {
+					res.writeHead(403);
+					return false;
+				}
+
+				res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`);
+				return true;
+			}
+		};
+	},
+
 	avatarsOnValidate(file) {
 		if (RocketChat.settings.get('Accounts_AvatarResize') !== true) {
 			return;
@@ -229,16 +248,30 @@ Object.assign(FileUpload, {
 		}
 		res.writeHead(404);
 		res.end();
+	},
+
+	copy(file, targetFile) {
+		const store = this.getStoreByName(file.store);
+		const out = fs.createWriteStream(targetFile);
+
+		file = FileUpload.addExtensionTo(file);
+
+		if (store.copy) {
+			store.copy(file, out);
+			return true;
+		}
+
+		return false;
 	}
 });
 
-
 export class FileUploadClass {
-	constructor({ name, model, store, get, insert, getStore }) {
+	constructor({ name, model, store, get, insert, getStore, copy }) {
 		this.name = name;
 		this.model = model || this.getModelFromName();
 		this._store = store || UploadFS.getStore(name);
 		this.get = get;
+		this.copy = copy;
 
 		if (insert) {
 			this.insert = insert;

packages/rocketchat-i18n/i18n/en.i18n.json

@@ -627,6 +627,7 @@
   "Domain_removed": "Domain Removed",
   "Domains": "Domains",
   "Domains_allowed_to_embed_the_livechat_widget": "Comma-separated list of domains allowed to embed the livechat widget. Leave blank to allow all domains.",
+  "Download_My_Data" : "Download My Data",
   "Download_Snippet": "Download",
   "Drop_to_upload_file": "Drop to upload file",
   "Dry_run": "Dry run",
@@ -784,6 +785,7 @@
   "Example_s": "Example: <code class=\"inline\">%s</code>",
   "Exclude_Botnames": "Exclude Bots",
   "Exclude_Botnames_Description": "Do not propagate messages from bots whose name matches the regular expression above. If left empty, all messages from bots will be propagated.",
+  "Export_My_Data" : "Export My Data",
   "External_Service": "External Service",
   "External_Queue_Service_URL": "External Queue Service URL",
   "Facebook_Page": "Facebook Page",
@@ -2178,6 +2180,18 @@
   "User_uploaded_file": "Uploaded a file",
   "User_uploaded_image": "Uploaded an image",
   "User_Presence": "User Presence",
+  "UserDataDownload" : "User Data Download",
+  "UserData_EnableDownload" : "Enable User Data Download",
+  "UserData_FileSystemPath" : "System Path (Exported Files)",
+  "UserData_FileSystemZipPath" : "System Path (Compressed File)",
+  "UserData_ProcessingFrequency" : "Processing Frequency (Minutes)",
+  "UserData_MessageLimitPerRequest" : "Message Limit per Request",
+  "UserDataDownload_EmailSubject" : "Your Data File is Ready to Download",
+  "UserDataDownload_EmailBody" : "Your data file is now ready to download. Click <a href=\"__download_link__\">here</a> to download it.",
+  "UserDataDownload_Requested" : "Download File Requested",
+  "UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.",
+  "UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.",
+  "UserDataDownload_CompletedRequestExisted_Text" : "Your data file was already generated. Check your email account for the download link.",
   "Username": "Username",
   "Username_and_message_must_not_be_empty": "Username and message must not be empty.",
   "Username_cant_be_empty": "The username cannot be empty",

packages/rocketchat-lib/client/models/UserDataFiles.js

@@ -0,0 +1,6 @@
+RocketChat.models.UserDataFiles = new class extends RocketChat.models._Base {
+	constructor() {
+		super();
+		this._initModel('userDataFiles');
+	}
+};

packages/rocketchat-lib/package.js

@@ -126,6 +126,8 @@ Package.onUse(function(api) {
 	api.addFiles('server/models/Subscriptions.js', 'server');
 	api.addFiles('server/models/Uploads.js', 'server');
 	api.addFiles('server/models/Users.js', 'server');
+	api.addFiles('server/models/ExportOperations.js', 'server');
+	api.addFiles('server/models/UserDataFiles.js', 'server');
 
 	api.addFiles('server/oauth/oauth.js', 'server');
 	api.addFiles('server/oauth/facebook.js', 'server');