diff --git a/README.rst b/README.rst index cb9baaec..6cad81bd 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Slack Dumper - Join the discussion in Telegram_ or Slack_. - `Read the overview on Medium.com`_ - |go ref| - +- `Mattermost migration`_ steps .. contents:: :depth: 2 @@ -30,7 +30,7 @@ There a three modes of operation (more on this in `User Guide`_) : #. List users/channels #. Dumping messages and threads -#. Creating a Slack Export. +#. Creating a Slack Export in Mattermost or Standard modes. Slackdump accepts two types of input (see `Dumping Conversations`_ section): @@ -188,6 +188,7 @@ Messages that were conveyed with the donations: .. _`Go templating`: https://pkg.go.dev/html/template .. _User Guide: doc/README.rst .. _Dumping Conversations: doc/usage-channels.rst +.. _Mattermost migration: doc/usage-export.rst .. _rusq/dlog: https://github.com/rusq/dlog .. _logrus: https://github.com/sirupsen/logrus .. _glog: https://github.com/golang/glog diff --git a/auth/auth_ui/cli.go b/auth/auth_ui/cli.go index 76b95e31..6a478669 100644 --- a/auth/auth_ui/cli.go +++ b/auth/auth_ui/cli.go @@ -32,9 +32,7 @@ func (cl *CLI) RequestWorkspace(w io.Writer) (string, error) { return workspace, nil } -func (*CLI) Stop() { - return -} +func (*CLI) Stop() {} func readln(r io.Reader) (string, error) { line, err := bufio.NewReader(r).ReadString('\n') diff --git a/cmd/slackdump/interactive.go b/cmd/slackdump/interactive.go index 6b6c5ab0..b4452a9a 100644 --- a/cmd/slackdump/interactive.go +++ b/cmd/slackdump/interactive.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" + "github.com/rusq/slackdump/v2/export" "github.com/rusq/slackdump/v2/internal/app" "github.com/rusq/slackdump/v2/internal/app/ui" "github.com/rusq/slackdump/v2/internal/structures" @@ -96,28 +97,59 @@ func surveyList(p *params) error { func surveyExport(p *params) error { var err error - p.appCfg.ExportName, err = ui.MustString( + + p.appCfg.ExportType, err = questExportType() + if err != nil { + return err + } + + p.appCfg.ExportName, err = ui.StringRequire( "Output directory or ZIP file: ", - "Enter the output directory or ZIP file name. Add \".zip\" to save to zip file", + "Enter the output directory or ZIP file name. Add \".zip\" extension to save to a zip file.\nFor Mattermost, zip file is recommended.", ) if err != nil { return err } - p.appCfg.Input.List, err = questConvoList() + p.appCfg.Input.List, err = questConversationList() + if err != nil { + return err + } + p.appCfg.Options.DumpFiles, err = ui.Confirm("Export files?", true) if err != nil { return err } return nil } +func questExportType() (export.ExportType, error) { + mode := &survey.Select{ + Message: "Export type: ", + Options: []string{export.TMattermost.String(), export.TStandard.String()}, + Description: func(value string, index int) string { + descr := []string{ + "Mattermost bulk upload compatible export (see doc)", + "Standard export format", + } + return descr[index] + }, + } + var resp string + if err := survey.AskOne(mode, &resp); err != nil { + return 0, err + } + var t export.ExportType + t.Set(resp) + return t, nil +} + func surveyDump(p *params) error { var err error - p.appCfg.Input.List, err = questConvoList() + p.appCfg.Input.List, err = questConversationList() return err } -// questConvoList enquires the channel list. -func questConvoList() (*structures.EntityList, error) { +// questConversationList enquires the channel list. +func questConversationList() (*structures.EntityList, error) { for { chanStr, err := ui.String( "List conversations: ", diff --git a/cmd/slackdump/main.go b/cmd/slackdump/main.go index 61b6d78e..fe177b65 100644 --- a/cmd/slackdump/main.go +++ b/cmd/slackdump/main.go @@ -19,6 +19,7 @@ import ( "github.com/slack-go/slack" "github.com/rusq/slackdump/v2" + "github.com/rusq/slackdump/v2/export" "github.com/rusq/slackdump/v2/internal/app" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/logger" @@ -232,7 +233,8 @@ func parseCmdLine(args []string) (params, error) { var p = params{ appCfg: app.Config{ - Options: slackdump.DefOptions, + Options: slackdump.DefOptions, + ExportType: export.TStandard, }, } @@ -248,6 +250,7 @@ func parseCmdLine(args []string) (params, error) { fs.BoolVar(&p.appCfg.ListFlags.Users, "list-users", false, "list users and their IDs. ") // - export fs.StringVar(&p.appCfg.ExportName, "export", "", "`name` of the directory or zip file to export the Slack workspace to."+zipHint) + fs.Var(&p.appCfg.ExportType, "export-type", "set the export type: 'standard' or 'mattermost' (default: standard)") // input-ouput options fs.StringVar(&p.appCfg.Output.Filename, "o", "-", "Output `filename` for users and channels.\nUse '-' for the Standard Output.") diff --git a/doc/usage-export.rst b/doc/usage-export.rst index f6118756..50bccc7a 100644 --- a/doc/usage-export.rst +++ b/doc/usage-export.rst @@ -4,21 +4,120 @@ Creating Slack Export .. contents:: -Exporting Slack Workspace -~~~~~~~~~~~~~~~~~~~~~~~~~ +This feature allows one to create a slack export of the Slack workspace in +standard or Mattermost compatible format. -This feature allows one to create a slack export of the slack workspace. To -run in Slack Export mode, one must start Slackdump specifying the -slack export directory, i.e.:: +By default, it generates the Standard type Export. - slackdump -export my-workspace +The export file or directory will include emails and attachments (if +``-download`` flag is specified). -Or, if you want to save export as a ZIP file:: +Mattermost export +~~~~~~~~~~~~~~~~~ +Mattermost mode is currently in alpha-stage. Export is generated in the +format that can be imported using Mattermost "bulk" import mode format using +``mmetl/mmctl`` tools (see quick guide below). - slackdump -export my-workspace.zip +The ``mattermost import slack`` command is not yet supported. -Slackdump will export the whole workspace. If ' ``-f``' flag is specified, -all files will be saved under the channel's '``attachments``' directory. +Mattermost export quick guide ++++++++++++++++++++++++++++++ + +To export to Mattermost, Slackdump should be started with ``-export-type +mattermost`` flag. Mattermost tools would require a ZIP file. + +Steps to export from Slack and import to Mattermost: + +#. Run Slackdump in mattermost mode to export the workspace:: + + slackdump -export my-workspace.zip -export-type mattermost -download + + optionally, you can specify list of conversation to export:: + + slackdump -export my-workspace.zip -export-type mattermost -download C12301120 D4012041 + +#. Download the ``mmetl`` tool for your architecture from `mmetl + github page`_. In the example we'll be using the Linux version:: + + curl -LO https://github.com/mattermost/mmetl/releases/download/0.0.1/mmetl.linux-amd64.tar.gz + + Unpack:: + + tar zxf mmetl.linux-amd64.tar.gz + +#. Run the ``mmetl`` tool to generate the mattermost bulk import + JSONL file:: + + ./mmetl transform slack -t Your_Team_Name -d bulk-export-attachments -f test.zip -o mattermost_import.jsonl + + For example, if your Mattermost team is "slackdump":: + + ./mmetl transform slack -t slackdump -d bulk-export-attachments -f test.zip -o mattermost_import.jsonl + + This will generate a directory ``bulk-export-attachments`` and + ``mattermost_import.jsonl`` file in the current directory. + +#. Create a zip archive in bulk format. Please ensure that the + ``bulk-export-attachments`` directory is placed inside ``data`` + directory by following the steps below:: + + mkdir data + mv bulk-export-attachments data + zip -r bulk_import.zip data mattermost_import.jsonl + +#. Copy the resulting file to the mattermost server, and upload it using ``mmctl`` tool:: + + mmctl import upload ./bulk_import.zip + + This will upload the zip file into the Mattermost. + + List all import files to find out the filename that will be used to + start the import process:: + + mmctl import list available + + The output will print the file with an ID prefix:: + + 9zgyay5wupdyzc1kqdin5re77e_bulk_import.zip + +#. Start the import process:: + + mmctl import process + + For example:: + + mmctl import process 9zgyay5wupdyzc1kqdin5re77e_bulk_import.zip + +#. To monitor the status of the job or to see if there are any + errors:: + + mmctl import job list + + and:: + + mmctl import job show --json + +After following all these steps, you should see the data in your +Mattermost team. + +More detailed instructions can be found in the `Mattermost +documentation`_ + +Standard export +~~~~~~~~~~~~~~~ + +To run in Slack Export standard mode, one must start Slackdump +specifying the slack export directory or zip file, i.e.:: + + slackdump -export my-workspace -export-type standard + + < OR, for a ZIP file > + + slackdump -export my-workspace.zip -export-type standard + +Slackdump will export the whole workspace. If ' ``-download``' flag is +specified, all files will be saved under the channel's '``attachments``' +directory. Inclusive and Exclusive export ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -26,7 +125,7 @@ Inclusive and Exclusive export It is possible to **include** or **exclude** channels in/from the Export. Exporting only the channels you need -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++++++++++++++ To **include** only those channels you're interested in, use the following syntax:: @@ -36,7 +135,7 @@ syntax:: The command above will export ONLY channels ``C12401724`` and ``C4812934``. Exporting everything except some unwanted channels -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++++++++++++++++++++++++++++ To **exclude** one or more channels from the export, prefix the channel with "^" character. For example, you want to export everything except channel C123456:: @@ -44,7 +143,7 @@ character. For example, you want to export everything except channel C123456:: slackdump -export my-workspace.zip ^C123456 Providing the list in a file -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++++++ You can specify the filename instead of listing all the channels on the command line. To include the channels from the file, use the "@" character prefix. The @@ -111,3 +210,5 @@ slack-like GUI. .. _`Scumbag Steve`: https://www.google.com/search?q=Scumbag+Steve .. _Index: README.rst +.. _mmetl github page: https://github.com/mattermost/mmetl +.. _Mattermost documentation: https://docs.mattermost.com/onboard/migrating-to-mattermost.html#migrating-from-slack-using-the-mattermost-mmetl-tool-and-bulk-import diff --git a/downloader/downloader.go b/downloader/downloader.go index 8dc7f851..80fe0864 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -22,11 +22,10 @@ import ( ) const ( - defDownloadDir = "." // default download directory is current. - defRetries = 3 // default number of retries if download fails - defNumWorkers = 4 // number of download processes - defLimit = 5000 // default API limit, in events per second. - defFileBufSz = 100 // default download channel buffer. + defRetries = 3 // default number of retries if download fails + defNumWorkers = 4 // number of download processes + defLimit = 5000 // default API limit, in events per second. + defFileBufSz = 100 // default download channel buffer. ) // Client is the instance of the downloader. @@ -43,8 +42,17 @@ type Client struct { fileRequests chan FileRequest wg *sync.WaitGroup started bool + + nameFn FilenameFunc } +// FilenameFunc is the file naming function that should return the output +// filename for slack.File. +type FilenameFunc func(*slack.File) string + +// Filename returns name of the file generated from the slack.File. +var Filename FilenameFunc = stdFilenameFn + // Downloader is the file downloader interface. It exists primarily for mocking // in tests. type Downloader interface { @@ -95,6 +103,16 @@ func Logger(l logger.Interface) Option { } } +func WithNameFunc(fn FilenameFunc) Option { + return func(c *Client) { + if fn != nil { + c.nameFn = fn + } else { + c.nameFn = Filename + } + } +} + // New initialises new file downloader. func New(client Downloader, fs fsadapter.FS, opts ...Option) *Client { if client == nil { @@ -107,6 +125,7 @@ func New(client Downloader, fs fsadapter.FS, opts ...Option) *Client { limiter: rate.NewLimiter(defLimit, 1), retries: defRetries, workers: defNumWorkers, + nameFn: Filename, } for _, opt := range opts { opt(c) @@ -172,18 +191,18 @@ func (c *Client) worker(ctx context.Context, reqC <-chan FileRequest) { if !moar { return } - c.l().Debugf("saving %q to %s, size: %d", Filename(req.File), req.Directory, req.File.Size) + c.l().Debugf("saving %q to %s, size: %d", c.nameFn(req.File), req.Directory, req.File.Size) n, err := c.saveFile(ctx, req.Directory, req.File) if err != nil { - c.l().Printf("error saving %q to %q: %s", Filename(req.File), req.Directory, err) + c.l().Printf("error saving %q to %q: %s", c.nameFn(req.File), req.Directory, err) break } - c.l().Printf("file %q saved to %s: %d bytes written", Filename(req.File), req.Directory, n) + c.l().Printf("file %q saved to %s: %d bytes written", c.nameFn(req.File), req.Directory, n) } } } -var errNoFS = errors.New("fs adapter not initialised") +var ErrNoFS = errors.New("fs adapter not initialised") // AsyncDownloader starts Client.worker goroutines to download files // concurrently. It will download any file that is received on fileDlQueue @@ -191,7 +210,7 @@ var errNoFS = errors.New("fs adapter not initialised") // closed once all downloads are complete. func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <-chan *slack.File) (chan struct{}, error) { if c.fs == nil { - return nil, errNoFS + return nil, ErrNoFS } done := make(chan struct{}) @@ -217,9 +236,9 @@ func (c *Client) AsyncDownloader(ctx context.Context, dir string, fileDlQueue <- // saveFileWithLimiter saves the file to specified directory, it will use the provided limiter l for throttling. func (c *Client) saveFile(ctx context.Context, dir string, sf *slack.File) (int64, error) { if c.fs == nil { - return 0, errNoFS + return 0, ErrNoFS } - filePath := filepath.Join(dir, Filename(sf)) + filePath := filepath.Join(dir, c.nameFn(sf)) tf, err := os.CreateTemp("", "") if err != nil { @@ -265,8 +284,7 @@ func (c *Client) saveFile(ctx context.Context, dir string, sf *slack.File) (int6 return int64(n), nil } -// Filename returns name of the file -func Filename(f *slack.File) string { +func stdFilenameFn(f *slack.File) string { return fmt.Sprintf("%s-%s", f.ID, f.Name) } diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go index 5b7aa82f..7890c54d 100644 --- a/downloader/downloader_test.go +++ b/downloader/downloader_test.go @@ -26,9 +26,6 @@ var ( file3 = slack.File{ID: "f3", Name: "filename3.ext", URLPrivateDownload: "file3_url", Size: 300} file4 = slack.File{ID: "f4", Name: "filename4.ext", URLPrivateDownload: "file4_url", Size: 400} file5 = slack.File{ID: "f5", Name: "filename5.ext", URLPrivateDownload: "file5_url", Size: 500} - file6 = slack.File{ID: "f6", Name: "filename6.ext", URLPrivateDownload: "file6_url", Size: 600} - file7 = slack.File{ID: "f7", Name: "filename7.ext", URLPrivateDownload: "file7_url", Size: 700} - file8 = slack.File{ID: "f8", Name: "filename8.ext", URLPrivateDownload: "file8_url", Size: 800} file9 = slack.File{ID: "f9", Name: "filename9.ext", URLPrivateDownload: "file9_url", Size: 900} ) @@ -40,6 +37,7 @@ func TestSession_SaveFileTo(t *testing.T) { fs fsadapter.FS retries int workers int + nameFn FilenameFunc } type args struct { ctx context.Context @@ -60,7 +58,9 @@ func TestSession_SaveFileTo(t *testing.T) { l: rate.NewLimiter(defLimit, 1), fs: fsadapter.NewDirectory(tmpdir), retries: defRetries, - workers: defNumWorkers}, + workers: defNumWorkers, + nameFn: Filename, + }, args{ context.Background(), "01", @@ -81,7 +81,9 @@ func TestSession_SaveFileTo(t *testing.T) { l: rate.NewLimiter(defLimit, 1), fs: fsadapter.NewDirectory(tmpdir), retries: defRetries, - workers: defNumWorkers}, + workers: defNumWorkers, + nameFn: Filename, + }, args{ context.Background(), "02", @@ -109,6 +111,7 @@ func TestSession_SaveFileTo(t *testing.T) { limiter: tt.fields.l, retries: tt.fields.retries, workers: tt.fields.workers, + nameFn: tt.fields.nameFn, } got, err := sd.SaveFile(tt.args.ctx, tt.args.dir, tt.args.f) if (err != nil) != tt.wantErr { @@ -130,6 +133,7 @@ func TestSession_saveFile(t *testing.T) { fs fsadapter.FS retries int workers int + nameFn FilenameFunc } type args struct { ctx context.Context @@ -151,6 +155,7 @@ func TestSession_saveFile(t *testing.T) { fs: fsadapter.NewDirectory(tmpdir), retries: defRetries, workers: defNumWorkers, + nameFn: Filename, }, args{ context.Background(), @@ -173,6 +178,7 @@ func TestSession_saveFile(t *testing.T) { fs: fsadapter.NewDirectory(tmpdir), retries: defRetries, workers: defNumWorkers, + nameFn: Filename, }, args{ context.Background(), @@ -201,6 +207,7 @@ func TestSession_saveFile(t *testing.T) { limiter: tt.fields.l, retries: tt.fields.retries, workers: tt.fields.workers, + nameFn: tt.fields.nameFn, } got, err := sd.saveFile(tt.args.ctx, tt.args.dir, tt.args.f) if (err != nil) != tt.wantErr { @@ -250,6 +257,7 @@ func TestSession_newFileDownloader(t *testing.T) { limiter: tl, retries: 3, workers: 4, + nameFn: Filename, } mc.EXPECT(). @@ -284,6 +292,7 @@ func TestSession_worker(t *testing.T) { limiter: tl, retries: defRetries, workers: defNumWorkers, + nameFn: Filename, } } @@ -350,6 +359,7 @@ func TestClient_startWorkers(t *testing.T) { fs: fsadapter.NewDirectory(t.TempDir()), limiter: rate.NewLimiter(5000, 1), workers: defNumWorkers, + nameFn: Filename, } dc.EXPECT().GetFile(gomock.Any(), gomock.Any()).Times(qSz).Return(nil) @@ -417,6 +427,7 @@ func clientWithMock(t *testing.T, dir string) *Client { fs: fsadapter.NewDirectory(dir), limiter: rate.NewLimiter(5000, 1), workers: defNumWorkers, + nameFn: Filename, } return c } diff --git a/export/export.go b/export/export.go index ebd472dc..719c93ca 100644 --- a/export/export.go +++ b/export/export.go @@ -6,18 +6,15 @@ import ( "errors" "fmt" "io" - "path" "path/filepath" "runtime/trace" "github.com/slack-go/slack" "github.com/rusq/slackdump/v2" - "github.com/rusq/slackdump/v2/downloader" "github.com/rusq/slackdump/v2/fsadapter" "github.com/rusq/slackdump/v2/internal/network" "github.com/rusq/slackdump/v2/internal/structures" - "github.com/rusq/slackdump/v2/internal/structures/files" "github.com/rusq/slackdump/v2/logger" "github.com/rusq/slackdump/v2/types" ) @@ -28,7 +25,7 @@ type Export struct { sd *slackdump.Session // Session instance lg logger.Interface - // time window + // options opts Options } @@ -67,7 +64,7 @@ func (se *Export) messages(ctx context.Context, users types.Users) error { ctx, task := trace.NewTask(ctx, "export.messages") defer task.End() - dl := downloader.New(se.sd.Client(), se.fs, downloader.Logger(se.l())) + dl := newFileExporter(se.opts.Type, se.fs, se.sd.Client(), se.l()) if se.opts.IncludeFiles { // start the downloader dl.Start(ctx) @@ -97,19 +94,19 @@ func (se *Export) messages(ctx context.Context, users types.Users) error { return nil } -func (se *Export) exportChannels(ctx context.Context, dl *downloader.Client, uidx structures.UserIndex, el *structures.EntityList) ([]slack.Channel, error) { +func (se *Export) exportChannels(ctx context.Context, proc fileProcessor, uidx structures.UserIndex, el *structures.EntityList) ([]slack.Channel, error) { if se.opts.List.HasIncludes() { // if there an Include list, we don't need to retrieve all channels, // only the ones that are specified. - return se.inclusiveExport(ctx, dl, uidx, se.opts.List) + return se.inclusiveExport(ctx, proc, uidx, se.opts.List) } else { - return se.exclusiveExport(ctx, dl, uidx, se.opts.List) + return se.exclusiveExport(ctx, proc, uidx, se.opts.List) } } // exclusiveExport exports all channels, excluding ones that are defined in // EntityList. If EntityList has Include channels, they are ignored. -func (se *Export) exclusiveExport(ctx context.Context, dl *downloader.Client, uidx structures.UserIndex, el *structures.EntityList) ([]slack.Channel, error) { +func (se *Export) exclusiveExport(ctx context.Context, proc fileProcessor, uidx structures.UserIndex, el *structures.EntityList) ([]slack.Channel, error) { ctx, task := trace.NewTask(ctx, "export.exclusive") defer task.End() @@ -123,7 +120,7 @@ func (se *Export) exclusiveExport(ctx context.Context, dl *downloader.Client, ui se.lg.Printf("skipping: %s", ch.ID) return nil } - if err := se.exportConversation(ctx, dl, uidx, ch); err != nil { + if err := se.exportConversation(ctx, proc, uidx, ch); err != nil { return err } @@ -139,7 +136,7 @@ func (se *Export) exclusiveExport(ctx context.Context, dl *downloader.Client, ui // inclusiveExport exports only channels that are defined in the // EntryList.Include. -func (se *Export) inclusiveExport(ctx context.Context, dl *downloader.Client, uidx structures.UserIndex, list *structures.EntityList) ([]slack.Channel, error) { +func (se *Export) inclusiveExport(ctx context.Context, proc fileProcessor, uidx structures.UserIndex, list *structures.EntityList) ([]slack.Channel, error) { ctx, task := trace.NewTask(ctx, "export.inclusive") defer task.End() @@ -169,7 +166,7 @@ func (se *Export) inclusiveExport(ctx context.Context, dl *downloader.Client, ui return nil, fmt.Errorf("error getting info for %s: %w", sl, err) } - if err := se.exportConversation(ctx, dl, uidx, *ch); err != nil { + if err := se.exportConversation(ctx, proc, uidx, *ch); err != nil { return nil, err } @@ -180,12 +177,11 @@ func (se *Export) inclusiveExport(ctx context.Context, dl *downloader.Client, ui } // exportConversation exports one conversation. -func (se *Export) exportConversation(ctx context.Context, dl *downloader.Client, userIdx structures.UserIndex, ch slack.Channel) error { +func (se *Export) exportConversation(ctx context.Context, proc fileProcessor, userIdx structures.UserIndex, ch slack.Channel) error { ctx, task := trace.NewTask(ctx, "export.conversation") defer task.End() - dlFn := se.downloadFn(dl, ch.Name) - messages, err := se.sd.DumpRaw(ctx, ch.ID, se.opts.Oldest, se.opts.Latest, dlFn) + messages, err := se.sd.DumpRaw(ctx, ch.ID, se.opts.Oldest, se.opts.Latest, proc.ProcessFunc(ch.Name)) if err != nil { return fmt.Errorf("failed to dump %q (%s): %w", ch.Name, ch.ID, err) } @@ -211,38 +207,6 @@ func (se *Export) exportConversation(ctx context.Context, dl *downloader.Client, return nil } -// downloadFn returns the process function that should be passed to -// DumpMessagesRaw that will handle the download of the files. If the -// downloader is not started, i.e. if file download is disabled, it will -// silently ignore the error and return nil. -func (se *Export) downloadFn(dl *downloader.Client, channelName string) func(msg []types.Message, channelID string) (slackdump.ProcessResult, error) { - const ( - entFiles = "files" - dirAttach = "attachments" - ) - - dir := filepath.Join(channelName, dirAttach) - return func(msg []types.Message, channelID string) (slackdump.ProcessResult, error) { - total := 0 - if err := files.Extract(msg, files.Root, func(file slack.File, addr files.Addr) error { - filename, err := dl.DownloadFile(dir, file) - if err != nil { - return err - } - se.l().Debugf("submitted for download: %s", file.Name) - total++ - return files.UpdateURLs(msg, addr, path.Join(dirAttach, path.Base(filename))) - }); err != nil { - if errors.Is(err, downloader.ErrNotStarted) { - return slackdump.ProcessResult{Entity: entFiles, Count: 0}, nil - } - return slackdump.ProcessResult{}, err - } - - return slackdump.ProcessResult{Entity: entFiles, Count: total}, nil - } -} - // validName returns the channel or user name. Following the naming convention // described by @niklasdahlheimer in this post (thanks to @Neznakomec for // discovering it): @@ -289,6 +253,7 @@ func serialize(w io.Writer, data any) error { return nil } +// l returns the current logger or the default one if no logger is set. func (se *Export) l() logger.Interface { if se.lg == nil { se.lg = logger.Default diff --git a/export/exporttype.go b/export/exporttype.go new file mode 100644 index 00000000..40473439 --- /dev/null +++ b/export/exporttype.go @@ -0,0 +1,29 @@ +package export + +import ( + "fmt" + "strings" +) + +//go:generate go install golang.org/x/tools/cmd/stringer@latest + +//go:generate stringer -type=ExportType -linecomment +type ExportType uint8 + +const ( + TStandard ExportType = iota // Standard + TMattermost // Mattermost +) + +func (e *ExportType) Set(v string) error { + v = strings.ToLower(v) + switch v { + default: + return fmt.Errorf("unknown format: %s", v) + case strings.ToLower(TStandard.String()): + *e = TStandard + case strings.ToLower(TMattermost.String()): + *e = TMattermost + } + return nil +} diff --git a/export/exporttype_string.go b/export/exporttype_string.go new file mode 100644 index 00000000..e719b42b --- /dev/null +++ b/export/exporttype_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type=ExportType -linecomment"; DO NOT EDIT. + +package export + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[TStandard-0] + _ = x[TMattermost-1] +} + +const _ExportType_name = "StandardMattermost" + +var _ExportType_index = [...]uint8{0, 8, 18} + +func (i ExportType) String() string { + if i >= ExportType(len(_ExportType_index)-1) { + return "ExportType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ExportType_name[_ExportType_index[i]:_ExportType_index[i+1]] +} diff --git a/export/files.go b/export/files.go new file mode 100644 index 00000000..a76d41c7 --- /dev/null +++ b/export/files.go @@ -0,0 +1,135 @@ +package export + +import ( + "context" + "errors" + "path" + "path/filepath" + + "github.com/rusq/slackdump/v2" + "github.com/rusq/slackdump/v2/downloader" + "github.com/rusq/slackdump/v2/fsadapter" + "github.com/rusq/slackdump/v2/internal/structures/files" + "github.com/rusq/slackdump/v2/logger" + "github.com/rusq/slackdump/v2/types" + "github.com/slack-go/slack" +) + +const entFiles = "files" + +// fileProcessor is the file exporter interface. +type fileProcessor interface { + // ProcessFunc returns the process function that should be passed to + // DumpMessagesRaw. It should be able to extract files from the messages + // and download them. If the downloader is not started, i.e. if file + // download is disabled, it should silently ignore the error and return + // nil. + ProcessFunc(channelName string) slackdump.ProcessFunc +} + +type fileExporter interface { + fileProcessor + Start(ctx context.Context) + Stop() +} + +type baseDownloader struct { + dl *downloader.Client + l logger.Interface +} + +func (bd *baseDownloader) Start(ctx context.Context) { + bd.dl.Start(ctx) +} + +func (bd *baseDownloader) Stop() { + bd.dl.Stop() +} + +type stdDownload struct { + baseDownloader +} + +type mattermostDownload struct { + baseDownloader +} + +func newFileExporter(t ExportType, fs fsadapter.FS, cl *slack.Client, l logger.Interface) fileExporter { + switch t { + default: + l.Printf("unknown export type %s, using standard format", t) + fallthrough + case TStandard: + return newStdDl(fs, cl, l) + case TMattermost: + return newMattermostDl(fs, cl, l) + } +} + +func newStdDl(fs fsadapter.FS, cl *slack.Client, l logger.Interface) *stdDownload { + return &stdDownload{baseDownloader: baseDownloader{ + dl: downloader.New(cl, fs, downloader.Logger(l)), + l: l, + }} +} + +func (d *stdDownload) ProcessFunc(channelName string) slackdump.ProcessFunc { + const ( + dirAttach = "attachments" + ) + + dir := filepath.Join(channelName, dirAttach) + return func(msg []types.Message, channelID string) (slackdump.ProcessResult, error) { + total := 0 + if err := files.Extract(msg, files.Root, func(file slack.File, addr files.Addr) error { + filename, err := d.dl.DownloadFile(dir, file) + if err != nil { + return err + } + d.l.Debugf("submitted for download: %s", file.Name) + total++ + return files.UpdateURLs(msg, addr, path.Join(dirAttach, path.Base(filename))) + }); err != nil { + if errors.Is(err, downloader.ErrNotStarted) { + return slackdump.ProcessResult{Entity: entFiles, Count: 0}, nil + } + return slackdump.ProcessResult{}, err + } + + return slackdump.ProcessResult{Entity: entFiles, Count: total}, nil + } +} + +func newMattermostDl(fs fsadapter.FS, cl *slack.Client, l logger.Interface) *mattermostDownload { + return &mattermostDownload{baseDownloader: baseDownloader{ + dl: downloader.New(cl, fs, downloader.Logger(l), downloader.WithNameFunc( + func(f *slack.File) string { + return f.Name + }, + )), l: l, + }} +} + +func (md *mattermostDownload) ProcessFunc(_ string) slackdump.ProcessFunc { + const ( + baseDir = "__uploads" + ) + return func(msg []types.Message, channelID string) (slackdump.ProcessResult, error) { + total := 0 + if err := files.Extract(msg, files.Root, func(file slack.File, addr files.Addr) error { + filedir := filepath.Join(baseDir, file.ID) + _, err := md.dl.DownloadFile(filedir, file) + if err != nil { + return err + } + total++ + return nil + }); err != nil { + if errors.Is(err, downloader.ErrNotStarted) { + return slackdump.ProcessResult{Entity: entFiles, Count: 0}, nil + } + return slackdump.ProcessResult{}, err + } + return slackdump.ProcessResult{Entity: entFiles, Count: total}, nil + } +} diff --git a/export/options.go b/export/options.go index b23bc1f9..20a59f56 100644 --- a/export/options.go +++ b/export/options.go @@ -14,4 +14,5 @@ type Options struct { IncludeFiles bool Logger logger.Interface List *structures.EntityList + Type ExportType } diff --git a/fsadapter/zipfs_test.go b/fsadapter/zipfs_test.go index 15cd9060..5d3fd094 100644 --- a/fsadapter/zipfs_test.go +++ b/fsadapter/zipfs_test.go @@ -179,7 +179,7 @@ func TestCreateConcurrency(t *testing.T) { } <-readySteadyGo - fw, err := fsa.Create(fmt.Sprintf("file%d", i)) + fw, err := fsa.Create(fmt.Sprintf("file%d", n)) if err != nil { panic(err) } diff --git a/internal/app/config.go b/internal/app/config.go index f908eb64..b2643e7c 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -9,6 +9,7 @@ import ( "github.com/slack-go/slack" "github.com/rusq/slackdump/v2" + "github.com/rusq/slackdump/v2/export" "github.com/rusq/slackdump/v2/internal/structures" "github.com/rusq/slackdump/v2/logger" "github.com/rusq/slackdump/v2/types" @@ -37,6 +38,7 @@ type Config struct { FilenameTemplate string ExportName string + ExportType export.ExportType Options slackdump.Options } diff --git a/internal/app/export.go b/internal/app/export.go index 31ccd561..3275c524 100644 --- a/internal/app/export.go +++ b/internal/app/export.go @@ -34,6 +34,7 @@ func Export(ctx context.Context, cfg Config, prov auth.Provider) error { IncludeFiles: cfg.Options.DumpFiles, Logger: cfg.Logger(), List: cfg.Input.List, + Type: cfg.ExportType, } fs, err := fsadapter.ForFilename(cfg.ExportName) if err != nil { diff --git a/internal/app/ui/input.go b/internal/app/ui/input.go index e13840d0..d906c95b 100644 --- a/internal/app/ui/input.go +++ b/internal/app/ui/input.go @@ -23,10 +23,12 @@ func Input(msg, help string, validator survey.Validator) (string, error) { return m.Value, nil } -func MustString(msg, help string) (string, error) { +// StringRequire requires user to input string. +func StringRequire(msg, help string) (string, error) { return Input(msg, help, survey.Required) } +// String asks user to input string, accepts an empty input. func String(msg, help string) (string, error) { return Input(msg, help, nil) } diff --git a/internal/structures/url_parse_test.go b/internal/structures/url_parse_test.go index cb0a2b94..66bf86f4 100644 --- a/internal/structures/url_parse_test.go +++ b/internal/structures/url_parse_test.go @@ -12,7 +12,6 @@ const ( sampleDMURL = "https://ora600.slack.com/archives/DL98HT3QA" sampleChannelID = "CHM82GF99" - sampleThreadTS = "p1577694990000400" ) func TestParseURL(t *testing.T) { diff --git a/processors.go b/processors.go index 79953f42..439a7e22 100644 --- a/processors.go +++ b/processors.go @@ -108,8 +108,7 @@ func pipeAndUpdateFiles(filesC chan<- *slack.File, msgs []types.Message, dir str _ = files.Extract(msgs, files.Root, func(file slack.File, addr files.Addr) error { filesC <- &file total++ - files.UpdateURLs(msgs, addr, path.Join(dir, downloader.Filename(&file))) - return nil + return files.UpdateURLs(msgs, addr, path.Join(dir, downloader.Filename(&file))) }) return total } diff --git a/tools/slackutil/main.go b/tools/slackutil/main.go index a94f0f99..2c42dce5 100644 --- a/tools/slackutil/main.go +++ b/tools/slackutil/main.go @@ -129,7 +129,7 @@ func delMessages(ctx context.Context, client *slack.Client, channelID string, ms if err != nil { return err } - pb.Add(1) + _ = pb.Add(1) } return nil } diff --git a/tools/testcookies/main.go b/tools/testcookies/main.go index 81c6312c..7cddcd46 100644 --- a/tools/testcookies/main.go +++ b/tools/testcookies/main.go @@ -11,7 +11,7 @@ import ( ) func init() { - playwright.Install(&playwright.RunOptions{Browsers: []string{"chromium"}}) + _ = playwright.Install(&playwright.RunOptions{Browsers: []string{"chromium"}}) } func main() {