diff --git a/server/cmd/mmctl/commands/mmctl_test.go b/server/cmd/mmctl/commands/mmctl_test.go
index 24fa2b6d374..c16e7cfdb41 100644
--- a/server/cmd/mmctl/commands/mmctl_test.go
+++ b/server/cmd/mmctl/commands/mmctl_test.go
@@ -15,7 +15,6 @@ import (
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/client"
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/mocks"
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@@ -81,8 +80,6 @@ func (s *MmctlE2ETestSuite) SetupMessageExportTestHelper() *api4.TestHelper {
jobs.DefaultWatcherPollingInterval = 100
s.th = api4.SetupEnterprise(s.T()).InitBasic()
s.th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
- messageExportImpl := message_export.MessageExportJobInterfaceImpl{Server: s.th.App.Srv()}
- s.th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
s.th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.MessageExportSettings.DownloadExportResults = true
*cfg.MessageExportSettings.EnableExport = true
diff --git a/server/enterprise/external_imports.go b/server/enterprise/external_imports.go
index a3127fb0587..09223ecef19 100644
--- a/server/enterprise/external_imports.go
+++ b/server/enterprise/external_imports.go
@@ -36,4 +36,12 @@ import (
_ "github.com/mattermost/enterprise/outgoing_oauth_connections"
// Needed to ensure the init() method in the EE gets run
_ "github.com/mattermost/enterprise/access_control"
+ // Needed to ensure the init() method in the EE gets run
+ _ "github.com/mattermost/enterprise/message_export"
+ // Needed to ensure the init() method in the EE gets run
+ _ "github.com/mattermost/enterprise/message_export/actiance_export"
+ // Needed to ensure the init() method in the EE gets run
+ _ "github.com/mattermost/enterprise/message_export/csv_export"
+ // Needed to ensure the init() method in the EE gets run
+ _ "github.com/mattermost/enterprise/message_export/global_relay_export"
)
diff --git a/server/enterprise/internal/file/utils.go b/server/enterprise/internal/file/utils.go
deleted file mode 100644
index 48ed3770db7..00000000000
--- a/server/enterprise/internal/file/utils.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package file
-
-import (
- "os"
-
- "github.com/mattermost/mattermost/server/public/shared/mlog"
-)
-
-// DeleteTemp removes a file and logs the error
-// Intended to be called in a defer after the creation of a temp file to ensure cleanup
-func DeleteTemp(logger mlog.LoggerIFace, file *os.File) {
- err := file.Close()
- if err != nil {
- logger.Warn("Failed to close temporary file", mlog.String("filename", file.Name()), mlog.Err(err))
- }
- err = os.Remove(file.Name())
- if err != nil {
- logger.Warn("Failed to delete temporary file", mlog.String("filename", file.Name()), mlog.Err(err))
- }
-}
diff --git a/server/enterprise/local_imports.go b/server/enterprise/local_imports.go
index 449f0a03a5d..7f0ddac8443 100644
--- a/server/enterprise/local_imports.go
+++ b/server/enterprise/local_imports.go
@@ -9,13 +9,5 @@ import (
// Needed to ensure the init() method in the EE gets run
_ "github.com/mattermost/mattermost/server/v8/enterprise/metrics"
// Needed to ensure the init() method in the EE gets run
- _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export"
- // Needed to ensure the init() method in the EE gets run
- _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
- // Needed to ensure the init() method in the EE gets run
- _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
- // Needed to ensure the init() method in the EE gets run
- _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
- // Needed to ensure the init() method in the EE gets run
_ "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch"
)
diff --git a/server/enterprise/message_export/actiance_export/actiance_export.go b/server/enterprise/message_export/actiance_export/actiance_export.go
deleted file mode 100644
index e29e0b6a273..00000000000
--- a/server/enterprise/message_export/actiance_export/actiance_export.go
+++ /dev/null
@@ -1,374 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package actiance_export
-
-import (
- "archive/zip"
- "bytes"
- "encoding/xml"
- "fmt"
- "io"
- "os"
- "slices"
- "strings"
- "time"
-
- "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-const (
- XMLNS = "http://www.w3.org/2001/XMLSchema-instance"
- ActianceExportFilename = "actiance_export.xml"
- ActianceWarningFilename = "warning.txt"
- EnsureLastMessageInASort = "zzzzzzzzzzzzzzzzzzzzzzzzzz"
-)
-
-// The root-level element of an actiance export
-type RootNode struct {
- XMLName xml.Name `xml:"FileDump"`
- XMLNS string `xml:"xmlns:xsi,attr"` // this should default to "http://www.w3.org/2001/XMLSchema-instance"
- Channels []ChannelExport // one element per channel (open or invite-only), group message, or direct message
-}
-
-// The Conversation element indicates an ad hoc IM conversation or a group chat room.
-// The messages from a persistent chat room are exported once a day so that a Conversation entry contains the messages posted to a chat room from 12:00:00 AM to 11:59:59 PM
-type ChannelExport struct {
- XMLName xml.Name `xml:"Conversation"`
- Perspective string `xml:"Perspective,attr"` // the value of this attribute doesn't seem to matter. Using the channel name makes the export more human readable
- ChannelId string `xml:"-"` // the unique id of the channel
- RoomId string `xml:"RoomID"`
- StartTime int64 `xml:"StartTimeUTC"` // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
- JoinEvents []JoinExport // start with a list of all users who were present in the channel during the export period
- Elements []Sortable
- LeaveEvents []LeaveExport // finish with a list of all users who were present in the channel during the export period
- EndTime int64 `xml:"EndTimeUTC"` // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
-}
-
-// The ParticipantEntered element indicates each user who participates in a conversation.
-// For chat rooms, there must be one ParticipantEntered element for each user present in the chat room at the beginning of the reporting period
-type JoinExport struct {
- XMLName xml.Name `xml:"ParticipantEntered"`
- UserEmail string `xml:"LoginName"` // the email of the person that joined the channel
- UserType string `xml:"UserType"` // the type of the user that joined the channel
- JoinTime int64 `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user joined. Example: 1366611728
- CorporateEmailID string `xml:"CorporateEmailID"`
-}
-
-// The ParticipantLeft element indicates the user who leaves an active IM or chat room conversation.
-// For chat rooms, there must be one ParticipantLeft element for each user present in the chat room at the end of the reporting period.
-type LeaveExport struct {
- XMLName xml.Name `xml:"ParticipantLeft"`
- UserEmail string `xml:"LoginName"` // the email of the person that left the channel
- UserType string `xml:"UserType"` // the type of the user that left the channel
- LeaveTime int64 `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user left. Example: 1366611728
- CorporateEmailID string `xml:"CorporateEmailID"`
-}
-
-type Sortable interface {
- SortVal() (int64, string)
-}
-
-func SortableSort(a, b Sortable) int {
- aTimestamp, aId := a.SortVal()
- bTimestamp, bId := b.SortVal()
- if aTimestamp == bTimestamp {
- return strings.Compare(aId, bId)
- }
- return int(aTimestamp - bTimestamp)
-}
-
-// The Message element indicates the message sent by a user
-type PostExport struct {
- XMLName xml.Name `xml:"Message"`
- MessageId string `xml:"MessageId"` // the message id in the db
- UserEmail string `xml:"LoginName"` // the email of the person that sent the post
- UserType shared.UserType `xml:"UserType"` // the type of the person that sent the post: "user" or "bot"
- CreateAt int64 `xml:"DateTimeUTC"` // utc timestamp (unix milliseconds), the post's createAt
-
- // Allows us to differentiate between:
- // - "EditedOriginalMsg": the newly created message (new Id), which holds the pre-edited message contents. The
- // "EditedNewMsgId" field will point to the message (original Id) which has the post-edited message content.
- // - "EditedNewMsg": the post-edited message content. This is confusing, so be careful: in the db, this EditedNewMsg
- // is actually the original messageId because we wanted an edited message to have the same messageId as the
- // pre-edited message. But for the purposes of exporting and to keep the mental model clear for end-users, we are
- // calling this the EditedNewMsg and EditedNewMsgId, because this will hold the NEW post-edited message contents,
- // and that's what's important to the end-user viewing the export.
- // - "UpdatedNoMsgChange": the message content hasn't changed, but the post was updated for some reason (reaction,
- // replied-to, a reply was edited, a reply was deleted (as of 10.2), perhaps other reasons)
- // - "Deleted": the message was deleted.
- // - "FileDeleted": this message is recording that a file was deleted.
- UpdatedType shared.PostUpdatedType `xml:"UpdatedType,omitempty"`
- UpdateAt int64 `xml:"UpdatedDateTimeUTC,omitempty"` // if this is an updated post, this is the updated time (same as deleted time for deleted posts).
-
- // when a message is edited, the EditedOriginalMsg points to the message Id that now has the newly edited message.
- EditedNewMsgId string `xml:"EditedNewMsgId,omitempty"`
-
- Message string `xml:"Content"` // the text body of the post
- PreviewsPost string `xml:"PreviewsPost,omitempty"` // the post id of the post that is previewed by the permalink preview feature
-}
-
-func (p PostExport) SortVal() (int64, string) {
- // updated messages are sorted by UpdateAt
- if p.UpdatedType != "" {
- return p.UpdateAt, p.MessageId
- }
- return p.CreateAt, p.MessageId
-}
-
-// The FileTransferStarted element indicates the beginning of a file transfer in a conversation
-type FileUploadStartExport struct {
- XMLName xml.Name `xml:"FileTransferStarted"`
- UserEmail string `xml:"LoginName"` // the email of the person that sent the file
- UploadStartTime int64 `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user started the upload. Example: 1366611728
- Filename string `xml:"UserFileName"` // the name of the file that was uploaded
- FilePath string `xml:"FileName"` // the path to the file, as stored on the server
-}
-
-func (f FileUploadStartExport) SortVal() (int64, string) {
- // file messages should come after the message they were with
- return f.UploadStartTime, EnsureLastMessageInASort
-}
-
-// The FileTransferEnded element indicates the end of a file transfer in a conversation
-type FileUploadStopExport struct {
- XMLName xml.Name `xml:"FileTransferEnded"`
- UserEmail string `xml:"LoginName"` // the email of the person that sent the file
- UploadStopTime int64 `xml:"DateTimeUTC"` // utc timestamp (seconds), time at which the user finished the upload. Example: 1366611728
- Filename string `xml:"UserFileName"` // the name of the file that was uploaded
- FilePath string `xml:"FileName"` // the path to the file, as stored on the server
- Status string `xml:"Status"` // set to either "Completed" or "Failed" depending on the outcome of the upload operation
-}
-
-func (f FileUploadStopExport) SortVal() (int64, string) {
- // file messages should come after the message they were with
- return f.UploadStopTime, EnsureLastMessageInASort
-}
-
-func ActianceExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
- start := time.Now()
-
- // Build the channel exports for the channels that had post or user join/leave activity this batch.
- exportData, err := shared.GetGenericExportData(p)
- if err != nil {
- return exportData.Results, err
- }
- var allUploadedFiles []*model.FileInfo
-
- // Convert the generic shared.ChannelExports to the Actiance-specific ChannelExports data.
- channelExports := make([]ChannelExport, 0, len(exportData.Exports))
- for _, channel := range exportData.Exports {
- joinEvents := make([]JoinExport, 0, len(channel.JoinEvents))
- leaveEvents := make([]LeaveExport, 0, len(channel.LeaveEvents))
-
- for _, j := range channel.JoinEvents {
- joinEvents = append(joinEvents, JoinExport{
- UserEmail: j.UserEmail,
- UserType: string(j.UserType),
- JoinTime: j.JoinTime,
- CorporateEmailID: j.UserEmail,
- })
- }
- for _, l := range channel.LeaveEvents {
- leaveEvents = append(leaveEvents, LeaveExport{
- UserEmail: l.UserEmail,
- UserType: string(l.UserType),
- LeaveTime: l.LeaveTime,
- CorporateEmailID: l.UserEmail,
- })
- }
-
- elements := make([]Sortable, 0, len(channel.Posts)+len(channel.DeletedFiles)+len(channel.UploadStarts)+len(channel.UploadStops))
- for _, p := range channel.Posts {
- elements = append(elements, PostExport{
- MessageId: *p.PostId,
- UserEmail: *p.UserEmail,
- UserType: p.UserType,
- CreateAt: *p.PostCreateAt,
- UpdatedType: p.UpdatedType,
- UpdateAt: p.UpdateAt,
- EditedNewMsgId: p.EditedNewMsgId,
- Message: p.Message,
- PreviewsPost: p.PreviewsPost,
- })
- }
- for _, p := range channel.DeletedFiles {
- elements = append(elements, PostExport{
- MessageId: *p.PostId,
- UserEmail: *p.UserEmail,
- UserType: p.UserType,
- CreateAt: *p.PostCreateAt,
- UpdatedType: p.UpdatedType,
- UpdateAt: p.UpdateAt,
- EditedNewMsgId: p.EditedNewMsgId,
- Message: p.Message,
- PreviewsPost: p.PreviewsPost,
- })
- }
- for _, u := range channel.UploadStarts {
- elements = append(elements, FileUploadStartExport{
- UserEmail: u.UserEmail,
- UploadStartTime: u.UploadStartTime,
- Filename: u.FileInfo.Name,
- FilePath: u.FileInfo.Path,
- })
- }
- for _, u := range channel.UploadStops {
- elements = append(elements, FileUploadStopExport{
- UserEmail: u.UserEmail,
- UploadStopTime: u.UploadStopTime,
- Filename: u.FileInfo.Name,
- FilePath: u.FileInfo.Path,
- Status: u.Status,
- })
- }
-
- // We need to sort all the elements by (updateAt, messageId) because they were added by type above.
- slices.SortStableFunc(elements, SortableSort)
-
- channelExports = append(channelExports, ChannelExport{
- Perspective: channel.DisplayName,
- ChannelId: channel.ChannelId,
- RoomId: fmt.Sprintf("%v - %v - %v", shared.ChannelTypeDisplayName(channel.ChannelType),
- channel.ChannelName, channel.ChannelId),
- StartTime: channel.StartTime,
- JoinEvents: joinEvents,
- Elements: elements,
- LeaveEvents: leaveEvents,
- EndTime: channel.EndTime,
- })
-
- allUploadedFiles = append(allUploadedFiles, channel.Files...)
- }
-
- export := &RootNode{
- XMLNS: XMLNS,
- Channels: channelExports,
- }
-
- results := exportData.Results
- results.ProcessingPostsMs = time.Since(start).Milliseconds()
-
- results.WriteExportResult, err = writeExport(rctx, export, allUploadedFiles, p.ExportBackend, p.FileAttachmentBackend, p.BatchPath)
- results.NumChannels = len(channelExports)
- return results, err
-}
-
-func writeExport(rctx request.CTX, export *RootNode, uploadedFiles []*model.FileInfo, exportBackend filestore.FileBackend, fileAttachmentBackend filestore.FileBackend, batchPath string) (res shared.WriteExportResult, err error) {
- start := time.Now()
- // marshal the export object to xml
- xmlData := &bytes.Buffer{}
- xmlData.WriteString(xml.Header)
-
- enc := xml.NewEncoder(xmlData)
- enc.Indent("", " ")
- if err = enc.Encode(export); err != nil {
- return res, fmt.Errorf("unable to convert export to XML: %w", err)
- }
- if err = enc.Flush(); err != nil {
- return res, fmt.Errorf("unable to flush the XML encoder: %w", err)
- }
-
- // Write this batch to a tmp zip, then copy the zip to the export directory.
- // Using a 2M buffer because the file backend may be s3 and this optimizes speed and
- // memory usage, see: https://github.com/mattermost/mattermost/pull/26629
- buf := make([]byte, 1024*1024*2)
- temp, err := os.CreateTemp("", "compliance-export-batch-*.zip")
- if err != nil {
- return res, fmt.Errorf("unable to create the batch temporary file: %w", err)
- }
- defer file.DeleteTemp(rctx.Logger(), temp)
-
- zipFile := zip.NewWriter(temp)
- w, err := zipFile.Create(ActianceExportFilename)
- if err != nil {
- return res, fmt.Errorf("unable to create the xml file in the zipFile created with the batch temporary file: %w", err)
- }
- if _, err = io.CopyBuffer(w, xmlData, buf); err != nil {
- return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
- }
- res.ProcessingXmlMs = time.Since(start).Milliseconds()
-
- start = time.Now()
-
- var missingFiles []string
- for _, fileInfo := range uploadedFiles {
- var attachmentReader io.ReadCloser
- attachmentReader, err = fileAttachmentBackend.Reader(fileInfo.Path)
- if err != nil {
- missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringBackendRead+" - "+fileInfo.Path)
- rctx.Logger().Warn(shared.MissingFileMessageDuringBackendRead,
- mlog.String("filename", fileInfo.Path),
- mlog.Err(err),
- )
- continue
- }
-
- // There could be many uploadedFiles, so be careful about closing readers.
- if err = func() error {
- defer attachmentReader.Close()
- var zipWriter io.Writer
- zipWriter, err = zipFile.Create(fileInfo.Path)
- if err != nil {
- return err
- }
-
- if _, err = io.CopyBuffer(zipWriter, attachmentReader, buf); err != nil {
- return err
- }
-
- return nil
- }(); err != nil {
- // s3 only errors _here_ if the object key wasn't found. So to handle that: if there is a read
- // error (even for local), let's add a warning instead of failing the export.
- // Failing the export would fail the entire export run, and every future run would also fail on
- // this non-existent file -- not good.
- missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringCopy+" - "+fileInfo.Path)
- rctx.Logger().Warn(shared.MissingFileMessageDuringCopy,
- mlog.String("filename", fileInfo.Path),
- mlog.Err(err),
- )
- }
- }
-
- res.TransferringFilesMs = time.Since(start).Milliseconds()
- res.NumWarnings = len(missingFiles)
- if res.NumWarnings > 0 {
- var w io.Writer
- w, err = zipFile.Create(ActianceWarningFilename)
- if err != nil {
- return res, fmt.Errorf("unable to create the warning file in the zipFile created with the batch temporary file: %w", err)
- }
- r := strings.NewReader(strings.Join(missingFiles, "\n"))
- if _, err = io.CopyBuffer(w, r, buf); err != nil {
- return res, fmt.Errorf("unable to write into the zipFile created with the batch temporary file: %w", err)
- }
- }
-
- if err = zipFile.Close(); err != nil {
- return res, fmt.Errorf("unable to close the zipFile created with the batch temporary file: %w", err)
- }
-
- _, err = temp.Seek(0, io.SeekStart)
- if err != nil {
- return res, fmt.Errorf("unable to seek to the beginning of the the batch temporary file: %w", err)
- }
-
- start = time.Now()
-
- // Try to write the file without a timeout due to the potential size of the file.
- _, err = filestore.TryWriteFileContext(rctx.Context(), exportBackend, temp, batchPath)
- if err != nil {
- return res, fmt.Errorf("unable to transfer the batch zip to the file backend: %w", err)
- }
-
- res.TransferringZipMs = time.Since(start).Milliseconds()
-
- return res, nil
-}
diff --git a/server/enterprise/message_export/actiance_export/actiance_export_test.go b/server/enterprise/message_export/actiance_export/actiance_export_test.go
deleted file mode 100644
index 86a4ee99282..00000000000
--- a/server/enterprise/message_export/actiance_export/actiance_export_test.go
+++ /dev/null
@@ -1,2336 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package actiance_export
-
-import (
- "archive/zip"
- "bytes"
- "encoding/xml"
- "fmt"
- "io"
- "os"
- "path"
- "strings"
- "testing"
-
- "github.com/mattermost/mattermost/server/public/shared/i18n"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-type MyReporter struct {
- mock.Mock
-}
-
-func (mr *MyReporter) ReportProgressMessage(message string) {
- mr.Called(message)
-}
-
-func TestActianceExport(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- runTestActianceExport(t, fileBackend, fileBackend)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(attachmentTempDir)
- assert.NoError(t, err)
- })
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
-
- runTestActianceExport(t, exportBackend, attachmentBackend)
- })
-}
-
-func runTestActianceExport(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
- rctx := request.TestContext(t)
- rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
-
- chanTypeDirect := model.ChannelTypeDirect
- actianceExportTests := []struct {
- name string
- jobEndTime int64
- activity []string
- channels model.ChannelList
- cmhs map[string][]*model.ChannelMemberHistoryResult
- posts []*model.MessageExport
- attachments map[string][]*model.FileInfo
- expectedData string
- expectedFiles int
- }{
- {
- name: "empty",
- jobEndTime: 30,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{},
- posts: []*model.MessageExport{},
- attachments: map[string][]*model.FileInfo{},
- expectedData: strings.Join([]string{
- xml.Header,
- "",
- }, ""),
- activity: []string{},
- expectedFiles: 2,
- },
- {
- name: "posts",
- jobEndTime: 500,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
- {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
- {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
- {JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostEditAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("edited message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("original message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- // deleted post
- {
- PostId: model.NewPointer("post-id2"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(4)),
- PostDeleteAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("message2"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- {
- PostId: model.NewPointer("post-id3"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message3"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test_bot@email\n",
- " bot\n",
- " 10\n",
- " test_bot@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " post-id2\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message2\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " EditedOriginalMsg\n",
- " 2\n",
- " post-original-id\n",
- " original message\n",
- " \n",
- " \n",
- " post-original-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " EditedNewMsg\n",
- " 2\n",
- " edited message\n",
- " \n",
- " \n",
- " post-id2\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " Deleted\n",
- " 4\n",
- " delete message2\n",
- " \n",
- " \n",
- " post-id3\n",
- " test@test.com\n",
- " user\n",
- " 100\n",
- " message3\n",
- " \n",
- " \n",
- " test_bot@email\n",
- " bot\n",
- " 20\n",
- " test_bot@email\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 500\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 500\n",
- " test@test.com\n",
- " \n",
- " 500\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 2,
- },
- {
- name: "post with permalink preview",
- jobEndTime: 600,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400))},
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer(`{"previewed_post":"n4w39mc1ff8y5fite4b8hacy1w"}`),
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer(`{"disable_group_highlight":true}`),
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " n4w39mc1ff8y5fite4b8hacy1w\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 100\n",
- " message\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 600\n",
- " test@test.com\n",
- " \n",
- " 600\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 0,
- },
- {
- name: "posts with attachments",
- jobEndTime: 700,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
- },
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- },
- },
- },
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " Completed\n",
- " \n",
- " \n",
- " post-id-2\n",
- " test@test.com\n",
- " user\n",
- " 100\n",
- " message\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 700\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 700\n",
- " test@test.com\n",
- " \n",
- " 700\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 3,
- },
-
- {
- name: "posts with deleted attachments, no deleted post, and at different time from original post",
- jobEndTime: 700,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
- },
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- UpdateAt: 2,
- DeleteAt: 2,
- },
- },
- },
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message1\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " Completed\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " FileDeleted\n",
- " 2\n",
- " delete test1-attachment\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 700\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 700\n",
- " test@test.com\n",
- " \n",
- " 700\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 3,
- },
- {
- name: "posts with deleted attachments and deleted post",
- jobEndTime: 700,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3",
- },
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("message1"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- UpdateAt: 2,
- DeleteAt: 2,
- },
- },
- },
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message1\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " \n",
- " \n",
- " test@test.com\n",
- " 1\n",
- " test1-attachment\n",
- " test1-attachment\n",
- " Completed\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " Deleted\n",
- " 2\n",
- " delete message1\n",
- " \n",
- " \n",
- " post-id-1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " FileDeleted\n",
- " 2\n",
- " delete test1-attachment\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 700\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 700\n",
- " test@test.com\n",
- " \n",
- " 700\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 3,
- },
- {
- name: "joins and leaves after last post, one batch, and post from bot",
- jobEndTime: 500,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
- {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
- {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
- {JoinTime: 450, ChannelId: "channel-id", UserId: "test4", UserEmail: "test4@email", Username: "test4name", LeaveTime: model.NewPointer(int64(460))},
- {JoinTime: 10, ChannelId: "channel-id", UserId: "test-bot", UserEmail: "test-bot@email", Username: "test-botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id2"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("edit message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id3"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(4)),
- PostDeleteAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- {
- PostId: model.NewPointer("post-id5"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(20)),
- PostUpdateAt: model.NewPointer(int64(20)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test-bot@email"),
- UserId: model.NewPointer("test-bot"),
- Username: model.NewPointer("test-botname"),
- IsBot: true,
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id4"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedData: strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test-bot@email\n",
- " bot\n",
- " 10\n",
- " test-bot@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " test4@email\n",
- " user\n",
- " 450\n",
- " test4@email\n",
- " \n",
- " \n",
- " post-id1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " post-id3\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " post-id2\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " UpdatedNoMsgChange\n",
- " 2\n",
- " edit message\n",
- " \n",
- " \n",
- " post-id3\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " Deleted\n",
- " 4\n",
- " delete message\n",
- " \n",
- " \n",
- " post-id5\n",
- " test-bot@email\n",
- " bot\n",
- " 20\n",
- " message\n",
- " \n",
- " \n",
- " post-id4\n",
- " test@test.com\n",
- " user\n",
- " 100\n",
- " message\n",
- " \n",
- " \n",
- " test-bot@email\n",
- " bot\n",
- " 20\n",
- " test-bot@email\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test4@email\n",
- " user\n",
- " 460\n",
- " test4@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 500\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 500\n",
- " test@test.com\n",
- " \n",
- " 500\n",
- " \n",
- "",
- }, ""),
- expectedFiles: 2,
- },
- }
-
- for _, tt := range actianceExportTests {
- t.Run(tt.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tt.attachments) > 0 {
- for post_id, attachments := range tt.attachments {
- call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false)
- call.Run(func(args mock.Arguments) {
- call.Return(attachments, nil)
- })
- _, err := attachmentBackend.WriteFile(bytes.NewReader([]byte{}), attachments[0].Path)
- require.NoError(t, err)
-
- t.Cleanup(func() {
- err = attachmentBackend.RemoveFile(attachments[0].Path)
- require.NoError(t, err)
- })
- }
- }
-
- if len(tt.cmhs) > 0 {
- for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
- Return(cmhs, nil)
- }
- }
-
- if tt.activity != nil {
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return(tt.activity, nil)
- }
-
- if tt.channels != nil {
- mockStore.ChannelStore.On("GetMany", tt.activity, true).
- Return(tt.channels, nil)
- }
-
- myMockReporter := MyReporter{}
- defer myMockReporter.AssertExpectations(t)
- if len(tt.activity) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
- }
- if len(tt.cmhs) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
- }
-
- channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
- shared.ChannelExportsParams{
- Store: shared.NewMessageExportStore(mockStore),
- ExportPeriodStartTime: 1,
- ExportPeriodEndTime: tt.jobEndTime,
- ChannelBatchSize: 100,
- ChannelHistoryBatchSize: 100,
- ReportProgressMessage: myMockReporter.ReportProgressMessage,
- })
- assert.NoError(t, err)
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
- res, err := ActianceExport(rctx, shared.ExportParams{
- ChannelMetadata: channelMetadata,
- Posts: tt.posts,
- ChannelMemberHistories: channelMemberHistories,
- BatchPath: exportFileName,
- BatchStartTime: 1,
- BatchEndTime: tt.jobEndTime,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- })
- assert.NoError(t, err)
- assert.Equal(t, 0, res.NumWarnings)
-
- zipBytes, err := exportBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- actiancexml, err := zipReader.File[0].Open()
- require.NoError(t, err)
- defer actiancexml.Close()
- xmlData, err := io.ReadAll(actiancexml)
- assert.NoError(t, err)
-
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><> actual:\n%s\n", string(xmlData))
- assert.Equal(t, tt.expectedData, string(xmlData))
-
- t.Cleanup(func() {
- err = exportBackend.RemoveFile(exportFileName)
- assert.NoError(t, err)
- })
- })
- }
-}
-
-func TestActianceExportMultipleBatches(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- runTestActianceExportMultipleBatches(t, fileBackend, fileBackend)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(attachmentTempDir)
- assert.NoError(t, err)
- })
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
-
- runTestActianceExportMultipleBatches(t, exportBackend, attachmentBackend)
- })
-}
-
-func runTestActianceExportMultipleBatches(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
- rctx := request.TestContext(t)
- rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
-
- chanTypeDirect := model.ChannelTypeDirect
- actianceMultiBatchExportTests := []struct {
- name string
- jobStartTime int64
- jobEndTime int64
- numBatches int
- activity []string
- channels model.ChannelList
- cmhs map[string][]*model.ChannelMemberHistoryResult
- posts [][]*model.MessageExport
- attachments map[string][]*model.FileInfo
- expectedData []string
- expectedFiles int
- }{
- {
- name: "joins and leaves after last post, and before second batch, two batches",
- jobStartTime: 1,
- jobEndTime: 500,
- numBatches: 2,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- // will be included in both batches:
- {JoinTime: 0, ChannelId: "channel-id", UserId: "test", UserEmail: "test@email", Username: "testname", LeaveTime: model.NewPointer(int64(400))},
- // Only first batch:
- {JoinTime: 2, ChannelId: "channel-id", UserId: "testA", UserEmail: "testA@email", Username: "testAname", LeaveTime: model.NewPointer(int64(3))},
- // Only second batch:
- {JoinTime: 8, ChannelId: "channel-id", UserId: "test2", UserEmail: "test2@email", Username: "test2name", LeaveTime: model.NewPointer(int64(80))},
- {JoinTime: 10, ChannelId: "channel-id", UserId: "test_bot", UserEmail: "test_bot@email", Username: "test_botname", IsBot: true, LeaveTime: model.NewPointer(int64(20))},
- {JoinTime: 400, ChannelId: "channel-id", UserId: "test3", UserEmail: "test3@email", Username: "test3name"},
- {JoinTime: 450, ChannelId: "channel-id", UserId: "test4", UserEmail: "test4@email", Username: "test4name", LeaveTime: model.NewPointer(int64(460))},
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: [][]*model.MessageExport{
- {
- {
- PostId: model.NewPointer("post-id1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id2"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(4)),
- PostDeleteAt: model.NewPointer(int64(4)),
- PostMessage: model.NewPointer("edit message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- {
- {
- PostId: model.NewPointer("post-id3"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(5)),
- PostDeleteAt: model.NewPointer(int64(5)),
- PostMessage: model.NewPointer("message2"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- {
- PostId: model.NewPointer("post-id4"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message3"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- }},
- attachments: map[string][]*model.FileInfo{},
- expectedData: []string{
- strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " test@test.com\n",
- " \n",
- " \n",
- " testA@email\n",
- " user\n",
- " 2\n",
- " testA@email\n",
- " \n",
- " \n",
- " post-id1\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " post-id2\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " EditedOriginalMsg\n",
- " 4\n",
- " post-original-id\n",
- " edit message\n",
- " \n",
- " \n",
- " testA@email\n",
- " user\n",
- " 3\n",
- " testA@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 4\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 4\n",
- " test@test.com\n",
- " \n",
- " 4\n",
- " \n",
- "",
- }, ""),
- strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 4\n",
- " \n",
- " test@email\n",
- " user\n",
- " 0\n",
- " test@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 4\n",
- " test@test.com\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 8\n",
- " test2@email\n",
- " \n",
- " \n",
- " test_bot@email\n",
- " bot\n",
- " 10\n",
- " test_bot@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 400\n",
- " test3@email\n",
- " \n",
- " \n",
- " test4@email\n",
- " user\n",
- " 450\n",
- " test4@email\n",
- " \n",
- " \n",
- " post-id3\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message2\n",
- " \n",
- " \n",
- " post-id3\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " Deleted\n",
- " 5\n",
- " delete message2\n",
- " \n",
- " \n",
- " post-id4\n",
- " test@test.com\n",
- " user\n",
- " 100\n",
- " message3\n",
- " \n",
- " \n",
- " test_bot@email\n",
- " bot\n",
- " 20\n",
- " test_bot@email\n",
- " \n",
- " \n",
- " test2@email\n",
- " user\n",
- " 80\n",
- " test2@email\n",
- " \n",
- " \n",
- " test@email\n",
- " user\n",
- " 400\n",
- " test@email\n",
- " \n",
- " \n",
- " test4@email\n",
- " user\n",
- " 460\n",
- " test4@email\n",
- " \n",
- " \n",
- " test3@email\n",
- " user\n",
- " 500\n",
- " test3@email\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 500\n",
- " test@test.com\n",
- " \n",
- " 500\n",
- " \n",
- "",
- }, ""),
- },
- expectedFiles: 2,
- },
- }
-
- for _, tt := range actianceMultiBatchExportTests {
- t.Run(tt.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tt.attachments) > 0 {
- for post_id, attachments := range tt.attachments {
- call := mockStore.FileInfoStore.On("GetForPost", post_id, true, true, false)
- call.Run(func(args mock.Arguments) {
- call.Return(attachments, nil)
- })
- _, err := attachmentBackend.WriteFile(bytes.NewReader([]byte{}), attachments[0].Path)
- require.NoError(t, err)
-
- t.Cleanup(func() {
- err = attachmentBackend.RemoveFile(attachments[0].Path)
- require.NoError(t, err)
- })
- }
- }
-
- if len(tt.cmhs) > 0 {
- for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
- Return(cmhs, nil)
- }
- }
-
- if tt.activity != nil {
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return(tt.activity, nil)
- }
-
- if tt.channels != nil {
- mockStore.ChannelStore.On("GetMany", tt.activity, true).
- Return(tt.channels, nil)
- }
-
- myMockReporter := MyReporter{}
- defer myMockReporter.AssertExpectations(t)
- if len(tt.activity) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
- }
- if len(tt.cmhs) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
- }
-
- channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
- shared.ChannelExportsParams{
- Store: shared.NewMessageExportStore(mockStore),
- ExportPeriodStartTime: 1,
- ExportPeriodEndTime: tt.jobEndTime,
- ChannelBatchSize: 100,
- ChannelHistoryBatchSize: 100,
- ReportProgressMessage: myMockReporter.ReportProgressMessage,
- })
- assert.NoError(t, err)
-
- batchStartTime := int64(1)
-
- for batch := 0; batch < tt.numBatches; batch++ {
- var batchEndTime int64
- if batch == tt.numBatches-1 {
- batchEndTime = tt.jobEndTime
- } else {
- batchEndTime = *tt.posts[batch][len(tt.posts[batch])-1].PostUpdateAt
- }
- exportFileName := path.Join("export", "jobName",
- fmt.Sprintf("jobName-batch00%d.zip", batch+1))
-
- res, err := ActianceExport(rctx, shared.ExportParams{
- ChannelMetadata: channelMetadata,
- Posts: tt.posts[batch],
- ChannelMemberHistories: channelMemberHistories,
- BatchPath: exportFileName,
- BatchStartTime: batchStartTime,
- BatchEndTime: batchEndTime,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- })
- assert.NoError(t, err)
- assert.Equal(t, 0, res.NumWarnings)
-
- zipBytes, err := exportBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- actiancexml, err := zipReader.File[0].Open()
- require.NoError(t, err)
- xmlData, err := io.ReadAll(actiancexml)
- actiancexml.Close()
- assert.NoError(t, err)
-
- assert.Equal(t, tt.expectedData[batch], string(xmlData), fmt.Sprintf("batch %v", batch))
-
- batchStartTime = *tt.posts[batch][len(tt.posts[batch])-1].PostUpdateAt
-
- t.Cleanup(func() {
- err = exportBackend.RemoveFile(exportFileName)
- assert.NoError(t, err)
- })
- }
- })
- }
-}
-
-func TestMultipleActianceExport(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- runTestMultipleActianceExport(t, fileBackend, fileBackend)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(attachmentTempDir)
- assert.NoError(t, err)
- })
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
-
- runTestMultipleActianceExport(t, exportBackend, attachmentBackend)
- })
-}
-
-func runTestMultipleActianceExport(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
- rctx := request.TestContext(t)
- rctx = rctx.WithT(i18n.IdentityTfunc()).(*request.Context)
-
- chanTypeDirect := model.ChannelTypeDirect
- multActianceExportTests := []struct {
- name string
- jobEndTime int64
- activity []string
- channels model.ChannelList
- cmhs map[string][]*model.ChannelMemberHistoryResult
- posts map[string][]*model.MessageExport
- attachments map[string][]*model.FileInfo
- expectedData map[string]string
- expectedFiles int
- }{
- {
- name: "post,export,delete,export",
- jobEndTime: 500,
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {JoinTime: 0, ChannelId: "channel-id", UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(400))},
- },
- },
- posts: map[string][]*model.MessageExport{
- "step1": {
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- "step2": {
- {
- PostId: model.NewPointer("post-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- },
- },
- expectedData: map[string]string{
- "step1": strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 0\n",
- " test@test.com\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 400\n",
- " test@test.com\n",
- " \n",
- " 500\n",
- " \n",
- "",
- }, ""),
- // We're redoing the export completely, so we'll get the original message then the deleted record
- "step2": strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 0\n",
- " test@test.com\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " post-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " Deleted\n",
- " 2\n",
- " delete message\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 400\n",
- " test@test.com\n",
- " \n",
- " 500\n",
- " \n",
- "",
- }, ""),
- },
- expectedFiles: 2,
- },
- {
- name: "post,export,edit,export",
- jobEndTime: 600,
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {JoinTime: 0, ChannelId: "channel-id", UserId: "user-id", UserEmail: "test@test.com", Username: "username", LeaveTime: model.NewPointer(int64(450))},
- },
- },
- activity: []string{"channel-id"},
- channels: model.ChannelList{{
- TeamId: "team-id",
- Id: "channel-id",
- Name: "channel-name",
- DisplayName: "channel-display-name",
- Type: model.ChannelTypeDirect,
- }},
- posts: map[string][]*model.MessageExport{
- "step1": {
- {
- PostId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- "step2": {
- // new post which holds the original message contents
- {
- PostId: model.NewPointer("post-id-new"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostDeleteAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- // old post which has been edited
- {
- PostId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(2)),
- PostEditAt: model.NewPointer(int64(2)),
- PostMessage: model.NewPointer("edit message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- },
- expectedData: map[string]string{
- "step1": strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 0\n",
- " test@test.com\n",
- " \n",
- " \n",
- " post-original-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " message\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 450\n",
- " test@test.com\n",
- " \n",
- " 600\n",
- " \n",
- "",
- }, ""),
- // We're redoing the export completely, so we'll get the original message then the deleted record
- "step2": strings.Join([]string{
- xml.Header,
- "\n",
- " \n",
- " direct - channel-name - channel-id\n",
- " 1\n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 0\n",
- " test@test.com\n",
- " \n",
- " \n",
- " post-id-new\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " EditedOriginalMsg\n",
- " 2\n",
- " post-original-id\n",
- " message\n",
- " \n",
- " \n",
- " post-original-id\n",
- " test@test.com\n",
- " user\n",
- " 1\n",
- " EditedNewMsg\n",
- " 2\n",
- " edit message\n",
- " \n",
- " \n",
- " test@test.com\n",
- " user\n",
- " 450\n",
- " test@test.com\n",
- " \n",
- " 600\n",
- " \n",
- "",
- }, ""),
- },
- expectedFiles: 2,
- },
- }
-
- for _, tt := range multActianceExportTests {
- t.Run(tt.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tt.cmhs) > 0 {
- for channelId, cmhs := range tt.cmhs {
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), tt.jobEndTime, []string{channelId}).
- Return(cmhs, nil)
- }
- }
-
- if tt.activity != nil {
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return(tt.activity, nil)
- }
-
- if tt.channels != nil {
- mockStore.ChannelStore.On("GetMany", tt.activity, true).
- Return(tt.channels, nil)
- }
-
- myMockReporter := MyReporter{}
- defer myMockReporter.AssertExpectations(t)
- if len(tt.activity) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.channel_message")
- }
- if len(tt.cmhs) > 0 {
- myMockReporter.On("ReportProgressMessage", "ent.message_export.actiance_export.calculate_channel_exports.activity_message")
- }
-
- channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
- shared.ChannelExportsParams{
- Store: shared.NewMessageExportStore(mockStore),
- ExportPeriodStartTime: 1,
- ExportPeriodEndTime: tt.jobEndTime,
- ChannelBatchSize: 100,
- ChannelHistoryBatchSize: 100,
- ReportProgressMessage: myMockReporter.ReportProgressMessage,
- })
- assert.NoError(t, err)
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
- res, err := ActianceExport(rctx, shared.ExportParams{
- ChannelMetadata: channelMetadata,
- Posts: tt.posts["step1"],
- ChannelMemberHistories: channelMemberHistories,
- BatchPath: exportFileName,
- BatchStartTime: 1,
- BatchEndTime: tt.jobEndTime,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- })
-
- assert.NoError(t, err)
- assert.Equal(t, 0, res.NumWarnings)
-
- zipBytes, err := exportBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- actiancexml, err := zipReader.File[0].Open()
- require.NoError(t, err)
- defer actiancexml.Close()
- xmlData, err := io.ReadAll(actiancexml)
- assert.NoError(t, err)
-
- assert.Equal(t, tt.expectedData["step1"], string(xmlData))
-
- res, err = ActianceExport(rctx, shared.ExportParams{
- ChannelMetadata: channelMetadata,
- Posts: tt.posts["step2"],
- ChannelMemberHistories: channelMemberHistories,
- BatchPath: exportFileName,
- BatchStartTime: 1,
- BatchEndTime: tt.jobEndTime,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- })
- assert.NoError(t, err)
- assert.Equal(t, 0, res.NumWarnings)
-
- zipBytes, err = exportBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
- zipReader, err = zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- actiancexml, err = zipReader.File[0].Open()
- require.NoError(t, err)
- defer actiancexml.Close()
- xmlData, err = io.ReadAll(actiancexml)
- assert.NoError(t, err)
-
- assert.Equal(t, tt.expectedData["step2"], string(xmlData))
-
- t.Cleanup(func() {
- err = exportBackend.RemoveFile(exportFileName)
- assert.NoError(t, err)
- })
- })
- }
-}
-
-func TestWriteExportWarnings(t *testing.T) {
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(tempDir)
- assert.NoError(t, err)
- })
-
- config := filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: tempDir,
- }
-
- fileBackend, err := filestore.NewFileBackend(config)
- assert.NoError(t, err)
-
- rctx := request.TestContext(t)
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- // Do not create the files, we want them to error
- uploadedFiles := []*model.FileInfo{
- {Name: "test", Id: "12345", Path: "missing.txt"},
- {Name: "test2", Id: "54321", Path: "missing.txt"},
- }
- export := &RootNode{
- XMLNS: XMLNS,
- Channels: []ChannelExport{},
- }
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
- res, err := writeExport(rctx, export, uploadedFiles, fileBackend, fileBackend, exportFileName)
- assert.NoError(t, err)
- assert.Equal(t, 2, res.NumWarnings)
-
- err = fileBackend.RemoveFile(exportFileName)
- require.NoError(t, err)
-}
-
-func Test_channelHasActivity(t *testing.T) {
- tests := []struct {
- name string
- cmhs []*model.ChannelMemberHistoryResult
- startTime int64
- endTime int64
- want bool
- }{
- {
- name: "no activity",
- cmhs: nil,
- startTime: 1000,
- endTime: 2000,
- want: false,
- },
- {
- name: "no activity in bounds (but activity out of bounds)",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 900,
- LeaveTime: nil,
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- {
- ChannelId: "channelid",
- UserId: "testid2",
- JoinTime: 2100,
- LeaveTime: nil,
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: false,
- },
- {
- name: "join on lower bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 1000,
- LeaveTime: nil,
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- {
- name: "join within bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 1500,
- LeaveTime: nil,
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- {
- name: "join on upper bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 2000,
- LeaveTime: nil,
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- {
- name: "leave on lower bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 100,
- LeaveTime: model.NewPointer[int64](1000),
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- {
- name: "leave within bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 100,
- LeaveTime: model.NewPointer[int64](1500),
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- {
- name: "leave on upper bounds",
- cmhs: []*model.ChannelMemberHistoryResult{
- {
- ChannelId: "channelid",
- UserId: "testid",
- JoinTime: 100,
- LeaveTime: model.NewPointer[int64](2000),
- UserEmail: "testemail@email.com",
- Username: "test_username",
- IsBot: false,
- UserDeleteAt: 0,
- },
- },
- startTime: 1000,
- endTime: 2000,
- want: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- assert.Equalf(t, tt.want, shared.ChannelHasActivity(tt.cmhs, tt.startTime, tt.endTime), "channelHasActivity(%v, %v, %v)", tt.cmhs, tt.startTime, tt.endTime)
- })
- }
-}
diff --git a/server/enterprise/message_export/actiance_export/main_test.go b/server/enterprise/message_export/actiance_export/main_test.go
deleted file mode 100644
index 4275303b85a..00000000000
--- a/server/enterprise/message_export/actiance_export/main_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package actiance_export
-
-import (
- "testing"
-
- "github.com/mattermost/mattermost/server/v8/channels/api4"
- "github.com/mattermost/mattermost/server/v8/channels/testlib"
-)
-
-var mainHelper *testlib.MainHelper
-
-func TestMain(m *testing.M) {
- mainHelper = testlib.NewMainHelper()
- defer mainHelper.Close()
- api4.SetMainHelper(mainHelper)
-
- mainHelper.Main(m)
-}
diff --git a/server/enterprise/message_export/actiance_export/test_helpers.go b/server/enterprise/message_export/actiance_export/test_helpers.go
deleted file mode 100644
index 79e28a5957c..00000000000
--- a/server/enterprise/message_export/actiance_export/test_helpers.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package actiance_export
-
-import (
- "encoding/xml"
- "io"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-type ChannelExportWithSpecifics struct {
- XMLName xml.Name `xml:"Conversation"`
- Perspective string `xml:"Perspective,attr"`
- ChannelId string `xml:"-"`
- RoomId string `xml:"RoomID"`
- StartTime int64 `xml:"StartTimeUTC"`
- JoinEvents []*JoinExport `xml:"ParticipantEntered"`
- Messages []*PostExport `xml:"Message"`
- FileStarts []*FileUploadStartExport `xml:"FileTransferStarted"`
- FileStops []*FileUploadStopExport `xml:"FileTransferEnded"`
- LeaveEvents []*LeaveExport `xml:"ParticipantLeft"`
- EndTime int64 `xml:"EndTimeUTC"`
-}
-
-func GetChannelExports(t *testing.T, r io.Reader) []*ChannelExportWithSpecifics {
- decoder := xml.NewDecoder(r)
- var exportedChannels []*ChannelExportWithSpecifics
- for {
- token, err := decoder.Token()
- if token == nil || err != nil {
- break
- }
- switch se := token.(type) {
- case xml.StartElement:
- if se.Name.Local == "Conversation" {
- var a *ChannelExportWithSpecifics
- err = decoder.DecodeElement(&a, &se)
- require.NoError(t, err)
- exportedChannels = append(exportedChannels, a)
- }
- default:
- }
- }
-
- return exportedChannels
-}
diff --git a/server/enterprise/message_export/common_export/common_export.go b/server/enterprise/message_export/common_export/common_export.go
deleted file mode 100644
index fa31e1d739e..00000000000
--- a/server/enterprise/message_export/common_export/common_export.go
+++ /dev/null
@@ -1,240 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package common_export
-
-import (
- "fmt"
- "strconv"
-
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
-)
-
-const MissingFileMessage = "File missing for post; cannot copy file to archive"
-
-type ChannelMemberJoin struct {
- UserId string
- IsBot bool
- Email string
- Username string
- Datetime int64
-}
-
-type ChannelMemberLeave struct {
- UserId string
- IsBot bool
- Email string
- Username string
- Datetime int64
-}
-
-type ChannelMember struct {
- UserId string
- IsBot bool
- Email string
- Username string
-}
-
-type MetadataChannel struct {
- TeamId *string
- TeamName *string
- TeamDisplayName *string
- ChannelId string
- ChannelName string
- ChannelDisplayName string
- ChannelType model.ChannelType
- RoomId string
- StartTime int64
- EndTime int64
- MessagesCount int
- AttachmentsCount int
-}
-
-type Metadata struct {
- Channels map[string]*MetadataChannel
- MessagesCount int
- AttachmentsCount int
- StartTime int64
- EndTime int64
-}
-
-func (metadata *Metadata) Update(post *model.MessageExport, attachments int) {
- channelMetadata, ok := metadata.Channels[*post.ChannelId]
- if !ok {
- channelMetadata = &MetadataChannel{
- TeamId: post.TeamId,
- TeamName: post.TeamName,
- TeamDisplayName: post.TeamDisplayName,
- ChannelId: *post.ChannelId,
- ChannelName: *post.ChannelName,
- ChannelDisplayName: *post.ChannelDisplayName,
- ChannelType: *post.ChannelType,
- RoomId: fmt.Sprintf("%v - %v", ChannelTypeDisplayName(*post.ChannelType), *post.ChannelId),
- StartTime: *post.PostCreateAt,
- MessagesCount: 0,
- AttachmentsCount: 0,
- }
- }
-
- channelMetadata.EndTime = *post.PostCreateAt
- channelMetadata.AttachmentsCount += attachments
- metadata.AttachmentsCount += attachments
- channelMetadata.MessagesCount += 1
- metadata.MessagesCount += 1
- if metadata.StartTime == 0 {
- metadata.StartTime = *post.PostCreateAt
- }
- metadata.EndTime = *post.PostCreateAt
- metadata.Channels[*post.ChannelId] = channelMetadata
-}
-
-type ChannelExportsParams struct {
- Store shared.MessageExportStore
- ExportPeriodStartTime int64
- ExportPeriodEndTime int64
- ChannelBatchSize int
- ChannelHistoryBatchSize int
- ReportProgressMessage func(message string)
-}
-
-// CalculateChannelExports returns the channel info ( map[channelId]*MetadataChannel ) and the channel user
-// joins/leaves ( map[channelId][]*model.ChannelMemberHistoryResult ) for any channel that has had activity
-// (posts or user join/leaves) between ExportPeriodStartTime and ExportPeriodEndTime.
-func CalculateChannelExports(rctx request.CTX, opt ChannelExportsParams) (map[string]*MetadataChannel, map[string][]*model.ChannelMemberHistoryResult, error) {
- // Which channels had user activity in the export period?
- activeChannelIds, err := opt.Store.ChannelMemberHistory().GetChannelsWithActivityDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime)
- if err != nil {
- return nil, nil, err
- }
-
- if len(activeChannelIds) == 0 {
- return nil, nil, nil
- }
-
- rctx.Logger().Debug("Started CalculateChannelExports", mlog.Int("export_period_start_time", opt.ExportPeriodStartTime), mlog.Int("export_period_end_time", opt.ExportPeriodEndTime), mlog.Int("num_active_channel_ids", len(activeChannelIds)))
- message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.channel_message", model.StringMap{"NumChannels": strconv.Itoa(len(activeChannelIds))})
- opt.ReportProgressMessage(message)
-
- // For each channel, get its metadata.
- channelMetadata := make(map[string]*MetadataChannel, len(activeChannelIds))
-
- // Use batches to reduce db load and network waste.
- for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelBatchSize {
- upTo := min(pos+opt.ChannelBatchSize, len(activeChannelIds))
- batch := activeChannelIds[pos:upTo]
- channels, err := opt.Store.Channel().GetMany(batch, true)
- if err != nil {
- return nil, nil, err
- }
-
- for _, channel := range channels {
- channelMetadata[channel.Id] = &MetadataChannel{
- TeamId: model.NewPointer(channel.TeamId),
- ChannelId: channel.Id,
- ChannelName: channel.Name,
- ChannelDisplayName: channel.DisplayName,
- ChannelType: channel.Type,
- RoomId: fmt.Sprintf("%v - %v", ChannelTypeDisplayName(channel.Type), channel.Id),
- StartTime: opt.ExportPeriodStartTime,
- EndTime: opt.ExportPeriodEndTime,
- }
- }
- }
-
- historiesByChannelId := make(map[string][]*model.ChannelMemberHistoryResult, len(activeChannelIds))
-
- // Now that we have metadata, get channelMemberHistories for each channel.
- // Use batches to reduce total db load and network waste.
- for pos := 0; pos < len(activeChannelIds); pos += opt.ChannelHistoryBatchSize {
- // This may take a while, so update the system console UI.
- message := rctx.T("ent.message_export.actiance_export.calculate_channel_exports.activity_message", model.StringMap{
- "NumChannels": strconv.Itoa(len(activeChannelIds)),
- "NumCompleted": strconv.Itoa(pos),
- })
- opt.ReportProgressMessage(message)
-
- upTo := min(pos+opt.ChannelHistoryBatchSize, len(activeChannelIds))
- batch := activeChannelIds[pos:upTo]
- channelMemberHistories, err := opt.Store.ChannelMemberHistory().GetUsersInChannelDuring(opt.ExportPeriodStartTime, opt.ExportPeriodEndTime, batch)
- if err != nil {
- return nil, nil, err
- }
-
- // collect the channelMemberHistories by channelId
- for _, entry := range channelMemberHistories {
- historiesByChannelId[entry.ChannelId] = append(historiesByChannelId[entry.ChannelId], entry)
- }
- }
-
- return channelMetadata, historiesByChannelId, nil
-}
-
-func GetJoinsAndLeavesForChannel(startTime int64, endTime int64, channelMembersHistory []*model.ChannelMemberHistoryResult,
- postAuthors map[string]ChannelMember) ([]ChannelMemberJoin, []ChannelMemberLeave) {
- var joins []ChannelMemberJoin
- var leaves []ChannelMemberLeave
-
- alreadyJoined := make(map[string]bool)
- for _, cmh := range channelMembersHistory {
- if cmh.UserDeleteAt > 0 && cmh.UserDeleteAt < startTime {
- continue
- }
-
- if cmh.JoinTime > endTime {
- continue
- }
-
- if cmh.LeaveTime != nil && *cmh.LeaveTime < startTime {
- continue
- }
-
- if cmh.JoinTime <= endTime {
- joins = append(joins, ChannelMemberJoin{
- UserId: cmh.UserId,
- IsBot: cmh.IsBot,
- Email: cmh.UserEmail,
- Username: cmh.Username,
- Datetime: cmh.JoinTime,
- })
- alreadyJoined[cmh.UserId] = true
- }
-
- if cmh.LeaveTime != nil && *cmh.LeaveTime <= endTime {
- leaves = append(leaves, ChannelMemberLeave{
- UserId: cmh.UserId,
- IsBot: cmh.IsBot,
- Email: cmh.UserEmail,
- Username: cmh.Username,
- Datetime: *cmh.LeaveTime,
- })
- }
- }
-
- for _, member := range postAuthors {
- if alreadyJoined[member.UserId] {
- continue
- }
-
- joins = append(joins, ChannelMemberJoin{
- UserId: member.UserId,
- IsBot: member.IsBot,
- Email: member.Email,
- Username: member.Username,
- Datetime: startTime,
- })
- }
- return joins, leaves
-}
-
-func ChannelTypeDisplayName(channelType model.ChannelType) string {
- return map[model.ChannelType]string{
- model.ChannelTypeOpen: "public",
- model.ChannelTypePrivate: "private",
- model.ChannelTypeDirect: "direct",
- model.ChannelTypeGroup: "group",
- }[channelType]
-}
diff --git a/server/enterprise/message_export/common_export/common_export_test.go b/server/enterprise/message_export/common_export/common_export_test.go
deleted file mode 100644
index 0e6717618e4..00000000000
--- a/server/enterprise/message_export/common_export/common_export_test.go
+++ /dev/null
@@ -1,237 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package common_export
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/mattermost/mattermost/server/public/model"
-)
-
-func TestUpdateMetadata(t *testing.T) {
- metadata := Metadata{
- Channels: map[string]*MetadataChannel{},
- MessagesCount: 0,
- AttachmentsCount: 0,
- StartTime: 0,
- EndTime: 0,
- }
-
- testString := "test"
- chanTypeDirect := model.ChannelTypeDirect
- createdAt := int64(12345)
-
- post := model.MessageExport{
- TeamId: &testString,
- TeamName: &testString,
- TeamDisplayName: &testString,
-
- ChannelId: &testString,
- ChannelName: &testString,
- ChannelDisplayName: &testString,
- ChannelType: &chanTypeDirect,
-
- UserId: &testString,
- UserEmail: &testString,
- Username: &testString,
-
- PostId: &testString,
- PostCreateAt: &createdAt,
- PostMessage: &testString,
- PostType: &testString,
- PostOriginalId: &testString,
- PostFileIds: []string{},
- }
- metadata.Update(&post, 2)
-
- assert.Len(t, metadata.Channels, 1)
- assert.Equal(t, 1, metadata.Channels["test"].MessagesCount)
- assert.Equal(t, 2, metadata.Channels["test"].AttachmentsCount)
- assert.Equal(t, 1, metadata.MessagesCount)
- assert.Equal(t, 2, metadata.AttachmentsCount)
-
- metadata.Update(&post, 2)
-
- assert.Len(t, metadata.Channels, 1)
- assert.Equal(t, 2, metadata.Channels["test"].MessagesCount)
- assert.Equal(t, 4, metadata.Channels["test"].AttachmentsCount)
- assert.Equal(t, 2, metadata.MessagesCount)
- assert.Equal(t, 4, metadata.AttachmentsCount)
-
- testString2 := "test2"
- post.ChannelId = &testString2
-
- metadata.Update(&post, 2)
-
- assert.Len(t, metadata.Channels, 2)
- assert.Equal(t, 2, metadata.Channels["test"].MessagesCount)
- assert.Equal(t, 4, metadata.Channels["test"].AttachmentsCount)
- assert.Equal(t, 1, metadata.Channels["test2"].MessagesCount)
- assert.Equal(t, 2, metadata.Channels["test2"].AttachmentsCount)
- assert.Equal(t, 3, metadata.MessagesCount)
- assert.Equal(t, 6, metadata.AttachmentsCount)
-}
-
-func TestGetJoinsAndLeavesForChannel(t *testing.T) {
- channel := MetadataChannel{
- StartTime: 100,
- EndTime: 200,
- ChannelId: "good-request-1",
- TeamId: model.NewPointer("test"),
- TeamName: model.NewPointer("test"),
- TeamDisplayName: model.NewPointer("test"),
- ChannelName: "test",
- ChannelDisplayName: "test",
- ChannelType: "O",
- }
-
- tt := []struct {
- name string
- channel MetadataChannel
- membersHistory []*model.ChannelMemberHistoryResult
- usersInPosts map[string]ChannelMember
- expectedJoins int
- expectedLeaves int
- }{
- {
- name: "no-joins-no-leaves",
- channel: channel,
- membersHistory: nil,
- usersInPosts: nil,
- expectedJoins: 0,
- expectedLeaves: 0,
- },
- {
- name: "joins-and-leaves-outside-the-range",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 1, LeaveTime: model.NewPointer(int64(10)), UserId: "test", UserEmail: "test", Username: "test"},
- {JoinTime: 250, LeaveTime: model.NewPointer(int64(260)), UserId: "test", UserEmail: "test", Username: "test"},
- {JoinTime: 300, UserId: "test", UserEmail: "test", Username: "test"},
- },
- usersInPosts: nil,
- expectedJoins: 0,
- expectedLeaves: 0,
- },
- {
- name: "join-and-leave-during-the-range",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 100, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
- },
- usersInPosts: nil,
- expectedJoins: 1,
- expectedLeaves: 1,
- },
- {
- name: "join-during-and-leave-after-the-range",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 150, LeaveTime: model.NewPointer(int64(300)), UserId: "test", UserEmail: "test", Username: "test"},
- },
- usersInPosts: nil,
- expectedJoins: 1,
- expectedLeaves: 0,
- },
- {
- name: "join-before-and-leave-during-the-range",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 99, LeaveTime: model.NewPointer(int64(150)), UserId: "test", UserEmail: "test", Username: "test"},
- },
- usersInPosts: nil,
- expectedJoins: 1,
- expectedLeaves: 1,
- },
- {
- name: "join-before-and-leave-after-the-range",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 99, LeaveTime: model.NewPointer(int64(350)), UserId: "test", UserEmail: "test", Username: "test"},
- },
- usersInPosts: nil,
- expectedJoins: 1,
- expectedLeaves: 0,
- },
- {
- name: "implicit-joins",
- channel: channel,
- membersHistory: nil,
- usersInPosts: map[string]ChannelMember{
- "test1": {UserId: "test1", Email: "test1", Username: "test1"},
- "test2": {UserId: "test2", Email: "test2", Username: "test2"},
- },
- expectedJoins: 2,
- expectedLeaves: 0,
- },
- {
- name: "implicit-joins-with-explicit-joins",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test3", UserEmail: "test3", Username: "test3"},
- },
- usersInPosts: map[string]ChannelMember{
- "test1": {UserId: "test1", Email: "test1", Username: "test1"},
- "test2": {UserId: "test2", Email: "test2", Username: "test2"},
- },
- expectedJoins: 3,
- expectedLeaves: 2,
- },
- {
- name: "join-leave-and-join-again",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1"},
- {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1"},
- },
- usersInPosts: nil,
- expectedJoins: 2,
- expectedLeaves: 2,
- },
- {
- name: "deactivated-members-dont-show",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
- {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
- },
- usersInPosts: nil,
- expectedJoins: 0,
- expectedLeaves: 0,
- },
- {
- name: "deactivated-members-show-if-deleted-after-latest-export",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
- {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
- },
- usersInPosts: nil,
- expectedJoins: 2,
- expectedLeaves: 2,
- },
- {
- name: "deactivated-members-show-and-dont-show",
- channel: channel,
- membersHistory: []*model.ChannelMemberHistoryResult{
- {JoinTime: 130, LeaveTime: model.NewPointer(int64(150)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 50},
- {JoinTime: 160, LeaveTime: model.NewPointer(int64(180)), UserId: "test1", UserEmail: "test1", Username: "test1", UserDeleteAt: 150},
- },
- usersInPosts: nil,
- expectedJoins: 1,
- expectedLeaves: 1,
- },
- }
-
- for _, tc := range tt {
- t.Run(tc.name, func(t *testing.T) {
- joins, leaves := GetJoinsAndLeavesForChannel(tc.channel.StartTime, tc.channel.EndTime, tc.membersHistory, tc.usersInPosts)
- assert.Len(t, joins, tc.expectedJoins)
- assert.Len(t, leaves, tc.expectedLeaves)
- })
- }
-}
diff --git a/server/enterprise/message_export/csv_export/csv_export.go b/server/enterprise/message_export/csv_export/csv_export.go
deleted file mode 100644
index bc20b313139..00000000000
--- a/server/enterprise/message_export/csv_export/csv_export.go
+++ /dev/null
@@ -1,370 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package csv_export
-
-import (
- "archive/zip"
- "encoding/csv"
- "encoding/json"
- "fmt"
- "io"
- "os"
- "path"
- "slices"
- "strconv"
- "strings"
-
- "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-const (
- EnterPostType = "enter"
- LeavePostType = "leave"
- PreviouslyJoinedPostType = "previously-joined"
- CSVWarningFilename = "warning.txt"
-)
-
-type Row struct {
- CreateAt int64
- UpdateAt int64
- UpdateType shared.PostUpdatedType
- TeamId string
- TeamName string
- TeamDisplayName string
- ChannelId string
- ChannelName string
- ChannelDisplayName string
- ChannelType string
- UserId string
- UserEmail string
- Username string
- PostId string
- EditedByPostId string
- RepliedToPostId string
- PostMessage string
- PostType string
- UserType string
- PreviewsPostId string
-}
-
-func CsvExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
- // Build the channel exports for the channels that had post or user join/leave activity this batch.
- exportData, err := shared.GetGenericExportData(p)
- results := exportData.Results
- if err != nil {
- return results, err
- }
-
- totalRows := results.CreatedPosts + results.EditedOrigMsgPosts + results.DeletedPosts + results.EditedNewMsgPosts +
- results.UpdatedPosts + results.UploadedFiles + results.DeletedFiles + results.Joins
- rows := make([]Row, 0, totalRows)
- for _, channel := range exportData.Exports {
- rows = append(rows, getJoinLeavePosts(channel)...)
- for _, p := range channel.Posts {
- rows = append(rows, postToRow(p, "message", p.PostCreateAt, p.Message))
- }
-
- for _, u := range channel.UploadStarts {
- rows = append(rows, attachmentToRow(shared.UploadStartToExportEntry(u)))
- }
- for _, d := range channel.DeletedFiles {
- rows = append(rows, attachmentToRow(d))
- }
- }
-
- // We need to sort all the elements by (CreateAt, PostId) because they were added by type and by channel above.
- slices.SortStableFunc(rows, func(a, b Row) int {
- if a.CreateAt == b.CreateAt {
- return strings.Compare(a.PostId, b.PostId)
- }
- return int(a.CreateAt - b.CreateAt)
- })
-
- // We've got the data, now its write time:
-
- // Write this batch to a tmp zip, then copy the zip to the export directory.
- // Using a 2M buffer because the file backend may be s3 and this optimizes speed and
- // memory usage, see: https://github.com/mattermost/mattermost/pull/26629
- buf := make([]byte, 1024*1024*2)
- temp, err := os.CreateTemp("", "compliance-export-batch-*.zip")
- if err != nil {
- return results, fmt.Errorf("unable to create temporary CSV export file: %w", err)
- }
- defer file.DeleteTemp(rctx.Logger(), temp)
-
- zipFile := zip.NewWriter(temp)
- csvFile, err := zipFile.Create("posts.csv")
- if err != nil {
- return results, fmt.Errorf("unable to create the zip export file: %w", err)
- }
- csvWriter := csv.NewWriter(csvFile)
- err = csvWriter.Write([]string{
- "Post Creation Time",
- "Post Update Time",
- "Post Update Type",
- "Team Id",
- "Team Name",
- "Team Display Name",
- "Channel Id",
- "Channel Name",
- "Channel Display Name",
- "Channel Type",
- "User Id",
- "User Email",
- "Username",
- "Post Id",
- "Edited By Post Id",
- "Replied to Post Id",
- "Post Message",
- "Post Type",
- "User Type",
- "Previews Post Id",
- })
-
- if err != nil {
- return results, fmt.Errorf("unable to add header to the CSV export: %w", err)
- }
-
- for _, row := range rows {
- if err = csvWriter.Write(rowToStringSlice(row)); err != nil {
- return results, fmt.Errorf("unable to export a row: %w", err)
- }
- }
-
- csvWriter.Flush()
-
- var missingFiles []string
- for _, post := range p.Posts {
- var attachments []*model.FileInfo
- attachments, err = shared.GetPostAttachments(p.Db, post)
- if err != nil {
- return results, err
- }
-
- for _, attachment := range attachments {
- var r io.ReadCloser
- r, err = p.FileAttachmentBackend.Reader(attachment.Path)
- if err != nil {
- missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringBackendRead+" - Post: "+*post.PostId+" - "+attachment.Path)
- rctx.Logger().Warn(shared.MissingFileMessageDuringBackendRead,
- mlog.String("post_id", *post.PostId),
- mlog.String("filename", attachment.Path),
- mlog.Err(err),
- )
- continue
- }
-
- // Probably don't need to be this careful (see actiance_export.go), but may as well be consistent.
- if err = func() error {
- defer r.Close()
- var attachmentDst io.Writer
- attachmentDst, err = zipFile.Create(path.Join("files", *post.PostId, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path))))
- if err != nil {
- return err
- }
-
- _, err = io.CopyBuffer(attachmentDst, r, buf)
- if err != nil {
- return err
- }
-
- return nil
- }(); err != nil {
- // s3 only errors _here_ if the object key wasn't found. So to handle that: if there is a read
- // error (even for local), let's add a warning instead of failing the export.
- // Failing the export would fail the entire export run, and every future run would also fail on
- // this non-existent file -- not good.
- missingFiles = append(missingFiles, "Warning:"+shared.MissingFileMessageDuringCopy+" - Post: "+*post.PostId+" - "+attachment.Path)
- rctx.Logger().Warn(shared.MissingFileMessageDuringCopy,
- mlog.String("post_id", *post.PostId),
- mlog.String("filename", attachment.Path),
- mlog.Err(err),
- )
- }
- }
- }
-
- results.NumWarnings = len(missingFiles)
- if results.NumWarnings > 0 {
- metadataFile, _ := zipFile.Create(CSVWarningFilename)
- for _, value := range missingFiles {
- _, err = metadataFile.Write([]byte(value + "\n"))
- if err != nil {
- return results, fmt.Errorf("unable to create the warning file: %w", err)
- }
- }
- }
-
- metadataFile, err := zipFile.Create("metadata.json")
- if err != nil {
- return results, fmt.Errorf("unable to create the zip file: %w", err)
- }
- data, err := json.MarshalIndent(exportData.Metadata, "", " ")
- if err != nil {
- return results, fmt.Errorf("unable to convert metadata to json: %w", err)
- }
- _, err = metadataFile.Write(data)
- if err != nil {
- return results, fmt.Errorf("unable to add metadata file to the zip file: %w", err)
- }
- err = zipFile.Close()
- if err != nil {
- return results, fmt.Errorf("unable to close the zip file: %w", err)
- }
-
- _, err = temp.Seek(0, 0)
- if err != nil {
- return results, fmt.Errorf("unable to seek to start of export file: %w", err)
- }
-
- // Try to write the file without a timeout due to the potential size of the file.
- _, err = filestore.TryWriteFileContext(rctx.Context(), p.ExportBackend, temp, p.BatchPath)
- if err != nil {
- return results, fmt.Errorf("unable to write the csv file: %w", err)
- }
- return results, nil
-}
-
-func getJoinLeavePosts(channel shared.ChannelExport) []Row {
- var joinLeavePosts []Row
-
- for _, join := range channel.JoinEvents {
- enterMessage := fmt.Sprintf("User %s (%s) joined the channel", join.Username, join.UserEmail)
- enterPostType := EnterPostType
- if join.JoinTime <= channel.StartTime {
- enterPostType = PreviouslyJoinedPostType
- enterMessage = fmt.Sprintf("User %s (%s) was already in the channel", join.Username, join.UserEmail)
- }
- joinLeavePosts = append(
- joinLeavePosts,
- postToRow(shared.PostExport{
- MessageExport: model.MessageExport{
- TeamId: &channel.TeamId,
- TeamName: &channel.TeamName,
- TeamDisplayName: &channel.TeamDisplayName,
- ChannelId: &channel.ChannelId,
- ChannelName: &channel.ChannelName,
- ChannelDisplayName: &channel.DisplayName,
- ChannelType: &channel.ChannelType,
- UserId: &join.UserId,
- UserEmail: &join.UserEmail,
- Username: &join.Username,
- IsBot: join.UserType == shared.Bot,
- PostId: model.NewPointer(""),
- PostCreateAt: &join.JoinTime,
- PostMessage: &enterMessage,
- PostType: &enterPostType,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: []string{},
- },
- }, enterPostType, &join.JoinTime, enterMessage),
- )
- }
- for _, leave := range channel.LeaveEvents {
- if leave.ClosedOut {
- // csv does not record closed-out leaves; see export_data.go for further explanation.
- continue
- }
- leaveMessage := fmt.Sprintf("User %s (%s) left the channel", leave.Username, leave.UserEmail)
- leavePostType := LeavePostType
-
- joinLeavePosts = append(
- joinLeavePosts,
- postToRow(shared.PostExport{
- MessageExport: model.MessageExport{
- TeamId: &channel.TeamId,
- TeamName: &channel.TeamName,
- TeamDisplayName: &channel.TeamDisplayName,
- ChannelId: &channel.ChannelId,
- ChannelName: &channel.ChannelName,
- ChannelDisplayName: &channel.DisplayName,
- ChannelType: &channel.ChannelType,
- UserId: &leave.UserId,
- UserEmail: &leave.UserEmail,
- Username: &leave.Username,
- IsBot: leave.UserType == shared.Bot,
- PostId: model.NewPointer(""),
- PostCreateAt: &leave.LeaveTime,
- PostMessage: &leaveMessage,
- PostType: &leavePostType,
- PostOriginalId: model.NewPointer(""),
- PostFileIds: []string{},
- },
- }, leavePostType, &leave.LeaveTime, leaveMessage),
- )
- }
-
- return joinLeavePosts
-}
-
-func postToRow(p shared.PostExport, postType string, createTime *int64, message string) Row {
- userType := "user"
- if p.IsBot {
- userType = "bot"
- }
- return Row{
- CreateAt: model.SafeDereference(createTime),
- UpdateAt: model.SafeDereference(p.PostUpdateAt),
- UpdateType: p.UpdatedType,
- TeamId: model.SafeDereference(p.TeamId),
- TeamName: model.SafeDereference(p.TeamName),
- TeamDisplayName: model.SafeDereference(p.TeamDisplayName),
- ChannelId: model.SafeDereference(p.ChannelId),
- ChannelName: model.SafeDereference(p.ChannelName),
- ChannelDisplayName: model.SafeDereference(p.ChannelDisplayName),
- ChannelType: shared.ChannelTypeDisplayName(model.SafeDereference(p.ChannelType)),
- UserId: model.SafeDereference(p.UserId),
- UserEmail: model.SafeDereference(p.UserEmail),
- Username: model.SafeDereference(p.Username),
- PostId: model.SafeDereference(p.PostId),
- EditedByPostId: p.EditedNewMsgId,
- RepliedToPostId: model.SafeDereference(p.PostRootId),
- PostMessage: message,
- PostType: postType,
- UserType: userType,
- PreviewsPostId: p.PreviewID(),
- }
-}
-
-func rowToStringSlice(r Row) []string {
- return []string{
- strconv.FormatInt(r.CreateAt, 10),
- strconv.FormatInt(r.UpdateAt, 10),
- string(r.UpdateType),
- r.TeamId,
- r.TeamName,
- r.TeamDisplayName,
- r.ChannelId,
- r.ChannelName,
- r.ChannelDisplayName,
- r.ChannelType,
- r.UserId,
- r.UserEmail,
- r.Username,
- r.PostId,
- r.EditedByPostId,
- r.RepliedToPostId,
- r.PostMessage,
- r.PostType,
- r.UserType,
- r.PreviewsPostId,
- }
-}
-
-func attachmentToRow(post shared.PostExport) Row {
- message := strings.TrimSpace(fmt.Sprintf("%s (files/%s/%s-%s)", post.FileInfo.Name, *post.PostId, post.FileInfo.Id, path.Base(post.FileInfo.Path)))
- postType := "attachment"
- if post.UpdatedType == shared.FileDeleted {
- postType = "deleted attachment"
- post.PostUpdateAt = model.NewPointer(post.FileInfo.DeleteAt)
- }
-
- return postToRow(post, postType, post.PostCreateAt, message)
-}
diff --git a/server/enterprise/message_export/csv_export/csv_export_test.go b/server/enterprise/message_export/csv_export/csv_export_test.go
deleted file mode 100644
index 0136544044b..00000000000
--- a/server/enterprise/message_export/csv_export/csv_export_test.go
+++ /dev/null
@@ -1,912 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package csv_export
-
-import (
- "archive/zip"
- "bytes"
- "fmt"
- "io"
- "os"
- "path"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-func TestPostToRow(t *testing.T) {
- chanTypeDirect := model.ChannelTypeDirect
- // these two posts were made in the same channel
- post := model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- }
-
- post_without_team := model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- }
-
- post_with_other_type := model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostType: model.NewPointer("other"),
- }
-
- post_with_other_type_bot := model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostType: model.NewPointer("other"),
- IsBot: true,
- }
-
- post_with_permalink_preview := model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostProps: model.NewPointer(`{"previewed_post":"n4w39mc1ff8y5fite4b8hacy1w"}`),
- }
- torowtests := []struct {
- name string
- in model.MessageExport
- out Row
- }{
- {
- "simple row",
- post,
- Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", ""},
- },
- {
- "without team data",
- post_without_team,
- Row{1, 1, "", "", "", "", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", ""},
- },
- {
- "with special post type",
- post_with_other_type,
- Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "other", "user", ""},
- },
- {
- "with special post type from bot",
- post_with_other_type_bot,
- Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "other", "bot", ""},
- },
- {
- "with permalink preview",
- post_with_permalink_preview,
- Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "message", "message", "user", "n4w39mc1ff8y5fite4b8hacy1w"},
- },
- }
-
- for _, tt := range torowtests {
- t.Run(tt.name, func(t *testing.T) {
- postType := model.SafeDereference(tt.in.PostType)
- if postType == "" {
- postType = "message"
- }
- in := shared.PostExport{
- MessageExport: tt.in,
- }
- assert.Equal(t, tt.out, postToRow(in, postType, tt.in.PostCreateAt, *tt.in.PostMessage))
- })
- }
-}
-
-func TestAttachmentToRow(t *testing.T) {
- chanTypeDirect := model.ChannelTypeDirect
- post := shared.PostExport{
- MessageExport: model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- },
- FileInfo: &model.FileInfo{
- Name: "test1",
- Id: "12345",
- Path: "filename.txt",
- },
- }
-
- postDeleted := shared.PostExport{
- MessageExport: model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(10)),
- PostDeleteAt: model.NewPointer(int64(10)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostProps: model.NewPointer(`{"deleteBy":"user-id"}`),
- },
- UpdatedType: shared.FileDeleted,
- FileInfo: &model.FileInfo{
- Name: "test2",
- Id: "12346",
- Path: "filename.txt",
- DeleteAt: 10,
- },
- }
-
- toRowTests := []struct {
- name string
- post shared.PostExport
- out Row
- }{
- {
- "simple attachment",
- post,
- Row{1, 1, "", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "test1 (files/post-id/12345-filename.txt)", "attachment", "user", ""},
- },
- {
- "simple deleted attachment",
- postDeleted,
- Row{1, 10, "FileDeleted", "team-id", "team-name", "team-display-name", "channel-id", "channel-name", "channel-display-name", "direct", "user-id", "test@test.com", "username", "post-id", "", "post-root-id", "test2 (files/post-id/12346-filename.txt)", "deleted attachment", "user", ""},
- },
- }
-
- for _, tt := range toRowTests {
- t.Run(tt.name, func(t *testing.T) {
- assert.Equal(t, tt.out, attachmentToRow(tt.post))
- })
- }
-}
-
-func TestGetPostAttachments(t *testing.T) {
- chanTypeDirect := model.ChannelTypeDirect
- post := &model.MessageExport{
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- }
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- files, err := shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
- assert.NoError(t, err)
- assert.Empty(t, files)
-
- post.PostFileIds = []string{"1", "2"}
-
- mockStore.FileInfoStore.On("GetForPost", *post.PostId, true, true, false).Return([]*model.FileInfo{{Name: "test"}, {Name: "test2"}}, nil)
-
- files, err = shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
- assert.NoError(t, err)
- assert.Len(t, files, 2)
-
- post.PostId = model.NewPointer("post-id-2")
-
- mockStore.FileInfoStore.On("GetForPost", *post.PostId, true, true, false).Return(nil, model.NewAppError("Test", "test", nil, "", 400))
-
- files, err = shared.GetPostAttachments(shared.NewMessageExportStore(mockStore), post)
- assert.Error(t, err)
- assert.Nil(t, files)
-}
-
-func TestCsvExport(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- runTestCsvExportDedicatedExportFilestore(t, fileBackend, fileBackend)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(exportTempDir)
- assert.NoError(t, err)
- })
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(attachmentTempDir)
- assert.NoError(t, err)
- })
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
-
- runTestCsvExportDedicatedExportFilestore(t, exportBackend, attachmentBackend)
- })
-}
-
-func runTestCsvExportDedicatedExportFilestore(t *testing.T, exportBackend filestore.FileBackend, attachmentBackend filestore.FileBackend) {
- rctx := request.TestContext(t)
-
- header := "Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id\n"
-
- chanTypeDirect := model.ChannelTypeDirect
- csvExportTests := []struct {
- name string
- cmhs map[string][]*model.ChannelMemberHistoryResult
- metadata map[string]*shared.MetadataChannel
- startTime int64
- endTime int64
- posts []*model.MessageExport
- attachments map[string][]*model.FileInfo
- expectedPosts string
- expectedMetadata string
- expectedFiles int
- }{
- {
- name: "empty",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{},
- posts: []*model.MessageExport{},
- attachments: map[string][]*model.FileInfo{},
- expectedPosts: header,
- expectedMetadata: "{\n \"Channels\": null,\n \"MessagesCount\": 0,\n \"AttachmentsCount\": 0,\n \"StartTime\": 0,\n \"EndTime\": 0\n}",
- expectedFiles: 2,
- },
- {
- name: "posts",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer(`{"previewed_post":"o4w39mc1ff8y5fite4b8hacy1x"}`),
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedPosts: strings.Join([]string{
- header,
- "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
- "8,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
- "80,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) left the channel,leave,user,\n",
- "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,post-root-id,message,message,user,\n",
- "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,post-root-id,message,message,user,o4w39mc1ff8y5fite4b8hacy1x\n",
- }, ""),
- expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0\n }\n },\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
- expectedFiles: 2,
- },
- {
- name: "deleted post",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100)),
- PostUpdateAt: model.NewPointer(int64(101)),
- PostDeleteAt: model.NewPointer(int64(101)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedPosts: strings.Join([]string{
- header,
- "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
- "100,101,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,message,message,user,\n",
- "100,101,Deleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id,,,delete message,message,user,\n",
- }, ""),
- expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0\n }\n },\n \"MessagesCount\": 3,\n \"AttachmentsCount\": 0,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
- expectedFiles: 2,
- },
-
- {
- name: "posts with deleted attachment and deleted post, and at different time from non-deleted original post",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message 1"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-3"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(2)),
- PostUpdateAt: model.NewPointer(int64(3)),
- PostDeleteAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("message 3"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test2"},
- PostProps: model.NewPointer("{\"deleteBy\":\"user-id\"}"),
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer(""),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100)),
- PostCreateAt: model.NewPointer(int64(100)),
- PostMessage: model.NewPointer("message 2"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- // NOTE: this post will be deleted, but the post-id-2 is not deleted.
- PostFileIds: []string{"test3"},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1",
- Id: "test1",
- Path: "test1",
- },
- },
- "post-id-3": {
- {
- Name: "test2",
- Id: "test2",
- Path: "test2",
- DeleteAt: 3,
- },
- },
- "post-id-2": {
- {
- Name: "test3",
- Id: "test3",
- Path: "test3",
- DeleteAt: 102,
- },
- },
- },
- expectedPosts: strings.Join([]string{
- header,
- "0,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test,test,test,,,,User test (test) was already in the channel,previously-joined,user,\n",
- "1,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,,,,User username (test@test.com) was already in the channel,previously-joined,user,\n",
- "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,,,message 1,message,user,\n",
- "1,1,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-1,,,test1 (files/post-id-1/test1-test1),attachment,user,\n",
- "2,3,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,message 3,message,user,\n",
- "2,3,Deleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,delete message 3,message,user,\n",
- "2,3,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,test2 (files/post-id-3/test2-test2),attachment,user,\n",
- "2,3,FileDeleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-3,,,test2 (files/post-id-3/test2-test2),deleted attachment,user,\n",
- "8,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) joined the channel,enter,user,\n",
- "80,0,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,test2,test2,test2,,,,User test2 (test2) left the channel,leave,user,\n",
- "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,message 2,message,user,\n",
- "100,100,,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,test3 (files/post-id-2/test3-test3),attachment,user,\n",
- "100,102,FileDeleted,team-id,team-name,team-display-name,channel-id,channel-name,channel-display-name,direct,user-id,test@test.com,username,post-id-2,,post-id-1,test3 (files/post-id-2/test3-test3),deleted attachment,user,\n",
- }, ""),
- expectedMetadata: "{\n \"Channels\": {\n \"channel-id\": {\n \"TeamId\": \"team-id\",\n \"TeamName\": \"team-name\",\n \"TeamDisplayName\": \"team-display-name\",\n \"ChannelId\": \"channel-id\",\n \"ChannelName\": \"channel-name\",\n \"ChannelDisplayName\": \"channel-display-name\",\n \"ChannelType\": \"D\",\n \"RoomId\": \"direct - channel-id\",\n \"StartTime\": 1,\n \"EndTime\": 100,\n \"MessagesCount\": 4,\n \"AttachmentsCount\": 3\n }\n },\n \"MessagesCount\": 4,\n \"AttachmentsCount\": 3,\n \"StartTime\": 1,\n \"EndTime\": 100\n}",
- expectedFiles: 5,
- },
- }
-
- for _, tt := range csvExportTests {
- t.Run(tt.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tt.attachments) > 0 {
- for postId, attachments := range tt.attachments {
- call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false)
- call.Run(func(args mock.Arguments) {
- call.Return(tt.attachments[args.Get(0).(string)], nil)
- })
- _, err := attachmentBackend.WriteFile(bytes.NewReader([]byte{}), attachments[0].Path)
- require.NoError(t, err)
- t.Cleanup(func() {
- err = attachmentBackend.RemoveFile(attachments[0].Path)
- require.NoError(t, err)
- })
- }
- }
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001-csv.zip")
- results, err := CsvExport(rctx, shared.ExportParams{
- ChannelMetadata: tt.metadata,
- Posts: tt.posts,
- ChannelMemberHistories: tt.cmhs,
- BatchPath: exportFileName,
- BatchStartTime: tt.startTime,
- BatchEndTime: tt.endTime,
- Config: nil,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- })
- assert.NoError(t, err)
- assert.Equal(t, 0, results.NumWarnings)
-
- zipBytes, err := exportBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
- t.Cleanup(func() {
- err = exportBackend.RemoveFile(exportFileName)
- require.NoError(t, err)
- })
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- assert.Len(t, zipReader.File, tt.expectedFiles)
-
- postsFile, err := zipReader.File[0].Open()
- require.NoError(t, err)
- defer postsFile.Close()
- postsFileData, err := io.ReadAll(postsFile)
- assert.NoError(t, err)
- postsFile.Close()
-
- metadataFile, err := zipReader.File[len(zipReader.File)-1].Open()
- require.NoError(t, err)
- defer metadataFile.Close()
- metadataFileData, err := io.ReadAll(metadataFile)
- require.NoError(t, err)
- err = metadataFile.Close()
- require.NoError(t, err)
-
- assert.Equal(t, tt.expectedPosts, string(postsFileData))
- assert.Equal(t, tt.expectedMetadata, string(metadataFileData))
- })
- }
-}
-
-func TestWriteExportWarnings(t *testing.T) {
- rctx := request.TestContext(t)
-
- chanTypeDirect := model.ChannelTypeDirect
- cmhs := map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test", UserEmail: "test", Username: "test", LeaveTime: model.NewPointer(int64(400)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2", Username: "test2", LeaveTime: model.NewPointer(int64(80)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3", Username: "test3",
- },
- },
- }
- metadata := map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- }
-
- posts := []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"post-id-1"},
- },
- {
- PostId: model.NewPointer("post-id-3"),
- PostOriginalId: model.NewPointer(""),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(2)),
- PostUpdateAt: model.NewPointer(int64(3)),
- PostDeleteAt: model.NewPointer(int64(3)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@test.com"),
- UserId: model.NewPointer("user-id"),
- Username: model.NewPointer("username"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"post-id-3"},
- },
- }
-
- attachments := map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1",
- Id: "test1",
- Path: "test1",
- },
- },
- "post-id-3": {
- {
- Name: "test2",
- Id: "test2",
- Path: "test2",
- },
- },
- }
-
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(tempDir)
- assert.NoError(t, err)
- })
-
- config := filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: tempDir,
- }
-
- fileBackend, err := filestore.NewFileBackend(config)
- assert.NoError(t, err)
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- for postId := range attachments {
- call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false).Times(2)
- call.Run(func(args mock.Arguments) {
- call.Return(attachments[args.Get(0).(string)], nil)
- })
- }
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001-csv.zip")
- results, err := CsvExport(rctx, shared.ExportParams{
- ExportType: "",
- ChannelMetadata: metadata,
- Posts: posts,
- ChannelMemberHistories: cmhs,
- BatchPath: exportFileName,
- BatchStartTime: 1,
- BatchEndTime: 100,
- Config: nil,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: fileBackend,
- ExportBackend: fileBackend,
- })
- assert.NoError(t, err)
- assert.Equal(t, 2, results.NumWarnings)
-
- zipBytes, err := fileBackend.ReadFile(exportFileName)
- assert.NoError(t, err)
-
- t.Cleanup(func() {
- err = fileBackend.RemoveFile(exportFileName)
- assert.NoError(t, err)
- })
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- assert.NoError(t, err)
- assert.Len(t, zipReader.File, 3)
- warningsTxt, err := zipReader.Open("warning.txt")
- assert.NoError(t, err)
- data, err := io.ReadAll(warningsTxt)
- assert.NoError(t, err)
- warnings := string(data)
-
- expectedWarnings := fmt.Sprintf("Warning:%[1]s - Post: post-id-1 - test1\nWarning:%[1]s - Post: post-id-3 - test2\n",
- shared.MissingFileMessageDuringBackendRead)
-
- assert.Equal(t, expectedWarnings, warnings)
-}
diff --git a/server/enterprise/message_export/global_relay_export/deliver.go b/server/enterprise/message_export/global_relay_export/deliver.go
deleted file mode 100644
index 329dc0a040c..00000000000
--- a/server/enterprise/message_export/global_relay_export/deliver.go
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "archive/zip"
- "context"
- "fmt"
- "io"
- "net/mail"
- "net/smtp"
- "os"
- "time"
-
- "github.com/mattermost/mattermost/server/public/model"
-)
-
-func Deliver(export *os.File, config *model.Config) error {
- info, err := export.Stat()
- if err != nil {
- return fmt.Errorf("unable to get the information of the export temporary file: %w", err)
- }
- zipFile, err := zip.NewReader(export, info.Size())
- if err != nil {
- return fmt.Errorf("unable to open the export temporary file: %w", err)
- }
-
- to := *config.MessageExportSettings.GlobalRelaySettings.EmailAddress
- ctx := context.Background()
- ctx, cancel := context.WithTimeout(ctx, time.Duration(*config.EmailSettings.SMTPServerTimeout)*time.Second)
- defer cancel()
-
- conn, err := connectToSMTPServer(ctx, config)
- if err != nil {
- return fmt.Errorf("unable to connect to the smtp server: %w", err)
- }
- defer conn.Close()
-
- mailsCount := 0
- for _, mail := range zipFile.File {
- from, err := getFrom(mail)
- if err != nil {
- return err
- }
- if err := deliverEmail(conn, mail, from, to); err != nil {
- return err
- }
-
- mailsCount++
- if mailsCount == MaxEmailsPerConnection {
- mailsCount = 0
- conn.Close()
-
- var nErr error
- conn, nErr = connectToSMTPServer(context.Background(), config)
- if nErr != nil {
- return fmt.Errorf("unable to connect to the smtp server: %w", nErr)
- }
- }
- }
- return nil
-}
-
-func deliverEmail(c *smtp.Client, mailFile *zip.File, from string, to string) error {
- mailData, err := mailFile.Open()
- if err != nil {
- return fmt.Errorf("unable to get the an email from the temporary file: %w", err)
- }
- defer mailData.Close()
-
- err = c.Mail(from)
- if err != nil {
- return fmt.Errorf("unable to set the email From address: %w", err)
- }
-
- err = c.Rcpt(to)
- if err != nil {
- return fmt.Errorf("unable to set the email To address: %w", err)
- }
-
- w, err := c.Data()
- if err != nil {
- return fmt.Errorf("unable to write the email message: %w", err)
- }
-
- _, err = io.Copy(w, mailData)
- if err != nil {
- return fmt.Errorf("unable to set the email message: %w", err)
- }
- err = w.Close()
- if err != nil {
- return fmt.Errorf("unable to deliver the email to Global Relay: %w", err)
- }
- return nil
-}
-
-func getFrom(mailFile *zip.File) (string, error) {
- mailData, err := mailFile.Open()
- if err != nil {
- return "", fmt.Errorf("unable to get the an email from the temporary file: %w", err)
- }
- defer mailData.Close()
-
- message, err := mail.ReadMessage(mailData)
- if err != nil {
- return "", fmt.Errorf("unable to read the email information: %w", err)
- }
- return message.Header.Get("From"), nil
-}
diff --git a/server/enterprise/message_export/global_relay_export/deliver_test.go b/server/enterprise/message_export/global_relay_export/deliver_test.go
deleted file mode 100644
index bf1d369fb8d..00000000000
--- a/server/enterprise/message_export/global_relay_export/deliver_test.go
+++ /dev/null
@@ -1,140 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "archive/zip"
- "fmt"
- "io"
- "os"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- gomail "gopkg.in/mail.v2"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/v8/platform/shared/mail"
-)
-
-func TestDeliver(t *testing.T) {
- config := &model.Config{}
- config.SetDefaults()
- config.MessageExportSettings.GlobalRelaySettings.CustomerType = model.NewPointer("INBUCKET")
- config.MessageExportSettings.GlobalRelaySettings.EmailAddress = model.NewPointer("test-globalrelay-mailbox@test")
-
- t.Run("Testing invalid zip file", func(t *testing.T) {
- emptyFile, err := os.CreateTemp("", "export")
- require.NoError(t, err)
- defer emptyFile.Close()
- defer os.Remove(emptyFile.Name())
-
- err = Deliver(emptyFile, config)
- assert.Error(t, err)
- })
-
- t.Run("Testing empty zip file", func(t *testing.T) {
- emptyZipFile, err := os.CreateTemp("", "export")
- require.NoError(t, err)
- zipFile := zip.NewWriter(emptyZipFile)
- err = zipFile.Close()
- require.NoError(t, err)
- defer emptyZipFile.Close()
- defer os.Remove(emptyZipFile.Name())
-
- err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.NoError(t, err)
-
- err = Deliver(emptyZipFile, config)
- assert.NoError(t, err)
-
- _, err = mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.Error(t, err)
- })
-
- t.Run("Testing zip file with one email", func(t *testing.T) {
- headers := map[string][]string{
- "From": {"test@test.com"},
- "To": {*config.MessageExportSettings.GlobalRelaySettings.EmailAddress},
- "Subject": {encodeRFC2047Word("test")},
- "Content-Transfer-Encoding": {"8bit"},
- "Auto-Submitted": {"auto-generated"},
- "Precedence": {"bulk"},
- GlobalRelayMsgTypeHeader: {"Mattermost"},
- GlobalRelayChannelNameHeader: {encodeRFC2047Word("test")},
- GlobalRelayChannelIDHeader: {encodeRFC2047Word("test")},
- GlobalRelayChannelTypeHeader: {encodeRFC2047Word("test")},
- }
-
- m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
- m.SetHeaders(headers)
- m.SetBody("text/plain", "test")
-
- emptyZipFile, err := os.CreateTemp("", "export")
- require.NoError(t, err)
- zipFile := zip.NewWriter(emptyZipFile)
- file, err := zipFile.Create("test")
- require.NoError(t, err)
- _, err = m.WriteTo(file)
- require.NoError(t, err)
-
- err = zipFile.Close()
- require.NoError(t, err)
- defer emptyZipFile.Close()
- defer os.Remove(emptyZipFile.Name())
-
- err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.NoError(t, err)
-
- err = Deliver(emptyZipFile, config)
- assert.NoError(t, err)
-
- mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.NoError(t, err)
- require.Len(t, mailbox, 1)
- })
-
- t.Run("Testing zip file with 50 emails", func(t *testing.T) {
- headers := map[string][]string{
- "From": {"test@test.com"},
- "To": {*config.MessageExportSettings.GlobalRelaySettings.EmailAddress},
- "Subject": {encodeRFC2047Word("test")},
- "Content-Transfer-Encoding": {"8bit"},
- "Auto-Submitted": {"auto-generated"},
- "Precedence": {"bulk"},
- GlobalRelayMsgTypeHeader: {"Mattermost"},
- GlobalRelayChannelNameHeader: {encodeRFC2047Word("test")},
- GlobalRelayChannelIDHeader: {encodeRFC2047Word("test")},
- GlobalRelayChannelTypeHeader: {encodeRFC2047Word("test")},
- }
- m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
- m.SetHeaders(headers)
- m.SetBody("text/plain", "test")
-
- emptyZipFile, err := os.CreateTemp("", "export")
- require.NoError(t, err)
- zipFile := zip.NewWriter(emptyZipFile)
- for x := range 50 {
- var file io.Writer
- file, err = zipFile.Create(fmt.Sprintf("test-%d", x))
- require.NoError(t, err)
- _, err = m.WriteTo(file)
- require.NoError(t, err)
- }
- err = zipFile.Close()
- require.NoError(t, err)
- defer emptyZipFile.Close()
- defer os.Remove(emptyZipFile.Name())
-
- err = mail.DeleteMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.NoError(t, err)
-
- err = Deliver(emptyZipFile, config)
- assert.NoError(t, err)
-
- mailbox, err := mail.GetMailBox(*config.MessageExportSettings.GlobalRelaySettings.EmailAddress)
- require.NoError(t, err)
- require.Len(t, mailbox, 50)
- })
-}
diff --git a/server/enterprise/message_export/global_relay_export/global_relay_export.go b/server/enterprise/message_export/global_relay_export/global_relay_export.go
deleted file mode 100644
index 070696245b8..00000000000
--- a/server/enterprise/message_export/global_relay_export/global_relay_export.go
+++ /dev/null
@@ -1,415 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "archive/zip"
- "encoding/json"
- "fmt"
- "io"
- "mime"
- "os"
- "sort"
- "strings"
- "time"
-
- "github.com/jaytaylor/html2text"
- gomail "gopkg.in/mail.v2"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/enterprise/internal/file"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
- "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
-)
-
-const (
- GlobalRelayMsgTypeHeader = "X-GlobalRelay-MsgType"
- GlobalRelayChannelNameHeader = "X-Mattermost-ChannelName"
- GlobalRelayChannelIDHeader = "X-Mattermost-ChannelID"
- GlobalRelayChannelTypeHeader = "X-Mattermost-ChannelType"
- MaxEmailsPerConnection = 400
-)
-
-// MaxEmailBytes is a var because it needs to be set in tests. Otherwise it shouldn't be touched.
-var MaxEmailBytes int64 = 250 * 1024 * 1024 // 250MB
-
-type ChannelExport struct {
- TeamId string
- TeamName string
- TeamDisplayName string
- ChannelId string // the unique id of the channel
- ChannelName string // the name of the channel
- ChannelDisplayName string
- ChannelType model.ChannelType // the channel type
- StartTime int64 // utc timestamp (seconds), start of export period or create time of channel, whichever is greater. Example: 1366611728.
- EndTime int64 // utc timestamp (seconds), end of export period or delete time of channel, whichever is lesser. Example: 1366611728.
- Participants []ParticipantRow // summary information about the conversation participants
- Messages []Message // the messages that were sent during the conversation
- ExportedOn int64 // utc timestamp (seconds), when this export was generated
- numUserMessages map[string]int // key is user id, value is number of messages that they sent during this period
- uploadedFiles []*model.FileInfo // any files that were uploaded to the channel during the export period
- bytes int64
-}
-
-// a row in the summary table at the top of the export
-type ParticipantRow struct {
- shared.JoinExport
- MessagesSent int
-}
-
-type Message struct {
- Id string
- SentTime int64
- SenderId string
- SenderUsername string
- PostUsername string
- SenderUserType string
- SenderEmail string
- Message string
- PreviewsPost string
- UpdateAt int64
- UpdateType shared.PostUpdatedType
- EditedNewMsgId string
-}
-
-func GlobalRelayExport(rctx request.CTX, p shared.ExportParams) (shared.RunExportResults, error) {
- exportData, err := shared.GetGenericExportData(p)
- results := exportData.Results
- if err != nil {
- return results, err
- }
-
- var attachmentsRemovedPostIDs []string
- allExports := make(map[string][]*ChannelExport, len(exportData.Exports))
-
- // save a pointer to the joins for each channel, to be used later in getParticipants
- joinsByChannel := make(map[string][]shared.JoinExport, len(exportData.Exports))
-
- for _, channel := range exportData.Exports {
- for _, post := range channel.Posts {
- attachmentsRemoved := addToExports(rctx, allExports, channel, post)
- attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, attachmentsRemoved...)
- }
- if len(channel.Posts) == 0 {
- // channel has no posts, but it was exported anyway, so it must have joins and leaves.
- if _, present := allExports[channel.ChannelId]; !present {
- // we found a new channel
- allExports[channel.ChannelId] = []*ChannelExport{genericChannelToChannelExport(channel)}
- }
- }
- joinsByChannel[channel.ChannelId] = channel.JoinEvents
- }
-
- tmpFile, err := os.CreateTemp("", "")
- if err != nil {
- return results, fmt.Errorf("unable to open the temporary export file: %w", err)
- }
- defer file.DeleteTemp(rctx.Logger(), tmpFile)
- zipFile := zip.NewWriter(tmpFile)
-
- // export each channelExport (sometimes multiple channelExport per real channel)
- for _, channelExportList := range allExports {
- for batchId, channelExport := range channelExportList {
- // we need to make the participant list for each channelExport "batch" (multiple "batches"per real channel)
- // because each batch will have its own number of messages for that participant.
- channelExport.Participants = getParticipants(channelExport, joinsByChannel[channelExport.ChannelId])
- channelExport.ExportedOn = p.JobStartTime
-
- var channelExportFile io.Writer
- channelExportFile, err = zipFile.Create(fmt.Sprintf("%s - (%s) - %d.eml", channelExport.ChannelName, channelExport.ChannelId, batchId))
- if err != nil {
- return results, fmt.Errorf("unable to create the eml file: %w", err)
- }
-
- if results.NumWarnings, err = generateEmail(rctx, p.FileAttachmentBackend, channelExport, p.Templates, channelExportFile); err != nil {
- return results, err
- }
- }
- }
-
- err = zipFile.Close()
- if err != nil {
- return results, fmt.Errorf("unable to close the zip file using tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
- }
-
- _, err = tmpFile.Seek(0, 0)
- if err != nil {
- return results, fmt.Errorf("unable to re-read the Global Relay temporary export file using tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
- }
-
- if p.ExportType == model.ComplianceExportTypeGlobalrelayZip {
- // Try to disable the write timeout for the potentially big export file.
- _, err = filestore.TryWriteFileContext(rctx.Context(), p.ExportBackend, tmpFile, p.BatchPath)
- if err != nil {
- return results, fmt.Errorf("unable to write the global relay file, using tmpFile.Name: %v, batchPath: %v, err: %w", tmpFile.Name(), p.BatchPath, err)
- }
- } else {
- err = Deliver(tmpFile, p.Config)
- if err != nil {
- return results, fmt.Errorf("unable to deliver tmpFile.Name: %v, err: %w", tmpFile.Name(), err)
- }
- }
-
- if len(attachmentsRemovedPostIDs) > 0 {
- rctx.Logger().Warn("Global Relay Attachments Removed because they were too large to send to Global Relay",
- mlog.Int("number_of_attachments_removed", len(attachmentsRemovedPostIDs)))
- rctx.Logger().Warn("List of posts which had attachments removed",
- mlog.Array("post_ids", attachmentsRemovedPostIDs))
- }
-
- return results, nil
-}
-
-// addToExports adds the post to the allExports collection. allExports keeps a map of channelId->[]*ChannelExport.
-// If a channelId has an existing []*ChannelExport, it adds post to the last ChannelExport in that list.
-// If the last ChannelExport is too big, it starts a new ChannelExport and appends it to the list (a new "batch").
-func addToExports(rctx request.CTX, allExports map[string][]*ChannelExport, genericChannel shared.ChannelExport,
- post shared.PostExport) []string {
- var channelExport *ChannelExport
- var attachmentsRemovedPostIDs []string
- if channelExports, present := allExports[*post.ChannelId]; !present {
- // we found a new channel
- channelExport = genericChannelToChannelExport(genericChannel)
- allExports[*post.ChannelId] = []*ChannelExport{channelExport}
- } else {
- // we already know about this channel
- channelExport = channelExports[len(channelExports)-1]
- }
-
- msgBytes := int64(len(*post.PostMessage))
-
- // Create a new ChannelExport if it would be too many bytes to add the post.
- // NOTE: we are only exporting attachment starts.
- attachmentStarts := make([]*model.FileInfo, 0, len(post.AttachmentCreates))
- for _, start := range post.AttachmentCreates {
- attachmentStarts = append(attachmentStarts, start.FileInfo)
- }
- fileBytes := fileInfoListBytes(attachmentStarts)
- postBytes := fileBytes + msgBytes
- // NOTE: This is only a rough estimate -- we're not including the txt or html portion of the email...
- attachmentsAloneTooLargeToSend := fileBytes > MaxEmailBytes // Attachments must be removed from export, they're too big to send.
- if attachmentsAloneTooLargeToSend {
- postBytes -= fileBytes
- }
- postTooLargeForChannelBatch := channelExport.bytes+postBytes > MaxEmailBytes
-
- if attachmentsAloneTooLargeToSend {
- attachmentsRemovedPostIDs = append(attachmentsRemovedPostIDs, *post.PostId)
- }
-
- // new "batch"
- if postTooLargeForChannelBatch {
- channelExport = genericChannelToChannelExport(genericChannel)
- allExports[*post.ChannelId] = append(allExports[*post.ChannelId], channelExport)
- }
-
- addPostToChannelExport(rctx, channelExport, post)
-
- // if this post includes files, add them to the collection
- addAttachmentsToChannelExport(channelExport, post, post.AttachmentCreates, post.AttachmentDeletes, attachmentsAloneTooLargeToSend)
- channelExport.bytes += postBytes
- return attachmentsRemovedPostIDs
-}
-
-func genericChannelToChannelExport(genericChannel shared.ChannelExport) *ChannelExport {
- return &ChannelExport{
- TeamId: genericChannel.TeamId,
- TeamName: genericChannel.TeamName,
- TeamDisplayName: genericChannel.TeamDisplayName,
- ChannelId: genericChannel.ChannelId,
- ChannelName: genericChannel.ChannelName,
- ChannelDisplayName: genericChannel.DisplayName,
- ChannelType: genericChannel.ChannelType,
- StartTime: genericChannel.StartTime,
- EndTime: genericChannel.EndTime,
- // we can't preallocate sizes here because we don't know how many will be in this "batch"
- Participants: make([]ParticipantRow, 0),
- Messages: make([]Message, 0),
- ExportedOn: 0,
- numUserMessages: make(map[string]int),
- uploadedFiles: make([]*model.FileInfo, 0),
- bytes: 0,
- }
-}
-
-func getParticipants(channelExport *ChannelExport, joinEvents []shared.JoinExport) []ParticipantRow {
- participants := make([]ParticipantRow, 0, len(joinEvents))
- for _, j := range joinEvents {
- participants = append(participants, ParticipantRow{
- JoinExport: j,
- MessagesSent: channelExport.numUserMessages[j.UserId],
- })
- }
-
- sort.Slice(participants, func(i, j int) bool {
- return participants[i].Username < participants[j].Username
- })
- return participants
-}
-
-func generateEmail(rctx request.CTX, fileAttachmentBackend filestore.FileBackend, channelExport *ChannelExport, templates *templates.Container, w io.Writer) (int, error) {
- var warningCount int
- participantEmailAddresses := getParticipantEmails(channelExport)
-
- // GlobalRelay expects the email to come from the person that initiated the conversation.
- // our conversations aren't really initiated, so we just use the first person we find
- from := participantEmailAddresses[0]
-
- // it also expects the email to be addressed to the other participants in the conversation
- mimeTo := strings.Join(participantEmailAddresses, ",")
-
- htmlBody, err := channelExportToHTML(rctx, channelExport, templates)
- if err != nil {
- return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
- }
-
- subject := fmt.Sprintf("Mattermost Compliance Export: %s", channelExport.ChannelDisplayName)
- htmlMessage := "\r\n" + htmlBody + ""
-
- txtBody, err := html2text.FromString(htmlBody)
- if err != nil {
- rctx.Logger().Warn("Error transforming html to plain text for GlobalRelay email", mlog.Err(err))
- txtBody = ""
- }
-
- headers := map[string][]string{
- "From": {from},
- "To": {mimeTo},
- "Subject": {encodeRFC2047Word(subject)},
- "Content-Transfer-Encoding": {"8bit"},
- "Auto-Submitted": {"auto-generated"},
- "Precedence": {"bulk"},
- GlobalRelayMsgTypeHeader: {"Mattermost"},
- GlobalRelayChannelNameHeader: {encodeRFC2047Word(channelExport.ChannelDisplayName)},
- GlobalRelayChannelIDHeader: {encodeRFC2047Word(channelExport.ChannelId)},
- GlobalRelayChannelTypeHeader: {encodeRFC2047Word(shared.ChannelTypeDisplayName(channelExport.ChannelType))},
- }
-
- m := gomail.NewMessage(gomail.SetCharset("UTF-8"))
- m.SetHeaders(headers)
- m.SetDateHeader("Date", time.Unix(channelExport.EndTime/1000, 0).UTC())
- m.SetBody("text/plain", txtBody)
- m.AddAlternative("text/html", htmlMessage)
-
- for _, fileInfo := range channelExport.uploadedFiles {
- path := fileInfo.Path
-
- m.Attach(fileInfo.Name, gomail.SetCopyFunc(func(writer io.Writer) error {
- var reader filestore.ReadCloseSeeker
- reader, err = fileAttachmentBackend.Reader(path)
- if err != nil {
- rctx.Logger().Warn("File not found for export", mlog.String("filename", path))
- warningCount += 1
- return nil
- }
- defer reader.Close()
-
- _, err = io.Copy(writer, reader)
- if err != nil {
- return fmt.Errorf("unable to add attachment to the Global Relay export: %w", err)
- }
- return nil
- }))
- }
-
- _, err = m.WriteTo(w)
- if err != nil {
- return warningCount, fmt.Errorf("unable to generate eml file data: %w", err)
- }
- return warningCount, nil
-}
-
-func getParticipantEmails(channelExport *ChannelExport) []string {
- participantEmails := make([]string, 0, len(channelExport.Participants))
- for _, participant := range channelExport.Participants {
- participantEmails = append(participantEmails, participant.UserEmail)
- }
- return participantEmails
-}
-
-func fileInfoListBytes(fileInfoList []*model.FileInfo) int64 {
- totalBytes := int64(0)
- for _, fileInfo := range fileInfoList {
- totalBytes += fileInfo.Size
- }
- return totalBytes
-}
-
-func addPostToChannelExport(rctx request.CTX, channelExport *ChannelExport, post shared.PostExport) {
- strPostProps := post.PostProps
- bytePostProps := []byte(*strPostProps)
-
- // Added to show the username if overridden by a webhook or API integration
- originalUsername := model.SafeDereference(post.Username)
- postUserName := originalUsername
- var postPropsLocal map[string]any
- err := json.Unmarshal(bytePostProps, &postPropsLocal)
- if err != nil {
- rctx.Logger().Warn("Failed to unmarshal post Props into JSON. Ignoring username override.", mlog.Err(err))
- } else {
- if overrideUsername, ok := postPropsLocal[model.PostPropsOverrideUsername]; ok {
- postUserName = overrideUsername.(string)
- }
-
- if postUserName == originalUsername {
- if overrideUsername, ok := postPropsLocal[model.PostPropsWebhookDisplayName]; ok {
- postUserName = overrideUsername.(string)
- }
- }
- }
-
- element := postToMessage(post)
- element.PostUsername = postUserName
- channelExport.Messages = append(channelExport.Messages, element)
- channelExport.numUserMessages[*post.UserId] += 1
-}
-
-func postToMessage(post shared.PostExport) Message {
- return Message{
- Id: model.SafeDereference(post.PostId),
- SentTime: model.SafeDereference(post.PostCreateAt),
- SenderId: model.SafeDereference(post.UserId),
- SenderUsername: model.SafeDereference(post.Username),
- PostUsername: model.SafeDereference(post.Username),
- SenderUserType: string(post.UserType),
- SenderEmail: model.SafeDereference(post.UserEmail),
- Message: post.Message,
- PreviewsPost: post.PreviewID(),
- UpdateAt: post.UpdateAt,
- UpdateType: post.UpdatedType,
- EditedNewMsgId: post.EditedNewMsgId,
- }
-}
-
-func addAttachmentsToChannelExport(channelExport *ChannelExport, post shared.PostExport,
- attachmentStarts []*shared.FileUploadStartExport, attachmentDeletes []shared.PostExport, removeAttachments bool) {
- for _, start := range attachmentStarts {
- var message string
-
- if removeAttachments {
- message = fmt.Sprintf("Uploaded file %q (id '%s') was removed because it was too large to send.",
- start.FileInfo.Name, start.FileInfo.Id)
- } else {
- channelExport.uploadedFiles = append(channelExport.uploadedFiles, start.FileInfo)
- message = fmt.Sprintf("Uploaded file %s", start.FileInfo.Name)
- }
-
- uploadElement := postToMessage(post)
- uploadElement.Message = message
- channelExport.Messages = append(channelExport.Messages, uploadElement)
- }
-
- for _, deleted := range attachmentDeletes {
- uploadElement := postToMessage(deleted)
- uploadElement.Message = fmt.Sprintf("Deleted file %s", deleted.FileInfo.Name)
- channelExport.Messages = append(channelExport.Messages, uploadElement)
- }
-}
-
-func encodeRFC2047Word(s string) string {
- return mime.BEncoding.Encode("utf-8", s)
-}
diff --git a/server/enterprise/message_export/global_relay_export/global_relay_export_test.go b/server/enterprise/message_export/global_relay_export/global_relay_export_test.go
deleted file mode 100644
index a6975832362..00000000000
--- a/server/enterprise/message_export/global_relay_export/global_relay_export_test.go
+++ /dev/null
@@ -1,3126 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "archive/zip"
- "bytes"
- "encoding/base64"
- "fmt"
- "io"
- "os"
- "path"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
- "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
-)
-
-func TestGlobalRelayExport(t *testing.T) {
- templatesDir, ok := fileutils.FindDir("templates")
- require.True(t, ok)
-
- templatesContainer, err := templates.New(templatesDir)
- require.NotNil(t, templatesContainer)
- require.NoError(t, err)
-
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(tempDir)
- assert.NoError(t, err)
- })
-
- rctx := request.TestContext(t)
-
- config := filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: tempDir,
- }
-
- fileBackend, err := filestore.NewFileBackend(config)
- assert.NoError(t, err)
- exportBackend := fileBackend
-
- chanTypeDirect := model.ChannelTypeDirect
- grExportTests := []struct {
- name string
- cmhs map[string][]*model.ChannelMemberHistoryResult
- metadata map[string]*shared.MetadataChannel
- startTime int64
- endTime int64
- posts []*model.MessageExport
- attachments map[string][]*model.FileInfo
- attachmentsContent map[string]string
- expectedAttachmentContent [][]string
- maxEmailBytes int64
- numExpectedEmls int
- expectedHeaders [][]string
- expectedTexts [][]string
- notExpectedTexts [][]string
- expectedHTMLs [][]string
- expectedWarnings int
- empty bool
- }{
- {
- name: "empty",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{},
- posts: []*model.MessageExport{},
- attachments: map[string][]*model.FileInfo{},
- expectedWarnings: 0,
- empty: true,
- },
- {
- name: "posts",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "id-test1", UserEmail: "test1@test.com", Username: "test1", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "id-test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "id-test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message 1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("id-test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id2"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message 2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("id-test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id1 1970-01-01T00:00:00Z @test1 id-test1 @test1 user (test1@test.com=",
- ") message 1",
- "* post-id2 1970-01-01T00:01:40Z @test1 id-test1 @test1 user (test1@test.com=",
- ") message 2",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | id-test1 | ",
- " @test1 | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | id-test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | id-test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " id-test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message 1",
- "",
- "",
- "",
- " post-id2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " id-test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message 2",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with attachments, size ok",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test2"},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- },
- },
- "post-id-2": {
- {
- Name: "test2-attachment",
- Id: "test2-attachment",
- Path: "test2-attachment",
- CreateAt: 1,
- },
- },
- },
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- "test2-attachment": "this is the second attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
- },
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message1",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message2",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message1",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message2",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with attachments, size too large, new channel export needed",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test2"},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- Size: 90, // 90 + 8 for message = message of 98 bytes. next one will be too big.
- },
- },
- "post-id-2": {
- {
- Name: "test2-attachment",
- Id: "test2-attachment",
- Path: "test2-attachment",
- Size: 56,
- },
- },
- },
- maxEmailBytes: 100,
- numExpectedEmls: 2,
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- "test2-attachment": "this is the second attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
- },
- expectedHeaders: [][]string{
- // eml 0
- {
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- },
- // eml 1
- {
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- },
- },
-
- expectedTexts: [][]string{
- // eml 0
- {
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message1",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- },
- // eml 1
- {
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message2",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test2-attachment",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test2-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test2-attachment\"",
- }, "\r\n"),
- },
- },
-
- expectedHTMLs: [][]string{
- // eml 0
- {
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 1 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message1",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- }, "\r\n"),
- },
- // eml 1
- {
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 1 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message2",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test2-attachment",
- "",
- }, "\r\n"),
- },
- },
- expectedWarnings: 0,
- },
-
- {
- name: "posts with attachments, attachment too large, remove it but only one eml needed",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- Size: 101,
- },
- },
- },
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- },
- expectedAttachmentContent: [][]string{
- {""},
- },
- maxEmailBytes: 100,
- numExpectedEmls: 1,
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message1",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file \"test1-attachment\" (id 'test1-attachment') was removed becaus=",
- "e it was too large to send.",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message2",
- }, "\r\n"),
- }},
-
- notExpectedTexts: [][]string{{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- }},
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message1",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file "test1-attachment" (id &#=",
- "39;test1-attachment') was removed because it was too large to send.",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message2",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- },
-
- {
- name: "posts with attachments, post size and attachment too large, new channel export needed and attachment removed",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test2"},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- Size: 20, // 20 + 8 for message = message of 28 bytes. next one will be too big.
- },
- },
- "post-id-2": {
- {
- Name: "test2-attachment",
- Id: "test2-attachment",
- Path: "test2-attachment",
- Size: 31, // too big, will be deleted
- },
- },
- },
- maxEmailBytes: 30,
- numExpectedEmls: 2,
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- "test2-attachment": "this is the second attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- {""},
- },
- notExpectedTexts: [][]string{
- // eml 0
- {
- "Content-Disposition: attachment; filename=\"test2-attachment\"",
- },
- // eml 1
- {
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Disposition: attachment; filename=\"test2-attachment\"",
- },
- },
-
- expectedHeaders: [][]string{
- // eml 0
- {
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- },
- // eml 1
- {
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- },
- },
-
- expectedTexts: [][]string{
- // eml 0
- {
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message1",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- },
- // eml 1
- {
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message2",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file \"test2-attachment\" (id 'test2-attachment') was removed becaus=",
- "e it was too large to send.",
- }, "\r\n"),
- },
- },
-
- expectedHTMLs: [][]string{
- // eml 0
- {
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 1 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message1",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- }, "\r\n"),
- },
- // eml 1
- {
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 1 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message2",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file "test2-attachment" (id &#=",
- "39;test2-attachment') was removed because it was too large to send.",
- "",
- }, "\r\n"),
- },
- },
- expectedWarnings: 0,
- },
-
- {
- name: "posts with multiple attachments, size ok",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message1"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1", "test1-2"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message2"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- },
- {
- Name: "test1-attachment-2",
- Id: "test1-attachment-2",
- Path: "test1-attachment-2",
- CreateAt: 1,
- },
- },
- },
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- "test1-attachment-2": "this is the second attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- {base64.StdEncoding.EncodeToString([]byte("this is the second attachment content"))},
- },
-
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message1",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment-2",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message2",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment-2\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment-2\"",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message1",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment-2",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message2",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with deleted post and deleted attachments",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 200_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostUpdateAt: model.NewPointer(int64(200_000)),
- PostDeleteAt: model.NewPointer(int64(200_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{\"deleteBy\":\"fy8j97gwii84bk4zxprbpc9d9w\"}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- UpdateAt: 200_000,
- DeleteAt: 200_000,
- },
- },
- },
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- },
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:03:20 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:03:20Z",
- "* Duration: 3 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Deleted file test1-attachment FileDeleted 1970-01-01T00:03:20Z",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "delete message Deleted 1970-01-01T00:03:20Z",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:03:20Z
",
- " - Duration:3 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 3 | ",
- "
",
- "", "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "", "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:03:20Z | ",
- " 3 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "", "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- "", "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Deleted file test1-attachment",
- " FileDeleted",
- " 1970-01-01T00:03:20Z",
- " ",
- "",
- "", "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "", "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " delete message",
- " Deleted",
- " 1970-01-01T00:03:20Z",
- " ",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with deleted attachments, no deleted post, and at different time from original post",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- UpdateAt: 200_000,
- DeleteAt: 200_000,
- },
- },
- },
- attachmentsContent: map[string]string{
- "test1-attachment": "this is the attachment content",
- },
- expectedAttachmentContent: [][]string{
- {base64.StdEncoding.EncodeToString([]byte("this is the attachment content"))},
- },
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Deleted file test1-attachment FileDeleted 1970-01-01T00:03:20Z",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Deleted file test1-attachment",
- " FileDeleted",
- " 1970-01-01T00:03:20Z",
- " ",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with missing attachments",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id-1"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{"test1"},
- },
- {
- PostId: model.NewPointer("post-id-2"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-id-1"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{
- "post-id-1": {
- {
- Name: "test1-attachment",
- Id: "test1-attachment",
- Path: "test1-attachment",
- CreateAt: 1,
- },
- },
- },
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- "* post-id-1 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) =",
- "Uploaded file test1-attachment",
- "* post-id-2 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) =",
- "message",
- }, "\r\n"),
- strings.Join([]string{
- "Content-Disposition: attachment; filename=\"test1-attachment\"",
- "Content-Transfer-Encoding: base64",
- "Content-Type: application/octet-stream; name=\"test1-attachment\"",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "",
- "",
- " post-id-1",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " Uploaded file test1-attachment",
- "",
- "",
- "",
- " post-id-2",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 1,
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with override_username property",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{\"from_webhook\":\"true\",\"html\":\"Test HTML\",\"override_username\":\"test_username_override\",\"webhook_display_name\":\"Test Bot\"}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
- "ssage",
- "* post-id 1970-01-01T00:01:40Z @test1 test1 @test_username_override user (t=",
- "est1@test.com) message",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "",
- "",
- " post-id",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test_username_override",
- " user",
- " (test1@test.com)",
- " message",
- "",
- }, "\r\n"),
- }},
- numExpectedEmls: 1,
- },
-
- {
- name: "posts with webhook_display_name property",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{\"from_webhook\":\"true\",\"webhook_display_name\":\"Test Bot\"}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
- "ssage",
- "* post-id 1970-01-01T00:01:40Z @test1 test1 @Test Bot user (test1@test.com)=",
- " message",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- "",
- "",
- " post-id",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @Test Bot",
- " user",
- " (test1@test.com)",
- " message",
- "",
- }, "\r\n"),
- }},
- numExpectedEmls: 1,
- },
-
- {
- name: "post with permalink preview",
- cmhs: map[string][]*model.ChannelMemberHistoryResult{
- "channel-id": {
- {
- JoinTime: 0, UserId: "test1", UserEmail: "test1@test.com", Username: "test", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 8, UserId: "test2", UserEmail: "test2@test.com", Username: "test2", LeaveTime: model.NewPointer(int64(100_000)),
- },
- {
- JoinTime: 400, UserId: "test3", UserEmail: "test3@test.com", Username: "test3",
- },
- },
- },
- metadata: map[string]*shared.MetadataChannel{
- "channel-id": {
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: "channel-id",
- ChannelName: "channel-name",
- ChannelDisplayName: "channel-display-name",
- ChannelType: chanTypeDirect,
- RoomId: "direct - channel-id",
- StartTime: 1,
- EndTime: 100,
- },
- },
- startTime: 1,
- endTime: 100_000,
- posts: []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostCreateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer(`{"previewed_post":"o4w39mc1ff8y5fite4b8hacy1x"}`),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- PostRootId: model.NewPointer("post-root-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostUpdateAt: model.NewPointer(int64(100_000)),
- PostCreateAt: model.NewPointer(int64(100_000)),
- PostMessage: model.NewPointer("message"),
- PostProps: model.NewPointer("{}"),
- PostType: model.NewPointer(""),
- UserEmail: model.NewPointer("test1@test.com"),
- UserId: model.NewPointer("test1"),
- Username: model.NewPointer("test1"),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- },
- attachments: map[string][]*model.FileInfo{},
- expectedHeaders: [][]string{{
- "MIME-Version: 1.0",
- "X-Mattermost-ChannelType: direct",
- "Content-Transfer-Encoding: 8bit",
- "Precedence: bulk",
- "X-GlobalRelay-MsgType: Mattermost",
- "X-Mattermost-ChannelID: channel-id",
- "X-Mattermost-ChannelName: channel-display-name",
- "Auto-Submitted: auto-generated",
- "Date: Thu, 01 Jan 1970 00:01:40 +0000",
- "From: test1@test.com",
- "To: test1@test.com,test2@test.com",
- "Subject: Mattermost Compliance Export: channel-display-name",
- }},
-
- expectedTexts: [][]string{{
- strings.Join([]string{
- "* TeamId: team-id",
- "* TeamName: team-name",
- "* TeamDisplayName: team-display-name",
- "* ChannelId: channel-id",
- "* ChannelName: channel-name",
- "* ChannelDisplayName: channel-display-name",
- "* Started: 1970-01-01T00:00:00Z",
- "* Ended: 1970-01-01T00:01:40Z",
- "* Duration: 2 minutes",
- }, "\r\n"),
- strings.Join([]string{
- "--------",
- "Messages",
- "--------",
- "",
- "* post-id 1970-01-01T00:00:00Z @test1 test1 @test1 user (test1@test.com) me=",
- "ssage o4w39mc1ff8y5fite4b8hacy1x",
- "* post-id 1970-01-01T00:01:40Z @test1 test1 @test1 user (test1@test.com) me=",
- "ssage",
- }, "\r\n"),
- }},
-
- expectedHTMLs: [][]string{{
- strings.Join([]string{
- " ",
- " - TeamId:team-id
",
- " - TeamName:team-name
",
- " - TeamDisplayName:team-display-name",
- "
- ChannelId:channel-id
",
- " - ChannelName:channel-name
",
- " - ChannelDisplayName:channel-display-=",
- "name
",
- " - Started:1970-01-01T00:00:00Z
",
- " - Ended:1970-01-01T00:01:40Z
",
- " - Duration:2 minutes
",
- "
",
- }, "\r\n"),
- strings.Join([]string{
- "",
- " | test1 | ",
- " @test | ",
- " user | ",
- " test1@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 2 | ",
- "
",
- "",
- "",
- " | test2 | ",
- " @test2 | ",
- " user | ",
- " test2@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- "",
- "",
- " | test3 | ",
- " @test3 | ",
- " user | ",
- " test3@test.com | ",
- " 1970-01-01T00:00:00Z | ",
- " 1970-01-01T00:01:40Z | ",
- " 2 minutes | ",
- " 0 | ",
- "
",
- }, "\r\n"),
-
- strings.Join([]string{
- "",
- " post-id",
- " 1970-01-01T00:00:00Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- " o4w39mc1ff8y5fite4b8hacy1x",
- "",
- "",
- "",
- " post-id",
- " 1970-01-01T00:01:40Z",
- " @test1",
- " test1",
- " @test1",
- " user",
- " (test1@test.com)",
- " message",
- "",
- }, "\r\n"),
- }},
- expectedWarnings: 0,
- numExpectedEmls: 1,
- },
- }
-
- for _, tt := range grExportTests {
- t.Run(tt.name, func(t *testing.T) {
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- if len(tt.attachments) > 0 {
- for postId, attachments := range tt.attachments {
- call := mockStore.FileInfoStore.On("GetForPost", postId, true, true, false)
- call.Run(func(args mock.Arguments) {
- call.Return(attachments, nil)
- })
- for _, attachment := range attachments {
- if tt.expectedWarnings == 0 {
- content, ok := tt.attachmentsContent[attachment.Id]
- require.True(t, ok, "attachment not found for id: %s", attachment.Id)
- _, err = fileBackend.WriteFile(strings.NewReader(content), attachment.Path)
- require.NoError(t, err)
-
- t.Cleanup(func() {
- err = fileBackend.RemoveFile(attachment.Path)
- assert.NoError(t, err)
- })
- }
- }
- }
- }
-
- if tt.maxEmailBytes > 0 {
- origMaxEmailBytes := MaxEmailBytes
- MaxEmailBytes = tt.maxEmailBytes
- t.Cleanup(func() {
- MaxEmailBytes = origMaxEmailBytes
- })
- }
-
- exportFileName := path.Join("export", "jobName", "jobName-batch001.zip")
- results, err := GlobalRelayExport(rctx, shared.ExportParams{
- ExportType: model.ComplianceExportTypeGlobalrelayZip,
- ChannelMetadata: tt.metadata,
- Posts: tt.posts,
- ChannelMemberHistories: tt.cmhs,
- BatchPath: exportFileName,
- BatchStartTime: tt.startTime,
- BatchEndTime: tt.endTime,
- Db: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: fileBackend,
- ExportBackend: exportBackend,
- Templates: templatesContainer,
- })
- assert.NoError(t, err)
- assert.Equal(t, tt.expectedWarnings, results.NumWarnings)
-
- // channel-name - (channel-id) - 0.eml
- if !tt.empty {
- if tt.numExpectedEmls == 0 {
- require.True(t, false, "need numExpectedEmls to be at least 1")
- }
- openZipAndAssertNumEmls(t, exportBackend, exportFileName, tt.numExpectedEmls)
-
- for batchNum := 0; batchNum < tt.numExpectedEmls; batchNum++ {
- metadata := tt.metadata["channel-id"]
- emlName := fmt.Sprintf("%s - (%s) - %d.eml", metadata.ChannelName,
- metadata.ChannelId, batchNum)
- eml := openZipAndReadFileStartingWith(t, exportBackend, exportFileName, emlName)
-
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><> batchNum %d actual: \n%s", batchNum, eml)
-
- t.Run("headers", func(t *testing.T) {
- for _, expectedHeader := range tt.expectedHeaders[batchNum] {
- assert.Contains(t, eml, expectedHeader, "batchNum %d, expected: %s", batchNum, expectedHeader)
- }
- })
-
- t.Run("text-version", func(t *testing.T) {
- for _, expectedText := range tt.expectedTexts[batchNum] {
- assert.Contains(t, eml, expectedText, "batchNum %d, expected: %s", batchNum, expectedText)
- }
- if len(tt.notExpectedTexts) > 0 {
- for _, notExpectedText := range tt.notExpectedTexts[batchNum] {
- assert.NotContains(t, eml, notExpectedText, "batchNum %d, expected: %s", batchNum, notExpectedText)
- }
- }
- })
-
- t.Run("html-version", func(t *testing.T) {
- for _, expectedHTML := range tt.expectedHTMLs[batchNum] {
- assert.Contains(t, eml, expectedHTML, "batchNum %d, \nexpected:\n %s\n\nactual:\n %s\n", batchNum, expectedHTML, eml)
- }
- })
-
- assert.Len(t, tt.expectedAttachmentContent, len(tt.attachmentsContent), "every attachmentsContent should have an expectedAttachmentContent")
-
- if len(tt.expectedAttachmentContent) > 0 {
- t.Run("file encoding", func(t *testing.T) {
- for _, expected := range tt.expectedAttachmentContent[batchNum] {
- assert.Contains(t, eml, expected, "batchNum %d, \nexpected:\n %s\n\nactual:\n %s\n", batchNum, expected, eml)
- }
- })
- }
- }
- }
- })
- }
-}
-
-func openZipAndReadFileStartingWith(t *testing.T, backend filestore.FileBackend, path string, startsWith string) string {
- zipBytes, err := backend.ReadFile(path)
- require.NoError(t, err)
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- var names []string
- for _, f := range zipReader.File {
- if strings.HasPrefix(f.Name, startsWith) {
- file, err := f.Open()
- require.NoError(t, err)
- contents, err := io.ReadAll(file)
- require.NoError(t, err)
- err = file.Close()
- require.NoError(t, err)
-
- return string(contents)
- }
- names = append(names, f.Name)
- }
-
- require.True(t, false, "called openZipAndReadFileStartingWith but didn't file file starting with: %s. Found: %v", startsWith, names)
- return ""
-}
-
-func openZipAndAssertNumEmls(t *testing.T, backend filestore.FileBackend, path string, numEmls int) {
- zipBytes, err := backend.ReadFile(path)
- require.NoError(t, err)
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- require.Len(t, zipReader.File, numEmls, "numEmls wrong")
-}
diff --git a/server/enterprise/message_export/global_relay_export/main_test.go b/server/enterprise/message_export/global_relay_export/main_test.go
deleted file mode 100644
index 3a1822aa6e9..00000000000
--- a/server/enterprise/message_export/global_relay_export/main_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "testing"
-
- "github.com/mattermost/mattermost/server/v8/channels/testlib"
-)
-
-var mainHelper *testlib.MainHelper
-
-func TestMain(m *testing.M) {
- var options = testlib.HelperOptions{
- EnableResources: true,
- }
-
- mainHelper = testlib.NewMainHelperWithOptions(&options)
- defer mainHelper.Close()
-
- mainHelper.Main(m)
-}
diff --git a/server/enterprise/message_export/global_relay_export/smtp.go b/server/enterprise/message_export/global_relay_export/smtp.go
deleted file mode 100644
index adbd5a3ab8d..00000000000
--- a/server/enterprise/message_export/global_relay_export/smtp.go
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "context"
- "net/smtp"
- "os"
- "strconv"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/v8/channels/utils"
- "github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
- "github.com/mattermost/mattermost/server/v8/platform/shared/mail"
-)
-
-const (
- GlobalRelayA9Server = "mailarchivespool1.globalrelay.com"
- GlobalRelayA10Server = "feeds.globalrelay.com"
- GlobalRelayA9IP = "208.81.212.70"
- GlobalRelayA10IP = "208.81.213.24"
-
- defaultSMTPPort = "25"
- defaultInbucketSMTPPort = "10025"
-)
-
-func connectToSMTPServer(ctx context.Context, config *model.Config) (*smtp.Client, error) {
- smtpServerName := ""
- smtpServerHost := ""
- smtpPort := defaultSMTPPort
- security := model.ConnSecurityStarttls
- auth := true
- if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "A10" {
- smtpServerName = GlobalRelayA10Server
- smtpServerHost = GlobalRelayA10IP
- } else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "A9" {
- smtpServerName = GlobalRelayA9Server
- smtpServerHost = GlobalRelayA9IP
- } else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == "INBUCKET" {
- inbucketSMTPPort := os.Getenv("CI_INBUCKET_SMTP_PORT")
- if inbucketSMTPPort == "" {
- inbucketSMTPPort = defaultInbucketSMTPPort
- }
- inbucketHost := os.Getenv("CI_INBUCKET_HOST")
- if inbucketHost == "" {
- intPort, err := strconv.Atoi(inbucketSMTPPort)
- if err != nil {
- intPort = 0
- }
- inbucketHost = testutils.GetInterface(intPort)
- }
- smtpServerName = inbucketHost
- smtpServerHost = inbucketHost
- smtpPort = inbucketSMTPPort
- auth = false
- } else if *config.MessageExportSettings.GlobalRelaySettings.CustomerType == model.GlobalrelayCustomerTypeCustom {
- customSMTPPort := *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPPort
- if customSMTPPort != "" {
- smtpPort = customSMTPPort
- }
- smtpServerName = *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPServerName
- smtpServerHost = *config.MessageExportSettings.GlobalRelaySettings.CustomSMTPServerName
- }
-
- smtpConfig := &mail.SMTPConfig{
- ConnectionSecurity: security,
- SkipServerCertificateVerification: false,
- Hostname: utils.GetHostnameFromSiteURL(*config.ServiceSettings.SiteURL),
- ServerName: smtpServerName,
- Server: smtpServerHost,
- Port: smtpPort,
- EnableSMTPAuth: auth,
- Username: *config.MessageExportSettings.GlobalRelaySettings.SMTPUsername,
- Password: *config.MessageExportSettings.GlobalRelaySettings.SMTPPassword,
- ServerTimeout: *config.MessageExportSettings.GlobalRelaySettings.SMTPServerTimeout,
- }
- conn, err1 := mail.ConnectToSMTPServerAdvanced(smtpConfig)
- if err1 != nil {
- return nil, err1
- }
-
- c, err2 := mail.NewSMTPClientAdvanced(
- ctx,
- conn,
- smtpConfig,
- )
- if err2 != nil {
- conn.Close()
- return nil, err2
- }
- return c, nil
-}
diff --git a/server/enterprise/message_export/global_relay_export/test_helpers.go b/server/enterprise/message_export/global_relay_export/test_helpers.go
deleted file mode 100644
index 28bc8aaf973..00000000000
--- a/server/enterprise/message_export/global_relay_export/test_helpers.go
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "net/mail"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func AssertHeaderContains(t *testing.T, msg string, expected map[string]string) {
- t.Helper()
- m, err := mail.ReadMessage(strings.NewReader(msg))
- require.NoError(t, err)
-
- for k, v := range expected {
- assert.Equal(t, v, m.Header.Get(k))
- }
-}
-
-func CleanTestOutput(msg string) string {
- msg = strings.Replace(msg, "=\r\n", "", -1)
- msg = strings.Replace(msg, "\r\n", "\n", -1)
- return msg
-}
diff --git a/server/enterprise/message_export/global_relay_export/to_html.go b/server/enterprise/message_export/global_relay_export/to_html.go
deleted file mode 100644
index 4b59992fe52..00000000000
--- a/server/enterprise/message_export/global_relay_export/to_html.go
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package global_relay_export
-
-import (
- "bytes"
- "html/template"
- "sort"
- "strings"
- "time"
-
- "github.com/hako/durafmt"
-
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
-)
-
-func TimestampConvert(timestampMS int64) string {
- return time.Unix(timestampMS/1000, 0).UTC().Format(time.RFC3339)
-
- // for testing: (keep in case we need to be specific -- helps when you have joins and leaves within millis of each other)
- //return fmt.Sprintf("%d", timestampMS)
-}
-
-func channelExportToHTML(rctx request.CTX, channelExport *ChannelExport, t *templates.Container) (string, error) {
- durationMilliseconds := channelExport.EndTime - channelExport.StartTime
- duration := time.Duration(durationMilliseconds) * time.Millisecond
-
- var participantRowsBuffer bytes.Buffer
- for i := range channelExport.Participants {
- participantHTML, err := participantToHTML(&channelExport.Participants[i], t)
- if err != nil {
- rctx.Logger().Error("Unable to render participant html for compliance export", mlog.Err(err))
- continue
- }
- participantRowsBuffer.WriteString(participantHTML)
- }
-
- var messagesBuffer bytes.Buffer
- sort.Slice(channelExport.Messages, func(i, j int) bool {
- if channelExport.Messages[i].SentTime == channelExport.Messages[j].SentTime {
- return !strings.HasPrefix(channelExport.Messages[i].Message, "Uploaded file") &&
- !strings.HasPrefix(channelExport.Messages[i].Message, "Deleted file") &&
- channelExport.Messages[i].UpdateType == ""
- }
- return channelExport.Messages[i].SentTime < channelExport.Messages[j].SentTime
- })
- for i := range channelExport.Messages {
- messageHTML, err := messageToHTML(&channelExport.Messages[i], t)
- if err != nil {
- rctx.Logger().Error("Unable to render message html for compliance export", mlog.Err(err))
- continue
- }
- messagesBuffer.WriteString(messageHTML)
- }
-
- data := templates.Data{
- Props: map[string]any{
- "TeamId": channelExport.TeamId,
- "TeamName": channelExport.TeamName,
- "TeamDisplayName": channelExport.TeamDisplayName,
- "ChannelId": channelExport.ChannelId,
- "ChannelName": channelExport.ChannelName,
- "ChannelDisplayName": channelExport.ChannelDisplayName,
- "Started": TimestampConvert(channelExport.StartTime),
- "Ended": TimestampConvert(channelExport.EndTime),
- "Duration": durafmt.Parse(duration.Round(time.Minute)).String(),
- "ParticipantRows": template.HTML(participantRowsBuffer.String()),
- "Messages": template.HTML(messagesBuffer.String()),
- "ExportDate": TimestampConvert(channelExport.ExportedOn),
- },
- }
-
- return t.RenderToString("globalrelay_compliance_export", data)
-}
-
-func participantToHTML(participant *ParticipantRow, t *templates.Container) (string, error) {
- durationMilliseconds := participant.LeaveTime - participant.JoinTime
- duration := time.Duration(durationMilliseconds) * time.Millisecond
-
- data := templates.Data{
- Props: map[string]any{
- "UserId": participant.UserId,
- "Username": participant.Username,
- "UserType": participant.UserType,
- "Email": participant.UserEmail,
- "Joined": TimestampConvert(participant.JoinTime),
- "Left": TimestampConvert(participant.LeaveTime),
- "Duration": durafmt.Parse(duration.Round(time.Minute)).String(),
- "NumMessages": participant.MessagesSent,
- },
- }
- return t.RenderToString("globalrelay_compliance_export_participant_row", data)
-}
-
-func messageToHTML(message *Message, t *templates.Container) (string, error) {
- postUsername := message.PostUsername
- // Added to improve readability
- if postUsername != "" {
- postUsername = "@" + postUsername
- }
- data := templates.Data{
- Props: map[string]any{
- "PostId": message.Id,
- "SentTime": TimestampConvert(message.SentTime),
- "UserId": message.SenderId,
- "Username": message.SenderUsername,
- "PostUsername": postUsername,
- "UserType": message.SenderUserType,
- "Email": message.SenderEmail,
- "Message": message.Message,
- "PreviewsPost": message.PreviewsPost,
- "UpdateTime": TimestampConvert(message.UpdateAt),
- "UpdateType": message.UpdateType,
- "EditedNewMsgId": message.EditedNewMsgId,
- },
- }
-
- return t.RenderToString("globalrelay_compliance_export_message", data)
-}
diff --git a/server/enterprise/message_export/main_test.go b/server/enterprise/message_export/main_test.go
deleted file mode 100644
index 794f0ca457b..00000000000
--- a/server/enterprise/message_export/main_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "testing"
-
- "github.com/mattermost/mattermost/server/v8/channels/api4"
- "github.com/mattermost/mattermost/server/v8/channels/testlib"
-)
-
-var mainHelper *testlib.MainHelper
-
-func TestMain(m *testing.M) {
- mainHelper = testlib.NewMainHelper()
- defer mainHelper.Close()
- api4.SetMainHelper(mainHelper)
-
- mainHelper.Main(m)
-}
diff --git a/server/enterprise/message_export/membership_map.go b/server/enterprise/message_export/membership_map.go
deleted file mode 100644
index 26dcb898972..00000000000
--- a/server/enterprise/message_export/membership_map.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-type MembershipMapUser struct {
- userId string
- email string
- username string
-}
-
-// Provides a clean interface for tracking the users that are present in any number of channels by channel id and user email
-type MembershipMap map[string]map[string]MembershipMapUser
-
-func (m *MembershipMap) init(channelId string) {
- if *m == nil {
- *m = make(map[string]map[string]MembershipMapUser)
- }
- if (*m)[channelId] == nil {
- (*m)[channelId] = make(map[string]MembershipMapUser)
- }
-}
-
-func (m *MembershipMap) AddUserToChannel(channelId string, user MembershipMapUser) {
- m.init(channelId)
- if !m.IsUserInChannel(channelId, user.email) {
- (*m)[channelId][user.email] = user
- }
-}
-
-func (m *MembershipMap) RemoveUserFromChannel(channelId string, userEmail string) {
- m.init(channelId)
- delete((*m)[channelId], userEmail)
-}
-
-func (m *MembershipMap) IsUserInChannel(channelId string, userEmail string) bool {
- m.init(channelId)
- _, exists := (*m)[channelId][userEmail]
- return exists
-}
-
-func (m *MembershipMap) GetUserEmailsInChannel(channelId string) []string {
- m.init(channelId)
- users := make([]string, 0, len((*m)[channelId]))
- for k := range (*m)[channelId] {
- users = append(users, k)
- }
- return users
-}
-
-func (m *MembershipMap) GetUsersInChannel(channelId string) []MembershipMapUser {
- m.init(channelId)
- users := make([]MembershipMapUser, 0, len((*m)[channelId]))
- for _, v := range (*m)[channelId] {
- users = append(users, v)
- }
- return users
-}
diff --git a/server/enterprise/message_export/membership_map_test.go b/server/enterprise/message_export/membership_map_test.go
deleted file mode 100644
index 05670d63575..00000000000
--- a/server/enterprise/message_export/membership_map_test.go
+++ /dev/null
@@ -1,79 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/mattermost/mattermost/server/public/model"
-)
-
-func TestMembershipMap(t *testing.T) {
- membershipMap := make(MembershipMap)
-
- channelId := model.NewId()
-
- user1 := &MembershipMapUser{
- email: model.NewId() + "@mattermost.com",
- username: model.NewId(),
- userId: model.NewId(),
- }
- user2 := &MembershipMapUser{
- email: model.NewId() + "@mattermost.com",
- username: model.NewId(),
- userId: model.NewId(),
- }
-
- assert.False(t, membershipMap.IsUserInChannel(channelId, user1.email))
- membershipMap.AddUserToChannel(channelId, *user1)
- assert.True(t, membershipMap.IsUserInChannel(channelId, user1.email))
-
- assert.False(t, membershipMap.IsUserInChannel(channelId, user2.email))
- membershipMap.AddUserToChannel(channelId, *user2)
- assert.True(t, membershipMap.IsUserInChannel(channelId, user2.email))
-
- // ensure that the correct user emails are returned
- emails := membershipMap.GetUserEmailsInChannel(channelId)
- assert.Len(t, emails, 2)
- assert.Contains(t, emails, user1.email)
- assert.Contains(t, emails, user2.email)
-
- // ensure that the correct user objects are returned
- users := membershipMap.GetUsersInChannel(channelId)
- assert.Len(t, users, 2)
- if users[0].userId == user1.userId {
- assert.Equal(t, user1.username, users[0].username)
- assert.Equal(t, user1.email, users[0].email)
- assert.Equal(t, user2.userId, users[1].userId)
- assert.Equal(t, user2.username, users[1].username)
- assert.Equal(t, user2.email, users[1].email)
- } else if users[0].userId == user2.userId {
- assert.Equal(t, user2.username, users[0].username)
- assert.Equal(t, user2.email, users[0].email)
- assert.Equal(t, user1.userId, users[1].userId)
- assert.Equal(t, user1.username, users[1].username)
- assert.Equal(t, user1.email, users[1].email)
- } else {
- assert.Fail(t, "First returned user is not recognized")
- }
-
- // remove user1 from the channel
- membershipMap.RemoveUserFromChannel(channelId, user1.email)
- assert.False(t, membershipMap.IsUserInChannel(channelId, user1.email))
- assert.True(t, membershipMap.IsUserInChannel(channelId, user2.email))
-
- // ensure that user2's email is returned
- emails = membershipMap.GetUserEmailsInChannel(channelId)
- assert.Len(t, emails, 1)
- assert.Contains(t, emails, user2.email)
-
- // ensure that only user2 is returned
- users = membershipMap.GetUsersInChannel(channelId)
- assert.Len(t, users, 1)
- assert.Equal(t, user2.userId, users[0].userId)
- assert.Equal(t, user2.username, users[0].username)
- assert.Equal(t, user2.email, users[0].email)
-}
diff --git a/server/enterprise/message_export/message_export.go b/server/enterprise/message_export/message_export.go
deleted file mode 100644
index b124d863208..00000000000
--- a/server/enterprise/message_export/message_export.go
+++ /dev/null
@@ -1,274 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "encoding/json"
- "errors"
- "strconv"
- "time"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/app"
- "github.com/mattermost/mattermost/server/v8/einterfaces"
- ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
-)
-
-const GlobalRelayExportFilename = "global-relay.zip"
-
-type MessageExportInterfaceImpl struct {
- Server *app.Server
-}
-
-type MessageExportJobInterfaceImpl struct {
- Server *app.Server
-}
-
-func init() {
- app.RegisterJobsMessageExportJobInterface(func(s *app.Server) ejobs.MessageExportJobInterface {
- return &MessageExportJobInterfaceImpl{s}
- })
- app.RegisterMessageExportInterface(func(app *app.App) einterfaces.MessageExportInterface {
- return &MessageExportInterfaceImpl{app.Srv()}
- })
-}
-
-func (m *MessageExportInterfaceImpl) StartSynchronizeJob(rctx request.CTX, exportFromTimestamp int64) (*model.Job, *model.AppError) {
- // if a valid export time was specified, put it in the job data
- jobData := make(map[string]string)
- if exportFromTimestamp >= 0 {
- jobData[shared.JobDataBatchStartTime] = strconv.FormatInt(exportFromTimestamp, 10)
- }
-
- // passing nil for job data will cause the worker to inherit start time from previously successful job
- job, err := m.Server.Jobs.CreateJob(rctx, model.JobTypeMessageExport, jobData)
- if err != nil {
- return nil, err
- }
-
- ticker := time.NewTicker(time.Second)
- defer ticker.Stop()
-
- for job.Status == model.JobStatusPending ||
- job.Status == model.JobStatusInProgress ||
- job.Status == model.JobStatusCancelRequested {
- select {
- case <-ticker.C:
- job, err = m.Server.Jobs.GetJob(rctx, job.Id)
- if err != nil {
- return nil, err
- }
- case <-rctx.Context().Done():
- return nil, model.NewAppError("StartSynchronizeJob", "ent.jobs.start_synchronize_job.timeout", nil, "", 0).Wrap(rctx.Context().Err())
- }
- }
-
- return job, nil
-}
-
-func RunBatch(rctx request.CTX, data shared.JobData, params shared.BackendParams) (shared.RunExportResults, shared.JobData, error) {
- start := time.Now()
- var err error
- var res shared.RunExportResults
- data, err = GetDataForBatch(rctx, data, params)
- if err != nil {
- return res, data, err
- }
-
- if data.Finished {
- return res, data, nil
- }
-
- // Now write the data to the export type.
- res, err = RunExportByType(rctx, DataToExportParams(data), params)
- if err != nil {
- return res, data, err
- }
-
- data.ProcessingPostsMs = append(data.ProcessingPostsMs, res.ProcessingPostsMs)
- data.ProcessingXmlMs = append(data.ProcessingXmlMs, res.ProcessingXmlMs)
- data.TransferringFilesMs = append(data.TransferringFilesMs, res.TransferringFilesMs)
- data.TransferringZipMs = append(data.TransferringZipMs, res.TransferringZipMs)
- data.TotalBatchMs = append(data.TotalBatchMs, time.Since(start).Milliseconds())
- data.WarningCount += res.NumWarnings
- data.BatchStartTime = data.BatchEndTime
-
- return res, data, err
-}
-
-// GetDataForBatch gets the posts for this batch and updates JobData with the current state.
-func GetDataForBatch(rctx request.CTX, data shared.JobData, params shared.BackendParams) (shared.JobData, error) {
- start := time.Now()
- var err error
- // Using BatchSize+1 is a trick to test whether or not we've reached the final batch.
- data.PostsToExport, data.Cursor, err = params.Store.Compliance().MessageExport(rctx, data.Cursor, data.BatchSize+1)
- if err != nil {
- return data, err
- }
- data.MessageExportMs = append(data.MessageExportMs, time.Since(start).Milliseconds())
-
- if len(data.PostsToExport) == data.BatchSize+1 {
- // We still have posts after this current batch.
- // Remove the last post, we have to leave it for the next batch.
- lastPostIdx := len(data.PostsToExport) - 1
- data.PostsToExport = data.PostsToExport[:lastPostIdx]
- lastPostIdx = len(data.PostsToExport) - 1
- data.Cursor.LastPostUpdateAt = *data.PostsToExport[lastPostIdx].PostUpdateAt
- data.Cursor.LastPostId = *data.PostsToExport[lastPostIdx].PostId
- data.BatchEndTime = data.Cursor.LastPostUpdateAt
- } else {
- // We've reached the final batch; we need to include all join/leave events that occur after the lastpost.
- // This will let us also pick up the joins/leaves that occur after lastPostUpdateAt but before JobEndTime.
- data.BatchEndTime = data.JobEndTime
- }
-
- if len(data.PostsToExport) == 0 {
- data.Finished = true
- return data, nil
- }
-
- rctx.Logger().Debug("Found posts to export", mlog.Int("num_posts", len(data.PostsToExport)))
- data.MessagesExported += len(data.PostsToExport)
- data.BatchNumber++
- data.BatchPath = shared.GetBatchPath(data.ExportDir, data.BatchStartTime, data.BatchEndTime, data.BatchNumber)
-
- return data, nil
-}
-
-type ExportParams struct {
- ExportType string
- ChannelMetadata map[string]*shared.MetadataChannel
- ChannelMemberHistories map[string][]*model.ChannelMemberHistoryResult
- PostsToExport []*model.MessageExport
- JobStartTime int64
- BatchPath string
- BatchStartTime int64
- BatchEndTime int64
-}
-
-func DataToExportParams(data shared.JobData) ExportParams {
- return ExportParams{
- ExportType: data.ExportType,
- ChannelMetadata: data.ChannelMetadata,
- ChannelMemberHistories: data.ChannelMemberHistories,
- PostsToExport: data.PostsToExport,
- JobStartTime: data.JobStartTime,
- BatchPath: data.BatchPath,
- BatchStartTime: data.BatchStartTime,
- BatchEndTime: data.BatchEndTime,
- }
-}
-
-func RunExportByType(rctx request.CTX, p ExportParams, b shared.BackendParams) (results shared.RunExportResults, err error) {
- preparePosts(rctx, p.PostsToExport)
-
- exportParams := shared.ExportParams{
- ExportType: p.ExportType,
- ChannelMetadata: p.ChannelMetadata,
- Posts: p.PostsToExport,
- ChannelMemberHistories: p.ChannelMemberHistories,
- JobStartTime: p.JobStartTime,
- BatchPath: p.BatchPath,
- BatchStartTime: p.BatchStartTime,
- BatchEndTime: p.BatchEndTime,
- Config: b.Config,
- Db: b.Store,
- FileAttachmentBackend: b.FileAttachmentBackend,
- ExportBackend: b.ExportBackend,
- Templates: b.HtmlTemplates,
- }
-
- switch p.ExportType {
- case model.ComplianceExportTypeCsv:
- rctx.Logger().Debug("Exporting CSV")
- return csv_export.CsvExport(rctx, exportParams)
-
- case model.ComplianceExportTypeActiance:
- rctx.Logger().Debug("Exporting Actiance")
- return actiance_export.ActianceExport(rctx, exportParams)
-
- case model.ComplianceExportTypeGlobalrelay, model.ComplianceExportTypeGlobalrelayZip:
- rctx.Logger().Debug("Exporting GlobalRelay")
- return global_relay_export.GlobalRelayExport(rctx, exportParams)
-
- default:
- return results, errors.New("Unknown output format: " + p.ExportType)
- }
-}
-
-func preparePosts(rctx request.CTX, postsToExport []*model.MessageExport) {
- // go through all the posts and if the post's props contain 'from_bot' - override the IsBot field, since it's possible that the sender is not a user, but was a Bot and vise-versa
- for _, post := range postsToExport {
- if post.PostProps != nil {
- props := map[string]any{}
-
- if json.Unmarshal([]byte(*post.PostProps), &props) == nil {
- if val, ok := props["from_bot"]; ok {
- post.IsBot = val == "true"
- }
- }
- }
-
- // Team info can be null for DM/GM channels.
- if post.TeamId == nil {
- post.TeamId = new(string)
- }
- if post.TeamName == nil {
- post.TeamName = new(string)
- }
- if post.TeamDisplayName == nil {
- post.TeamDisplayName = new(string)
- }
-
- // make sure user information is present. Set defaults and log an error otherwise.
- if post.ChannelId == nil {
- rctx.Logger().Warn("ChannelId is missing for post", mlog.String("post_id", *post.PostId))
- post.ChannelId = new(string)
- }
- if post.ChannelName == nil {
- rctx.Logger().Warn("ChannelName is missing for post", mlog.String("post_id", *post.PostId))
- post.ChannelName = new(string)
- }
- if post.ChannelDisplayName == nil {
- rctx.Logger().Warn("ChannelDisplayName is missing for post", mlog.String("post_id", *post.PostId))
- post.ChannelDisplayName = new(string)
- }
- if post.ChannelType == nil {
- rctx.Logger().Warn("ChannelType is missing for post", mlog.String("post_id", *post.PostId))
- post.ChannelType = new(model.ChannelType)
- }
-
- if post.UserId == nil {
- rctx.Logger().Warn("UserId is missing for post", mlog.String("post_id", *post.PostId))
- post.UserId = new(string)
- }
- if post.UserEmail == nil {
- rctx.Logger().Warn("UserEmail is missing for post", mlog.String("post_id", *post.PostId))
- post.UserEmail = new(string)
- }
- if post.Username == nil {
- rctx.Logger().Warn("Username is missing for post", mlog.String("post_id", *post.PostId))
- post.Username = new(string)
- }
-
- if post.PostType == nil {
- rctx.Logger().Warn("Type is missing for post", mlog.String("post_id", *post.PostId))
- post.PostType = new(string)
- }
- if post.PostMessage == nil {
- rctx.Logger().Warn("Message is missing for post", mlog.String("post_id", *post.PostId))
- post.PostMessage = new(string)
- }
- if post.PostCreateAt == nil {
- rctx.Logger().Warn("CreateAt is missing for post", mlog.String("post_id", *post.PostId))
- post.PostCreateAt = new(int64)
- }
- }
-}
diff --git a/server/enterprise/message_export/message_export_test.go b/server/enterprise/message_export/message_export_test.go
deleted file mode 100644
index 8ef2103e178..00000000000
--- a/server/enterprise/message_export/message_export_test.go
+++ /dev/null
@@ -1,2601 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "archive/zip"
- "bytes"
- _ "embed"
- "encoding/xml"
- "fmt"
- "io"
- "os"
- "path"
- "slices"
- "strconv"
- "strings"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/actiance_export"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-//go:embed testdata/actianceXMLHeader.tmpl
-var actianceXMLHeader string
-
-//go:embed testdata/actianceE2E1Batch1ch2.tmpl
-var actianceE2E1Batch1ch2tmpl string
-
-//go:embed testdata/actianceE2E1Batch1ch3.tmpl
-var actianceE2E1Batch1ch3tmpl string
-
-//go:embed testdata/actianceE2E1Batch2.tmpl
-var actianceE2E1Batch2tmpl string
-
-//go:embed testdata/actianceE2E1Batch3ch2.tmpl
-var actianceE2E1Batch3ch2tmpl string
-
-//go:embed testdata/actianceE2E1Batch3ch4.tmpl
-var actianceE2E1Batch3ch4tmpl string
-
-//go:embed testdata/actianceE2E2.tmpl
-var actianceE2E2 string
-
-//go:embed testdata/grE2E1Batch1Summary.tmpl
-var grE2E1Batch1Summary string
-
-//go:embed testdata/grE2E1Batch1.tmpl
-var grE2E1Batch1 string
-
-//go:embed testdata/grE2E1Batch1SummaryCh3.tmpl
-var grE2E1Batch1SummaryCh3 string
-
-//go:embed testdata/grE2E1Batch1Ch3.tmpl
-var grE2E1Batch1Ch3 string
-
-//go:embed testdata/grE2E1Batch2Summary.tmpl
-var grE2E1Batch2Summary string
-
-//go:embed testdata/grE2E1Batch2.tmpl
-var grE2E1Batch2 string
-
-//go:embed testdata/grE2E1Batch3Summary.tmpl
-var grE2E1Batch3Summary string
-
-//go:embed testdata/grE2E1Batch3.tmpl
-var grE2E1Batch3 string
-
-//go:embed testdata/grE2E1Batch3SummaryCh4.tmpl
-var grE2E1Batch3SummaryCh4 string
-
-//go:embed testdata/grE2E1Batch3Ch4.tmpl
-var grE2E1Batch3Ch4 string
-
-//go:embed testdata/csvE2E1Batch1.tmpl
-var csvE2E1Batch1 string
-
-//go:embed testdata/csvE2E1Batch2.tmpl
-var csvE2E1Batch2 string
-
-//go:embed testdata/csvE2E1Batch3.tmpl
-var csvE2E1Batch3 string
-
-//go:embed testdata/grE2E2Batch1Summary.tmpl
-var grE2E2Batch1Summary string
-
-//go:embed testdata/grE2E2Batch1.tmpl
-var grE2E2Batch1 string
-
-//go:embed testdata/csvE2E2Batch1.tmpl
-var csvE2E2Batch1 string
-
-//go:embed testdata/grE2E3Batch1SummaryPerm1.tmpl
-var grE2E3Batch1SummaryPerm1 string
-
-//go:embed testdata/grE2E3Batch1SummaryPerm2.tmpl
-var grE2E3Batch1SummaryPerm2 string
-
-//go:embed testdata/grE2E3Batch1SummaryPerm3.tmpl
-var grE2E3Batch1SummaryPerm3 string
-
-//go:embed testdata/grE2E3Batch1SummaryPerm4.tmpl
-var grE2E3Batch1SummaryPerm4 string
-
-//go:embed testdata/grE2E3Batch1Perm1.tmpl
-var grE2E3Batch1Perm1 string
-
-//go:embed testdata/grE2E3Batch1Perm2.tmpl
-var grE2E3Batch1Perm2 string
-
-//go:embed testdata/grE2E3Batch1Perm3.tmpl
-var grE2E3Batch1Perm3 string
-
-//go:embed testdata/grE2E3Batch1Perm4.tmpl
-var grE2E3Batch1Perm4 string
-
-//go:embed testdata/grE2E3Batch2SummaryPerm1.tmpl
-var grE2E3Batch2SummaryPerm1 string
-
-//go:embed testdata/grE2E3Batch2SummaryPerm2.tmpl
-var grE2E3Batch2SummaryPerm2 string
-
-//go:embed testdata/grE2E3Batch2Perm1.tmpl
-var grE2E3Batch2Perm1 string
-
-//go:embed testdata/grE2E3Batch2Perm2.tmpl
-var grE2E3Batch2Perm2 string
-
-//go:embed testdata/grE2E4Summary.tmpl
-var grE2E4Summary string
-
-//go:embed testdata/csvE2E3Batch1Perm1.tmpl
-var csvE2E3Batch1Perm1 string
-
-//go:embed testdata/csvE2E3Batch1Perm2.tmpl
-var csvE2E3Batch1Perm2 string
-
-//go:embed testdata/csvE2E3Batch1Perm3.tmpl
-var csvE2E3Batch1Perm3 string
-
-//go:embed testdata/csvE2E3Batch1Perm4.tmpl
-var csvE2E3Batch1Perm4 string
-
-//go:embed testdata/csvE2E3Batch2Perm1.tmpl
-var csvE2E3Batch2Perm1 string
-
-//go:embed testdata/csvE2E3Batch2Perm2.tmpl
-var csvE2E3Batch2Perm2 string
-
-//go:embed testdata/csvE2E4Batch1.tmpl
-var csvE2E4Batch1 string
-
-func conv(dateTime int64) string {
- return global_relay_export.TimestampConvert(dateTime)
-}
-
-type MyReporter struct {
- mock.Mock
-}
-
-func (mr *MyReporter) ReportProgressMessage(message string) {
- mr.Called(message)
-}
-
-func TestRunExportByType(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- testRunExportByType(t, fileBackend, exportTempDir, fileBackend, exportTempDir)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
- require.NoError(t, err)
-
- testRunExportByType(t, exportBackend, exportTempDir, attachmentBackend, attachmentTempDir)
- })
-}
-
-func testRunExportByType(t *testing.T, exportBackend filestore.FileBackend, exportDir string, attachmentBackend filestore.FileBackend, attachmentDir string) {
- rctx := request.TestContext(t)
-
- chanTypeDirect := model.ChannelTypeDirect
-
- t.Run("missing user info", func(t *testing.T) {
- t.Cleanup(func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- })
-
- posts := []*model.MessageExport{
- {
- PostId: model.NewPointer("post-id"),
- PostOriginalId: model.NewPointer("post-original-id"),
- TeamId: model.NewPointer("team-id"),
- TeamName: model.NewPointer("team-name"),
- TeamDisplayName: model.NewPointer("team-display-name"),
- ChannelId: model.NewPointer("channel-id"),
- ChannelName: model.NewPointer("channel-name"),
- ChannelDisplayName: model.NewPointer("channel-display-name"),
- PostCreateAt: model.NewPointer(int64(1)),
- PostUpdateAt: model.NewPointer(int64(1)),
- PostMessage: model.NewPointer("message"),
- UserEmail: model.NewPointer("test@example.com"),
- Username: model.NewPointer("Mr. Test"),
- UserId: model.NewPointer(st.NewTestID()),
- ChannelType: &chanTypeDirect,
- PostFileIds: []string{},
- },
- }
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", int64(1), int64(1)).
- Return([]string{"channel-id"}, nil)
- mockStore.ChannelStore.On("GetMany", []string{"channel-id"}, true).
- Return(model.ChannelList{{
- Id: "channel-id",
- DisplayName: "channel-display-name",
- Name: "channel-name",
- Type: chanTypeDirect,
- }}, nil)
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", int64(1), int64(1), []string{"channel-id"}).Return([]*model.ChannelMemberHistoryResult{}, nil)
-
- myMockReporter := MyReporter{}
- defer myMockReporter.AssertExpectations(t)
- myMockReporter.On("ReportProgressMessage", "Exporting channel information for 1 channels.")
- myMockReporter.On("ReportProgressMessage", "Calculating channel activity: 0/1 channels completed.")
-
- channelMetadata, channelMemberHistories, err := shared.CalculateChannelExports(rctx,
- shared.ChannelExportsParams{
- Store: shared.NewMessageExportStore(mockStore),
- ExportPeriodStartTime: 1,
- ExportPeriodEndTime: 1,
- ChannelBatchSize: 100,
- ChannelHistoryBatchSize: 100,
- ReportProgressMessage: myMockReporter.ReportProgressMessage,
- })
- assert.NoError(t, err)
-
- res, err := RunExportByType(rctx, ExportParams{
- ExportType: model.ComplianceExportTypeActiance,
- ChannelMetadata: channelMetadata,
- ChannelMemberHistories: channelMemberHistories,
- PostsToExport: posts,
- BatchPath: "testZipName",
- BatchStartTime: 1,
- BatchEndTime: 1,
- }, shared.BackendParams{
- Store: shared.NewMessageExportStore(mockStore),
- FileAttachmentBackend: attachmentBackend,
- ExportBackend: exportBackend,
- HtmlTemplates: nil,
- Config: nil,
- })
- require.NoError(t, err)
- require.Zero(t, res.NumWarnings)
- })
-}
-
-func TestRunExportJobE2EByType(t *testing.T) {
- t.Run("no dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- fileBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- testRunExportJobE2E(t, fileBackend, exportTempDir, fileBackend, exportTempDir)
- })
-
- t.Run("using dedicated export filestore", func(t *testing.T) {
- exportTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- exportBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: exportTempDir,
- })
- assert.NoError(t, err)
-
- attachmentTempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
-
- attachmentBackend, err := filestore.NewFileBackend(filestore.FileBackendSettings{
- DriverName: model.ImageDriverLocal,
- Directory: attachmentTempDir,
- })
- require.NoError(t, err)
-
- testRunExportJobE2E(t, exportBackend, exportTempDir, attachmentBackend, attachmentTempDir)
- })
-}
-
-func testRunExportJobE2E(t *testing.T, exportBackend filestore.FileBackend, exportDir string,
- attachmentBackend filestore.FileBackend, attachmentDir string) {
- if testing.Short() {
- t.Skip("skipping test in short mode.")
- }
-
- t.Run("conflicting timestamps", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- time.Sleep(1 * time.Millisecond)
- now := model.GetMillis()
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.ExportFromTimestamp = now - 1
- *cfg.MessageExportSettings.BatchSize = 2
- })
-
- for range 3 {
- _, err2 := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now,
- })
- require.NoError(t, err2)
- }
-
- job := runJobForTest(t, th, nil)
-
- warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
- require.NoError(t, err)
- require.Equal(t, 0, warnings)
-
- numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
- require.NoError(t, err)
- require.Equal(t, int64(3), numExported)
- })
-
- t.Run("actiance -- multiple batches, 1 zip per batch, output to a single directory", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- res := generateActianceBatchTest1(t, th, attachmentDir, exportDir, attachmentBackend)
-
- files, err := exportBackend.ListDirectory(res.jobExportDir)
- require.NoError(t, err)
- require.ElementsMatch(t, res.batches, files)
-
- fileContents := openZipAndReadFile(t, exportBackend, res.batches[0], res.attachments[0].Path)
-
- require.EqualValuesf(t, res.contents[0], fileContents, "file contents not equal")
- })
-
- t.Run("actiance -- multiple batches, using UntilUpdateAt", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- res := generateActianceBatchTest2(t, th, attachmentDir, exportDir)
-
- files, err := exportBackend.ListDirectory(res.jobExportDir)
- require.NoError(t, err)
- require.ElementsMatch(t, res.batches, files)
- })
-
- t.Run("actiance e2e 1", func(t *testing.T) {
- tests := []struct {
- name string
- testStopping bool
- }{
- {
- name: "full tests, no stopping",
- testStopping: false,
- },
- {
- name: "full tests, stopped and resumed",
- // This uses the same output as the previous e2e test, but tests that the job can be stopped and
- // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
- // We want to be confident that jobs can resume without data missing or added from the original run.
- testStopping: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeActiance, attachmentDir,
- exportDir, attachmentBackend, exportBackend, tt.testStopping)
- channel2 := ret.channels[0]
- channel3 := ret.channels[1]
- channel4 := ret.channels[2]
- start := ret.start
- jl := ret.joinLeaves
- posts := ret.posts
- createUpdateTimes := ret.createUpdateTimes
- attachments := ret.attachments
- contents := ret.contents
- jobEndTime := ret.jobEndTime
- batches := ret.batches
-
- // Expected data:
- // - batch1 has two channels, and we're not sure which will come first. What a pain.
-
- // actiance exports, for batch 1:
- // for ch2:
- // 3 participants entered
- // message 0
- // file for message 0 (start and ended)
- // message 1
- // file for message 1 (start and ended)
- // message 2
- // file for message 2 (start and ended)
- // 3 participants left
- // for ch3:
- // 1 participant entered
- // 1 participant left
-
- // actiance exports, for batch 2:
- // for ch2:
- // 2 participants entered
- // message 3
- // file for message 3 (start and ended)
- // message 4
- // file for message 4 (start and ended)
- // message 5
- // file for message 5 (start and ended)
- // 2 participants left
-
- // actiance exports, for batch 3:
- // for ch2:
- // 3 participants entered
- // message 6
- // file for message 6 (start and ended)
- // message 7
- // file for message 7 (start and ended)
- // message 8
- // file for message 8 (start and ended)
- // 3 participants left
- // for ch4:
- // 1 participant entered
- // 1 participant left
-
- batch1ch2 := fmt.Sprintf(actianceE2E1Batch1ch2tmpl, channel2.Id, start, jl[0].join, jl[1].join, jl[2].join,
- posts[0].Id, createUpdateTimes[0], createUpdateTimes[0], createUpdateTimes[0],
- posts[1].Id, createUpdateTimes[1], createUpdateTimes[1], createUpdateTimes[1],
- posts[2].Id, createUpdateTimes[2], createUpdateTimes[2], createUpdateTimes[2],
- jl[1].leave, jl[2].leave, createUpdateTimes[2], createUpdateTimes[2])
-
- batch1ch3 := fmt.Sprintf(actianceE2E1Batch1ch3tmpl, channel3.Id, start, jl[6].join, jl[6].leave, createUpdateTimes[2])
-
- xmlHeader := strings.TrimSpace(actianceXMLHeader)
- batch1Possibility1 := fmt.Sprintf(xmlHeader, batch1ch2, batch1ch3)
- batch1Possibility2 := fmt.Sprintf(xmlHeader, batch1ch3, batch1ch2)
-
- batch2xml := fmt.Sprintf(actianceE2E1Batch2tmpl, channel2.Id, createUpdateTimes[2], jl[0].join, jl[3].join,
- posts[3].Id, createUpdateTimes[3], createUpdateTimes[3], createUpdateTimes[3],
- posts[4].Id, createUpdateTimes[4], createUpdateTimes[4], createUpdateTimes[4],
- posts[5].Id, createUpdateTimes[5], createUpdateTimes[5], createUpdateTimes[5],
- jl[3].leave, createUpdateTimes[5], createUpdateTimes[5])
- batch2xml = strings.TrimSpace(batch2xml)
-
- // Batch3 has two channels, and we're not sure which will come first. What a pain.
- batch3ch2 := fmt.Sprintf(actianceE2E1Batch3ch2tmpl, channel2.Id, createUpdateTimes[5], jl[0].join, jl[4].join, jl[5].join,
- posts[6].Id, createUpdateTimes[6], createUpdateTimes[6], createUpdateTimes[6],
- posts[7].Id, createUpdateTimes[7], createUpdateTimes[7], createUpdateTimes[7],
- posts[8].Id, createUpdateTimes[8], createUpdateTimes[8], createUpdateTimes[8],
- jl[4].leave, jl[5].leave, jobEndTime, jobEndTime)
-
- batch3ch4 := fmt.Sprintf(actianceE2E1Batch3ch4tmpl, channel4.Id, createUpdateTimes[5], jl[7].join, jobEndTime, jobEndTime)
-
- batch3Possibility1 := fmt.Sprintf(xmlHeader, batch3ch2, batch3ch4)
-
- batch3Possibility2 := fmt.Sprintf(xmlHeader, batch3ch4, batch3ch2)
-
- for b, batchName := range batches {
- xmlContents := openZipAndReadFile(t, exportBackend, batchName, "actiance_export.xml")
-
- // this is so clunky, sorry. but it's simple.
- if b == 0 {
- if xmlContents != batch1Possibility1 && xmlContents != batch1Possibility2 {
- // to make some output
- assert.Equal(t, batch1Possibility1, xmlContents, "batch 1 possibility 1")
- assert.Equal(t, batch1Possibility2, xmlContents, "batch 1 possibility 2")
- }
- }
-
- if b == 1 {
- require.Equal(t, batch2xml, xmlContents, "batch 2")
- }
-
- if b == 2 {
- if xmlContents != batch3Possibility1 && xmlContents != batch3Possibility2 {
- // to make some output
- assert.Equal(t, batch3Possibility1, xmlContents, "batch 3 possibility 1")
- assert.Equal(t, batch3Possibility2, xmlContents, "batch 3 possibility 2")
- }
- }
-
- zipBytes, err := exportBackend.ReadFile(batchName)
- require.NoError(t, err)
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- for i := range 3 {
- num := b*3 + i
- attachmentInZip, err := zipReader.Open(attachments[num].Path)
- require.NoError(t, err)
- attachmentInZipContents, err := io.ReadAll(attachmentInZip)
- require.NoError(t, err)
- err = attachmentInZip.Close()
- require.NoError(t, err)
- require.EqualValuesf(t, contents[num], string(attachmentInZipContents), "file contents not equal")
- }
- }
- })
- }
- })
-
- t.Run("GlobalRelay e2e 1", func(t *testing.T) {
- tests := []struct {
- name string
- testStopping bool
- }{
- {
- name: "full tests, no stopping",
- testStopping: false,
- },
- {
- name: "full tests, stopped and resumed",
- // This uses the same output as the previous e2e test, but tests that the job can be stopped and
- // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
- // We want to be confident that jobs can resume without data missing or added from the original run.
- testStopping: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
- exportDir, attachmentBackend, exportBackend, tt.testStopping)
- teams := ret.teams
- channel2 := ret.channels[0]
- channel3 := ret.channels[1]
- channel4 := ret.channels[2]
- users := ret.users
- posts := ret.posts
- batchTimes := ret.batchTimes
- jobStartTime := ret.start
- jl := ret.joinLeaves
- batches := ret.batches
- cu := ret.createUpdateTimes
-
- // aligned with actiance exports, for batch 1:
- // ** except that there is no closed-out leaves at batch end
- // for ch2:
- // 3 participants entered (1, 2, 3)
- // message 0
- // file for message 0 (start and ended)
- // message 1
- // file for message 1 (start and ended)
- // message 2
- // file for message 2 (start and ended)
- // 1 participants left
- // for ch3:
- // 1 participant entered
- // 1 participant left
-
- // for batch 2:
- // for ch2:
- // 2 participants entered
- // message 3
- // file for message 3 (start and ended)
- // message 4
- // file for message 4 (start and ended)
- // message 5
- // file for message 5 (start and ended)
- // 1 participant left
-
- // for batch 3:
- // for ch2:
- // 3 participants entered
- // message 6
- // file for message 6 (start and ended)
- // message 7
- // file for message 7 (start and ended)
- // message 8
- // file for message 8 (start and ended)
- // 2 participants left
- // for ch4:
- // 1 participant entered
-
- for batchNum, batchName := range batches {
- data1 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel2.Name)
- // clean some bad csrf if present
- msg1 := global_relay_export.CleanTestOutput(data1)
-
- batchStartTime := batchTimes[batchNum].start
- batchEndTime := batchTimes[batchNum].end
- expectedBatchExportCh2 := []string{
- // batch 1 Summary
- fmt.Sprintf(grE2E1Batch1Summary,
- // 1 2 3 4 5
- channel2.DisplayName, conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 6 7 8 9
- conv(jl[1].join), conv(jl[1].leave), conv(jl[2].join), conv(jl[2].leave),
- // 10 11 12
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt),
- // 13 14 15 16
- channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
- // 17 18 19 20 21 22
- users[0].Id, users[1].Id, users[2].Id, posts[0].Id, posts[1].Id, posts[2].Id),
-
- // batch 1
- fmt.Sprintf(grE2E1Batch1,
- // 1 2 3 4 5
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime), conv(jl[1].join),
- // 6 7 8 9 10
- conv(jl[1].leave), conv(jl[2].join), conv(jl[2].leave), conv(cu[0]), conv(cu[1]),
- // 11 12 13 14 15
- conv(cu[2]), conv(jobStartTime), teams[0].Id, teams[0].Name, teams[0].DisplayName,
- // 16 17 18 19 20 21
- users[0].Id, users[1].Id, users[2].Id, posts[0].Id, posts[1].Id, posts[2].Id,
- // 22
- channel2.Id),
-
- // batch 2 Summary
- fmt.Sprintf(grE2E1Batch2Summary,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(jl[3].join), conv(jl[3].leave), conv(posts[3].CreateAt), conv(posts[4].CreateAt),
- // 9
- conv(posts[5].CreateAt),
- // 10 11 12 13
- channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
- // 14 15 16 17 18
- users[0].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- ),
-
- // batch 2
- fmt.Sprintf(grE2E1Batch2,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8 9
- conv(jl[3].join), conv(jl[3].leave), conv(cu[3]), conv(cu[4]), conv(cu[5]),
- // 10 11 12 13
- conv(jobStartTime), teams[0].Id, teams[0].Name, teams[0].DisplayName,
- // 14 15 16 17 18 19
- users[0].Id, users[1].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 20
- channel2.Id),
-
- // batch 3 Summary
- fmt.Sprintf(grE2E1Batch3Summary,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(jl[4].join), conv(jl[4].leave), conv(jl[5].join), conv(jl[5].leave),
- // 9 10 11
- conv(posts[6].CreateAt), conv(posts[7].CreateAt), conv(posts[8].CreateAt),
- // 12 13 14 15
- channel2.Id, channel2.TeamId, teams[0].Name, teams[0].DisplayName,
- // 16 17 18 19 20 21
- users[0].Id, users[1].Id, users[2].Id, posts[6].Id, posts[7].Id, posts[8].Id,
- // 22 23
- users[4].Id, users[5].Id),
-
- // batch 3
- fmt.Sprintf(grE2E1Batch3,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(jl[4].join), conv(jl[4].leave), conv(jl[5].join), conv(jl[5].leave),
- // 9 10 11 12
- conv(cu[6]), conv(cu[7]), conv(cu[8]), conv(jobStartTime),
- // 13 14 15
- teams[0].Id, teams[0].Name, teams[0].DisplayName,
- // 16 17 18 19 20 21
- users[0].Id, users[4].Id, users[5].Id, posts[6].Id, posts[7].Id, posts[8].Id,
- // 22
- channel2.Id),
- }
-
- if batchNum == 0 {
- global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
- "Subject": "Mattermost Compliance Export: the Channel Two",
- "From": "user1@email",
- "X-Mattermost-ChannelName": "the Channel Two",
- "To": "user1@email,user2@email,user3@email",
- "X-Mattermost-ChannelID": channel2.Id,
- "X-Mattermost-ChannelType": "private",
- })
- assert.Contains(t, msg1, expectedBatchExportCh2[0], "batch 1 Ch2 summary")
- assert.Contains(t, msg1, expectedBatchExportCh2[1], "batch 1 Ch2")
-
- // now read second channel's export
- data2 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel3.Name)
- // clean some bad csrf if present
- msg2 := global_relay_export.CleanTestOutput(data2)
-
- expectedBatchExportCh3 := []string{
- // batch 1 Summary
- fmt.Sprintf(grE2E1Batch1SummaryCh3,
- // 1 2 3 4
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channel3.Id,
- // 5 6 7 8 9
- conv(batchStartTime), conv(batchEndTime), users[6].Id, conv(jl[6].join), conv(jl[6].leave)),
-
- // batch 1
- fmt.Sprintf(grE2E1Batch1Ch3,
- // 1 2 3 4
- teams[0].Id, channel3.Id, conv(batchStartTime), conv(batchEndTime),
- // 5 6 7 8
- users[6].Id, conv(jl[6].join), conv(jl[6].leave), conv(jobStartTime)),
- }
-
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><>batch1 Ch3 actual\n\n%s\n\n<><>batch1 Ch3 Summary:\n\n%s\n\n",
- // msg2, expectedBatchExportCh3[0])
-
- // Channel 3
- global_relay_export.AssertHeaderContains(t, msg2, map[string]string{
- "Subject": "Mattermost Compliance Export: the Channel Three",
- "From": "user7@email",
- "X-Mattermost-ChannelName": "the Channel Three",
- "To": "user7@email",
- "X-Mattermost-ChannelID": channel3.Id,
- "X-Mattermost-ChannelType": "public",
- })
- assert.Contains(t, msg2, expectedBatchExportCh3[0], "batch 1 Ch3 summary")
- assert.Contains(t, msg2, expectedBatchExportCh3[1], "batch 1 Ch3")
- }
-
- if batchNum == 1 {
- global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
- "Subject": "Mattermost Compliance Export: the Channel Two",
- "From": "user1@email",
- "X-Mattermost-ChannelName": "the Channel Two",
- "To": "user1@email,user4@email",
- "X-Mattermost-ChannelID": channel2.Id,
- "X-Mattermost-ChannelType": "private",
- })
- assert.Contains(t, msg1, expectedBatchExportCh2[2], "batch 2 ch2 summary")
- assert.Contains(t, msg1, expectedBatchExportCh2[3], "batch 2 ch2")
- }
-
- if batchNum == 2 {
- global_relay_export.AssertHeaderContains(t, msg1, map[string]string{
- "Subject": "Mattermost Compliance Export: the Channel Two",
- "From": "user1@email",
- "X-Mattermost-ChannelName": "the Channel Two",
- "To": "user1@email,user5@email,user6@email",
- "X-Mattermost-ChannelID": channel2.Id,
- "X-Mattermost-ChannelType": "private",
- })
- assert.Contains(t, msg1, expectedBatchExportCh2[4], "batch 3 ch2 summary")
- assert.Contains(t, msg1, expectedBatchExportCh2[5], "batch 3 ch2")
-
- // now read second channel's export
- data2 := openZipAndReadFileStartingWith(t, exportBackend, batchName, channel4.Name)
- // clean some bad csrf if present
- msg2 := global_relay_export.CleanTestOutput(data2)
-
- expectedBatchExportCh4 := []string{
- // batch 1 Summary
- fmt.Sprintf(grE2E1Batch3SummaryCh4,
- // 1 2 3 4
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channel4.Id,
- // 5 6 7 8
- conv(batchStartTime), conv(batchEndTime), users[7].Id, conv(jl[7].join)),
-
- // batch 1
- fmt.Sprintf(grE2E1Batch3Ch4,
- // 1 2 3 4
- teams[0].Id, channel4.Id, conv(batchStartTime), conv(batchEndTime),
- // 5 6 7 8
- users[7].Id, conv(jl[7].join), conv(jl[7].leave), conv(jobStartTime)),
- }
-
- // Channel 3
- global_relay_export.AssertHeaderContains(t, msg2, map[string]string{
- "Subject": "Mattermost Compliance Export: the Channel Four",
- "From": "user8@email",
- "X-Mattermost-ChannelName": "the Channel Four",
- "To": "user8@email",
- "X-Mattermost-ChannelID": channel4.Id,
- "X-Mattermost-ChannelType": "public",
- })
- assert.Contains(t, msg2, expectedBatchExportCh4[0], "batch 3 Ch4 summary")
- assert.Contains(t, msg2, expectedBatchExportCh4[1], "batch 3 Ch4")
- }
- }
- })
- }
- })
-
- t.Run("CSV e2e 1", func(t *testing.T) {
- tests := []struct {
- name string
- testStopping bool
- }{
- {
- name: "full tests, no stopping",
- testStopping: false,
- },
- {
- name: "full tests, stopped and resumed",
- // This uses the same output as the previous e2e test, but tests that the job can be stopped and
- // resumed with no change to the directory, files, file contents, or job.Data that shouldn't change.
- // We want to be confident that jobs can resume without data missing or added from the original run.
- testStopping: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType1Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
- exportDir, attachmentBackend, exportBackend, tt.testStopping)
- jl := ret.joinLeaves
- cu := ret.createUpdateTimes
- posts := ret.posts
- files := ret.attachments
- users := ret.users
- channels := ret.channels
- teams := ret.teams
- batches := ret.batches
-
- // aligned with actiance exports, for batch 1:
- // ** except that there is no closed-out leaves at batch end
- // for ch2:
- // 3 participants entered
- // message 0
- // file for message 0 (start and ended)
- // message 1
- // file for message 1 (start and ended)
- // message 2
- // file for message 2 (start and ended)
- // 1 participants left
- // for ch3:
- // 1 participant entered
- // 1 participant left
-
- // for batch 2:
- // for ch2:
- // 2 participants entered
- // message 3
- // file for message 3 (start and ended)
- // message 4
- // file for message 4 (start and ended)
- // message 5
- // file for message 5 (start and ended)
- // 1 participant left
-
- // for batch 3:
- // for ch2:
- // 3 participants entered
- // message 6
- // file for message 6 (start and ended)
- // message 7
- // file for message 7 (start and ended)
- // message 8
- // file for message 8 (start and ended)
- // 2 participants left
- // for ch4:
- // 1 participant entered
-
- expectedBatchExport := []string{
- // batch 1
- fmt.Sprintf(csvE2E1Batch1,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, channels[1].Id,
- // 6 7 8 9
- users[0].Id, users[1].Id, users[2].Id, users[6].Id,
- // 10 11 12 13 14 15
- posts[0].Id, posts[1].Id, posts[2].Id, files[0].Id, files[1].Id, files[2].Id,
- // 16 17 18 19 20 21 22
- jl[0].join, jl[1].join, jl[1].leave, cu[0], jl[2].join, cu[1], jl[2].leave,
- // 23 24 25
- jl[6].join, jl[6].leave, cu[2]),
-
- // batch 2
- fmt.Sprintf(csvE2E1Batch2,
- // 1 2 3 4
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id,
- // 5 6 7 8 9
- users[0].Id, users[3].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 10 11 12
- files[3].Id, files[4].Id, files[5].Id,
- // 13 14 15 16 17 18
- jl[0].join, jl[3].join, jl[3].leave, cu[3], cu[4], cu[5]),
-
- // batch 3
- fmt.Sprintf(csvE2E1Batch3,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, channels[2].Id,
- // 6 7 8 9
- users[0].Id, users[4].Id, users[5].Id, users[7].Id,
- // 10 11 12
- posts[6].Id, posts[7].Id, posts[8].Id,
- // 13 14 15
- files[6].Id, files[7].Id, files[8].Id,
- // 16 17 18 19 20 21
- jl[0].join, jl[4].join, jl[4].leave, cu[6], cu[7], cu[8],
- // 22 23 24
- jl[5].join, jl[5].leave, jl[7].join),
- }
-
- for batchNum, batchName := range batches {
- export := openZipAndReadFileNum(t, exportBackend, batchName, 0)
-
- exportLines := strings.Split(export, "\n")
- expectedLines := strings.Split(expectedBatchExport[batchNum], "\n")
-
- // the export is not always sorted when there are > 1 channels, so do this:
- assert.Len(t, exportLines, len(expectedLines))
- for _, l := range expectedLines {
- assert.Contains(t, exportLines, l, "batch %d, batchName: %s, \nExpected:\n\n%s\n\nGot:\n\n%v\n\n", batchNum+1, batchName, l, exportLines)
- }
- }
- })
- }
- })
-
- t.Run("actiance e2e 2 - post from user not in channel", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeActiance, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- channel2 := ret.channels[0]
- start := ret.start
- jl := ret.joinLeaves
- posts := ret.posts
- createUpdateTimes := ret.createUpdateTimes
- jobEndTime := ret.jobEndTime
- batches := ret.batches
-
- // actiance export:
- // 2 participants entered
- // message 1
- // message 2
- // 2 participants left
-
- // Expected data:
- batch1xml := fmt.Sprintf(strings.TrimSpace(actianceE2E2), channel2.Id, start, jl[0].join, start,
- posts[0].Id, createUpdateTimes[0],
- posts[1].Id, createUpdateTimes[1],
- jobEndTime, jobEndTime, jobEndTime)
-
- xmlContents := openZipAndReadFile(t, exportBackend, batches[0], "actiance_export.xml")
-
- require.Equal(t, batch1xml, xmlContents, "batch 1")
- })
-
- t.Run("GlobalRelay e2e 2 - post from user not in channel", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- posts := ret.posts
- batchTimes := ret.batchTimes
- jobStartTime := ret.start
- jl := ret.joinLeaves
- batches := ret.batches
- cu := ret.createUpdateTimes
- users := ret.users
- channels := ret.channels
- teams := ret.teams
-
- // to align with actiance export:
- // 2 participants entered
- // message 1
- // message 2
- // 2 participants left
-
- batchStartTime := batchTimes[0].start
- batchEndTime := batchTimes[0].end
-
- expectedBatchExport := []string{
- // batch 1 Summary
- fmt.Sprintf(grE2E2Batch1Summary,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(batchStartTime), conv(batchEndTime), conv(posts[0].CreateAt), conv(posts[1].CreateAt),
- // 9 10 11 12 13 14
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, users[1].Id,
- // 15 16
- posts[0].Id, posts[1].Id,
- ),
-
- // batch 1
- fmt.Sprintf(grE2E2Batch1,
- // 1 2 3 4 5
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime), conv(batchStartTime),
- // 6 7 8 9
- conv(batchEndTime), conv(cu[0]), conv(cu[1]), conv(jobStartTime),
- // 10 11 12 13 14 15
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, users[1].Id,
- // 16 17
- posts[0].Id, posts[1].Id,
- ),
- }
-
- data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
- // clean some bad csrf if present
- msg := global_relay_export.CleanTestOutput(data)
-
- // For debugging, better keep it in case we need it again.
- //t.Logf("<><>actual\n\n%s\n\n<><>batch1 Summary:\n\n%s\n\n<><>batch1:\n%s\n", msg, expectedBatchExport[0], expectedBatchExport[1])
-
- assert.Contains(t, msg, expectedBatchExport[0], "batch1 summary")
- assert.Contains(t, msg, expectedBatchExport[1], "batch1")
- })
-
- t.Run("CSV e2e 2 - post from user not in channel", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- ret := generateE2ETestType2Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- posts := ret.posts
- jl := ret.joinLeaves
- batches := ret.batches
- batchTimes := ret.batchTimes
- cu := ret.createUpdateTimes
- users := ret.users
- channels := ret.channels
- teams := ret.teams
- batchStartTime := batchTimes[0].start
-
- // to align with actiance export: (remember no close-out message on batch end)o
- // 2 participants entered
- // message 1
- // message 2
-
- expectedExport := fmt.Sprintf(csvE2E2Batch1,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10 11 12
- users[1].Id, posts[0].Id, posts[1].Id, jl[0].join, batchStartTime, cu[0], cu[1])
-
- export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
- assert.Equal(t, expectedExport, export)
- })
-
- t.Run("actiance e2e 3 - test create, update, delete xml fields", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret, type3Ret := generateE2ETestType3Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
- batches := ret.batches
- posts := ret.posts
- users := ret.users
- attachments := ret.attachments
- contents := ret.contents
-
- // actiance export:
- // message 0
- // message 1
- // message 1 deleted
- // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
- // message 3 created
- // file 3 upload start and stopped
- // message 3 deleted
- // file 3 deleted
- // message 4 -- same update at as below
- // edited message 4 -- same update at as above
- // message 6 -- same update at as below
- // edited message 6 -- same update at as above
-
- // 2 participants left
-
- for b, batchName := range batches {
- xmlContents := openZipAndReadFile(t, exportBackend, batchName, "actiance_export.xml")
-
- // Because 7 and 8 fall on the boundary, they could be in either batch, so save them here
- // and then test for one or the other in each batch.
- // The new post that got modified by message8
- message7 := &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[6].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[6].CreateAt,
- Message: posts[6].Message, // the original message
- UpdateAt: posts[6].UpdateAt, // the edit update at
- UpdatedType: shared.EditedOriginalMsg,
- EditedNewMsgId: posts[7].Id,
- }
- // The old post which has been edited
- message8 := &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[7].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[7].CreateAt,
- Message: posts[7].Message, // edited message
- UpdateAt: posts[7].UpdateAt,
- UpdatedType: shared.EditedNewMsg,
- }
-
- if b == 0 {
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><> xml contents: \n\n%s\n\n", xmlContents)
- exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages := exportedChannels[0].Messages
- require.Len(t, messages, 10) // batch size 7 + deleted msg1, deleted ms3, 1 deleted file msg
-
- // message 3's deleted file was uploaded in this batch period
- fileTransferStarted := exportedChannels[0].FileStarts
- require.Len(t, fileTransferStarted, 1)
- fileTransferStopped := exportedChannels[0].FileStops
- require.Len(t, fileTransferStopped, 1)
-
- // 0 - post create
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: posts[0].Message,
- }, messages[0])
-
- // 1 - post created
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: posts[1].Message,
- }, messages[1])
-
- // 1 - post deleted
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: "delete " + posts[1].Message,
- UpdateAt: type3Ret.message1DeleteAt,
- UpdatedType: shared.Deleted,
- }, messages[2])
-
- // 2 - post updated not edited (e.g., reaction)
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[2].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[2].CreateAt,
- Message: posts[2].Message,
- UpdateAt: type3Ret.updatedPost2.UpdateAt,
- UpdatedType: shared.UpdatedNoMsgChange,
- }, messages[3])
-
- // 3 - post created
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[3].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[3].CreateAt,
- Message: posts[3].Message,
- }, messages[4])
-
- // 3 - post deleted with a deleted file
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[3].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[3].CreateAt,
- Message: "delete " + posts[3].Message,
- UpdateAt: type3Ret.message3AndFileInfoDeleteAt,
- UpdatedType: shared.Deleted,
- }, messages[5])
-
- // file deleted message
- assert.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: messages[4].MessageId, // cheating bc we don't have this messageId
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[3].CreateAt,
- Message: "delete " + attachments[0].Path,
- UpdateAt: type3Ret.message3AndFileInfoDeleteAt,
- UpdatedType: shared.FileDeleted,
- }, messages[6])
-
- // the next messages 5, 6 can be in any order because all have equal `updateAt`s
- // 4 - original post
- equalUpdateAts := []*actiance_export.PostExport{
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[4].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[4].CreateAt,
- Message: posts[4].Message,
- UpdateAt: posts[4].UpdateAt, // will be the edit update at
- UpdatedType: shared.EditedOriginalMsg,
- EditedNewMsgId: posts[5].Id,
- },
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[5].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[5].CreateAt,
- Message: posts[5].Message,
- UpdateAt: posts[5].UpdateAt,
- UpdatedType: shared.EditedNewMsg,
- },
- }
- require.ElementsMatch(t, equalUpdateAts, []*actiance_export.PostExport{
- messages[7], messages[8]})
- require.ElementsMatch(t, []string{posts[4].Id, posts[5].Id}, []string{messages[7].MessageId, messages[8].MessageId})
-
- // the last message is one of the two edited or original
- if messages[9].MessageId == message7.MessageId {
- assert.Equal(t, message7, messages[9])
- } else {
- assert.Equal(t, message8, messages[9])
- }
-
- // only batch one has files, but they're deleted
- zipBytes, err := exportBackend.ReadFile(batchName)
- require.NoError(t, err)
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- attachmentInZip, err := zipReader.Open(attachments[0].Path)
- require.NoError(t, err)
- attachmentInZipContents, err := io.ReadAll(attachmentInZip)
- require.NoError(t, err)
- err = attachmentInZip.Close()
- require.NoError(t, err)
- assert.EqualValuesf(t, contents[0], string(attachmentInZipContents), "file contents not equal")
- }
-
- if b == 1 {
- exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages := exportedChannels[0].Messages
- require.Len(t, messages, 1)
-
- // check for either message 7 or message8
- if messages[0].MessageId == message7.MessageId {
- assert.Equal(t, message7, messages[0])
- } else {
- assert.Equal(t, message8, messages[0])
- }
- }
- }
- })
-
- t.Run("GlobalRelay e2e 3 - test create, update, delete fields", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret, type3Ret := generateE2ETestType3Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- jl := ret.joinLeaves
- posts := ret.posts
- batchTimes := ret.batchTimes
- jobStartTime := ret.start
- batches := ret.batches
- users := ret.users
- channels := ret.channels
- teams := ret.teams
-
- // to align with actiance export:
- // message 0
- // message 1
- // message 1 deleted
- // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
- // message 3 created
- // file 3 upload start and stopped
- // message 3 deleted
- // file 3 deleted
- // message 4 -- same update at as below
- // edited message 4 -- same update at as above
- // message 6 -- same update at as below
- // edited message 6 -- same update at as above
-
- // 2 participants left
-
- batchStartTime := batchTimes[0].start
- batchEndTime := batchTimes[0].end
-
- // The comments on 10, 11, 12 show the permutation -- this is needed because 10 & 11 have same UpdateAt,
- // and 12 & 1 (in batch 2) have same UpdateAt
- expectedBatch1Summaries := []string{
- // batch 1 Summary -- Permutation 1
- fmt.Sprintf(grE2E3Batch1SummaryPerm1,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt)),
-
- // batch 1 Summary -- Permutation 2
- fmt.Sprintf(grE2E3Batch1SummaryPerm2,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt)),
-
- // batch 1 summary - Permutation 3
- fmt.Sprintf(grE2E3Batch1SummaryPerm3,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt)),
-
- // batch 1 Summary -- Permutation 4
- fmt.Sprintf(grE2E3Batch1SummaryPerm4,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt)),
- }
-
- // The comments on 10, 11, 12 show the permutation -- this is needed because 10 & 11 have same UpdateAt,
- // and 12 & 1 (in batch 2) have same UpdateAt
- expectedBatch1 := []string{
- // batch 1 -- Permutation 1
- fmt.Sprintf(grE2E3Batch1Perm1,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt),
- ),
-
- // batch 1 -- Permutation 2
- fmt.Sprintf(grE2E3Batch1Perm2,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 edited 11 original
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[5].CreateAt), conv(posts[4].CreateAt),
- // 12 original
- conv(posts[6].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt),
- ),
-
- // batch 1 - Permutation 3
- fmt.Sprintf(grE2E3Batch1Perm3,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 original 11 edited
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[4].CreateAt), conv(posts[5].CreateAt),
- // 12 edited
- conv(posts[7].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt),
- ),
-
- // batch 1 -- Permutation 4
- fmt.Sprintf(grE2E3Batch1Perm4,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 9 10 edited 11 original
- conv(type3Ret.message3AndFileInfoDeleteAt), conv(posts[5].CreateAt), conv(posts[4].CreateAt),
- // 12 edited
- conv(posts[7].CreateAt),
- // 13 14 15 16 17
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 18 19 20 21 22 23 24 25
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id, posts[6].Id, posts[7].Id,
- // 26 27 28 message 4 orig/edited
- conv(type3Ret.message1DeleteAt), conv(type3Ret.updatedPost2.UpdateAt), conv(posts[4].UpdateAt),
- // 29 editedOriginal
- conv(posts[6].UpdateAt),
- ),
- }
-
- batchStartTime = batchTimes[1].start
- batchEndTime = batchTimes[1].end
-
- expectedBatch2Summaries := []string{
- // batch 2 Summary -- Permutation 1
- fmt.Sprintf(grE2E3Batch2SummaryPerm1,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 edited
- conv(posts[7].CreateAt),
- // 6 7 8 9 10 11
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[0].Id,
- // 12 13 14
- conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
- ),
-
- // batch 2 Summary -- Permutation 2
- fmt.Sprintf(grE2E3Batch2SummaryPerm2,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 original
- conv(posts[6].CreateAt),
- // 6 7 8 9 10 11
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
- // 12 13 14
- conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id),
- }
-
- expectedBatch2 := []string{
- // batch 2 Summary -- Permutation 1
- fmt.Sprintf(grE2E3Batch2Perm1,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 edited
- conv(posts[7].CreateAt),
- // 6 7 8 9 10 11
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
- // 12 13 14
- conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
- // 15
- conv(jobStartTime)),
-
- // batch 2 Summary -- Permutation 2
- fmt.Sprintf(grE2E3Batch2Perm2,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 original
- conv(posts[6].CreateAt),
- // 6 7 8 9 10 11
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id, posts[7].Id,
- // 12 13 14
- conv(posts[6].UpdateAt), posts[6].Id, posts[7].Id,
- // 15
- conv(jobStartTime)),
- }
-
- data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
- // clean some bad csrf if present
- msg := global_relay_export.CleanTestOutput(data)
-
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><>batch1 actual\n\n%s\n\n<><>batch1 Summary Perm1:\n\n%s\n\n<><>batch1 Summary Perm2:\n\n%s\n\n<><>batch1 Summary Perm3:\n\n%s\n\n<><>batch1 Summary Perm4:\n\n%s\n\n",
- // msg, expectedBatch1Summaries[0], expectedBatch1Summaries[1], expectedBatch1Summaries[2], expectedBatch1Summaries[3])
- //t.Logf("<><>batch1 actual\n\n%s\n\n<><>batch1 Perm1:\n\n%s\n\n<><>batch1 Perm2:\n\n%s\n\n<><>batch1 Perm3:\n\n%s\n\n<><>batch1 Perm4:\n\n%s\n\n",
- // msg, expectedBatch1[0], expectedBatch1[1], expectedBatch1[2], expectedBatch1[3])
-
- matched := dataContainsOneOfExpected(msg, expectedBatch1Summaries)
-
- assert.True(t, matched, "batch 1 summary didn't match one of the expected permutations")
-
- matched = dataContainsOneOfExpected(msg, expectedBatch1)
- assert.True(t, matched, "batch 1 body didn't match one of the expected permutations")
-
- data = openZipAndReadFileNum(t, exportBackend, batches[1], 0)
- // clean some bad csrf if present
- msg = global_relay_export.CleanTestOutput(data)
-
- // Note: for debugging, better keep this in case we need it again.
- //t.Logf("<><>batch2 actual\n\n%s\n\n<><>batch2 Summary Perm1:\n\n%s\n\n<><>batch2 Summary Perm2:\n\n%s\n\n",
- // msg, expectedBatch2Summaries[0], expectedBatch2Summaries[1])
- //t.Logf("<><>batch2 actual\n\n%s\n\n<><>batch2 Perm1:\n\n%s\n\n<><>batch2 Perm2:\n\n%s\n\n",
- // msg, expectedBatch2[0], expectedBatch2[1])
-
- matched = dataContainsOneOfExpected(msg, expectedBatch2Summaries)
- assert.True(t, matched, "batch 2 summary didn't match one of the expected permutations")
-
- matched = dataContainsOneOfExpected(msg, expectedBatch2)
- assert.True(t, matched, "batch 2 body didn't match one of the expected permutations")
- })
-
- t.Run("CSV e2e 3 - test create, update, delete fields", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret, ret3 := generateE2ETestType3Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- jl := ret.joinLeaves
- posts := ret.posts
- //cu := ret.createUpdateTimes
- //jobStartTime := ret.start
- batches := ret.batches
- users := ret.users
- channels := ret.channels
- teams := ret.teams
- attachments := ret.attachments
-
- // aligned with actiance export:
- // message 0
- // message 1
- // message 1 deleted
- // message 2 updated (reaction): post2 createdAt, updatedPost2 updateAt
- // message 3 created
- // file 3 upload start and stopped
- // message 3 deleted
- // file 3 deleted
- // message 4 -- same update at as below
- // edited message 4 -- same update at as above
- // message 6 -- same update at as below
- // edited message 6 -- same update at as above
-
- // NOTE: the comments describe the order of the last three messages (21 22 23),
- // eg in perm1: message 4, edited message 4, and message 6
- expectedExports := []string{
- // original, edited, "original" msg 6
- fmt.Sprintf(csvE2E3Batch1Perm1,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10 11 edited
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 12 13
- posts[6].Id, attachments[0].Id,
- // 14 15 16 17 18
- jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
- // 19 20 21 22 23
- posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
- // 24 editedBy for message 6, 25 26
- posts[7].Id, ret3.updatedPost2.UpdateAt, posts[6].UpdateAt),
-
- // edited, original, original
- fmt.Sprintf(csvE2E3Batch1Perm2,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10 11 edited
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 12 13
- posts[6].Id, attachments[0].Id,
- // 14 15 16 17 18
- jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
- // 19 20 21 22 23
- posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
- // 24 editedBy for message 6, 25 26
- posts[7].Id, ret3.updatedPost2.UpdateAt, posts[6].UpdateAt),
-
- // original, edited, edited
- fmt.Sprintf(csvE2E3Batch1Perm3,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10 11 edited
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 12 id of edited msg6 13
- posts[7].Id, attachments[0].Id,
- // 14 15 16 17 18
- jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
- // 19 20 21 22 23
- posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt, posts[6].CreateAt,
- // 24 25 26
- ret3.updatedPost2.UpdateAt, posts[6].CreateAt, posts[6].UpdateAt),
-
- // edited, original, edited
- fmt.Sprintf(csvE2E3Batch1Perm4,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10 11 edited
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id, posts[5].Id,
- // 12 id of edited msg6 13
- posts[7].Id, attachments[0].Id,
- // 14 15 16 17 18
- jl[0].join, posts[0].CreateAt, posts[1].CreateAt, ret3.message1DeleteAt, posts[2].CreateAt,
- // 19 20 21 22
- posts[3].CreateAt, ret3.message3AndFileInfoDeleteAt, posts[4].CreateAt, posts[4].UpdateAt,
- // 23 24 25 26
- posts[6].CreateAt, ret3.updatedPost2.UpdateAt, posts[6].CreateAt, posts[6].UpdateAt),
- }
-
- export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
-
- matched := slices.Contains(expectedExports, export)
- assert.True(t, matched, "batch 1 didn't match one of the expected permutations")
-
- expectedExports = []string{
- // original message 6 (which says "message 6" in the message, and has new id -- which is post 6's id)
- fmt.Sprintf(csvE2E3Batch2Perm1,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6
- posts[6].Id,
- // 7 this is the editedBy for message 6 -- editedBy is confusing (cause it's the original id), but it is what it is
- posts[7].Id,
- // 8 9 10
- posts[6].CreateAt, jl[0].join, posts[6].UpdateAt),
-
- // edited message 6 (which says "edited message 6" in the message, and has original id)
- fmt.Sprintf(csvE2E3Batch2Perm2,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9
- posts[7].Id, posts[6].CreateAt, jl[0].join, posts[6].UpdateAt),
- }
-
- export = openZipAndReadFileNum(t, exportBackend, batches[1], 0)
-
- matched = slices.Contains(expectedExports, export)
- assert.True(t, matched, "batch 2 didn't match one of the expected permutations")
- })
-
- t.Run("actiance e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
- batch001 := ret.batches[0]
- posts := ret.posts
- users := ret.users
-
- xmlContents := openZipAndReadFile(t, exportBackend, batch001, "actiance_export.xml")
-
- exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages := exportedChannels[0].Messages
- require.Len(t, messages, 5)
-
- // the messages can be in any order because all have equal `updateAt`s
- equalUpdateAts := []*actiance_export.PostExport{
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: posts[0].Message,
- UpdateAt: posts[1].UpdateAt, // the edit update at
- UpdatedType: shared.EditedOriginalMsg,
- EditedNewMsgId: posts[1].Id,
- },
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: posts[1].Message,
- UpdateAt: posts[1].UpdateAt,
- UpdatedType: shared.EditedNewMsg,
- },
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[2].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[2].CreateAt,
- Message: posts[2].Message,
- },
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[3].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[3].CreateAt,
- Message: posts[3].Message,
- },
- {
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[4].Id,
- UserEmail: users[0].Email,
- UserType: "user",
- CreateAt: posts[4].CreateAt,
- Message: posts[4].Message,
- },
- }
- require.ElementsMatch(t, equalUpdateAts, []*actiance_export.PostExport{
- messages[0], messages[1], messages[2], messages[3], messages[4]})
- })
-
- t.Run("GlobalRelay e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- jl := ret.joinLeaves
- posts := ret.posts
- batchTimes := ret.batchTimes
- //jobStartTime := ret.start
- batches := ret.batches
- users := ret.users
- channels := ret.channels
- teams := ret.teams
-
- batchStartTime := batchTimes[0].start
- batchEndTime := batchTimes[0].end
-
- // summaryHeader
- allExpected := []string{fmt.Sprintf(grE2E4Summary,
- // 1 2 3 4
- conv(batchStartTime), conv(batchEndTime), conv(jl[0].join), conv(batchEndTime),
- // 5 6 7 8 9
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 10 11 12 13
- conv(posts[0].CreateAt), conv(posts[1].CreateAt), conv(posts[2].CreateAt), conv(posts[3].CreateAt),
- // 14 15 16 17 18 19
- conv(posts[4].CreateAt), posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id,
- )}
-
- // summary body
- allExpected = append(allExpected,
- fmt.Sprintf("%[1]s %[2]s @user1 %[3]s @user1 user (user1@email) edited message 0 EditedNewMsg %[4]s",
- posts[1].Id, conv(posts[0].CreateAt), users[0].Id, conv(posts[0].UpdateAt)),
- fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 0 EditedOriginalMsg %[4]s %[5]s",
- posts[0].Id, conv(posts[1].CreateAt), users[0].Id, conv(posts[0].UpdateAt), posts[1].Id),
- fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 2",
- posts[2].Id, conv(posts[2].CreateAt), users[0].Id),
- fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 3",
- posts[3].Id, conv(posts[3].CreateAt), users[0].Id),
- fmt.Sprintf("* %[1]s %[2]s @user1 %[3]s @user1 user (user1@email) message 4",
- posts[4].Id, conv(posts[4].CreateAt), users[0].Id),
- )
-
- // first two are channel and participants, rest are messages
- allExpected = append(allExpected,
- fmt.Sprintf("\n
\n - TeamId:%[1]s
\n - TeamName:%[2]s
\n - TeamDisplayName:%[3]s
\n - ChannelId:%[4]s
\n - ChannelName:channel_two_name
\n - ChannelDisplayName:the Channel Two
\n - Started:%[5]s
\n - Ended:%[6]s
\n - Duration:0 seconds
\n
\n
%[1]s | \n @user1 | \n user | \n user1@email | \n %[2]s | \n %[3]s | \n 0 seconds | \n 5 | ",
- users[0].Id, conv(jl[0].join), conv(batchEndTime)),
- fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n edited message 0\n EditedNewMsg\n %[4]s\n ",
- posts[1].Id, conv(posts[1].CreateAt), users[0].Id, conv(posts[1].UpdateAt)),
- fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 0\n EditedOriginalMsg\n %[4]s\n %[5]s",
- posts[0].Id, conv(posts[0].CreateAt), users[0].Id, conv(posts[0].UpdateAt), posts[1].Id),
- fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 2",
- posts[2].Id, conv(posts[2].CreateAt), users[0].Id, conv(posts[2].UpdateAt)),
- fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 3",
- posts[3].Id, conv(posts[3].CreateAt), users[0].Id, conv(posts[3].UpdateAt)),
- fmt.Sprintf("%[1]s\n %[2]s\n @user1\n %[3]s\n @user1\n user\n (user1@email)\n message 4",
- posts[4].Id, conv(posts[4].CreateAt), users[0].Id, conv(posts[4].UpdateAt)),
- )
-
- data := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
- // clean some bad csrf if present
- msg := global_relay_export.CleanTestOutput(data)
-
- for _, expected := range allExpected {
- assert.Contains(t, msg, expected, "expected exported msg to contain: \n%s\n\nExported msg:\n%s\n", expected, msg)
- if !strings.Contains(msg, expected) {
- break
- }
- }
- })
-
- t.Run("CSV e2e 4 - test edits with multiple simultaneous updates", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- ret := generateE2ETestType4Results(t, th, model.ComplianceExportTypeCsv, attachmentDir,
- exportDir, attachmentBackend, exportBackend)
- jl := ret.joinLeaves
- posts := ret.posts
- batches := ret.batches
- users := ret.users
- channels := ret.channels
- teams := ret.teams
-
- // fill out the lines using the template (easier to read than if it were an inline string)
- tmpl := fmt.Sprintf(csvE2E4Batch1,
- // 1 2 3 4 5
- teams[0].Id, teams[0].Name, teams[0].DisplayName, channels[0].Id, users[0].Id,
- // 6 7 8 9 10
- posts[0].Id, posts[1].Id, posts[2].Id, posts[3].Id, posts[4].Id,
- // 11 12 13 14
- jl[0].join, posts[0].CreateAt, posts[2].CreateAt, posts[1].UpdateAt)
-
- allExpected := strings.Split(tmpl, "\n")
-
- export := openZipAndReadFileNum(t, exportBackend, batches[0], 0)
-
- assert.Len(t, strings.Split(export, "\n"), len(allExpected)+1) // +1 for header line
-
- for _, expected := range allExpected {
- assert.Contains(t, export, expected, "expected export to contain: \n%s\n\nExport:\n%s\n", expected, export)
- if !strings.Contains(export, expected) {
- break
- }
- }
- })
-
- t.Run("actiance e2e 5 - test delete and update semantics", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
-
- rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeActiance, attachmentDir, exportDir, attachmentBackend, exportBackend)
- posts := rets[0].posts
- message0DeleteAt := ret5s[0].message0DeleteAt
- zipBytes := ret5s[0].zipBytes[0]
-
- //
- // Job 1
- //
- xmlContents := readFileFromZip(t, zipBytes, "actiance_export.xml")
-
- exportedChannels := actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages := exportedChannels[0].Messages
- require.Len(t, messages, 2) // 1 posted, 1 deleted
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: posts[0].Message,
- }, messages[0])
-
- // 1 - post deleted
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: "delete " + posts[0].Message,
- UpdateAt: message0DeleteAt,
- UpdatedType: shared.Deleted,
- }, messages[1])
-
- //
- // Job 2
- //
- posts = rets[1].posts
- message0DeleteAt = ret5s[1].message0DeleteAt
- zipBytes = ret5s[1].zipBytes[0]
- zipBytes2 := ret5s[1].zipBytes[1]
-
- xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
-
- exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages = exportedChannels[0].Messages
- require.Len(t, messages, 1) // 1 posted
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: posts[1].Message,
- }, messages[0])
-
- xmlContents = readFileFromZip(t, zipBytes2, "actiance_export.xml")
-
- exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages = exportedChannels[0].Messages
- require.Len(t, messages, 2) // message0's create and delete messages
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: posts[0].Message,
- }, messages[0])
-
- // 1 - post deleted
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: "delete " + posts[0].Message,
- UpdateAt: message0DeleteAt,
- UpdatedType: shared.Deleted,
- }, messages[1])
-
- //
- // Job 3
- //
- posts = rets[2].posts
- zipBytes = ret5s[2].zipBytes[0]
-
- xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
-
- exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages = exportedChannels[0].Messages
- require.Len(t, messages, 2) // 2 posted
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: posts[0].Message,
- }, messages[0])
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: posts[1].Message,
- }, messages[1])
-
- //
- // Job 4
- //
- posts = rets[3].posts
- updatedPost1 := ret5s[3].updatedPost1
- message0DeleteAt = ret5s[3].message0DeleteAt
- zipBytes = ret5s[3].zipBytes[0]
-
- xmlContents = readFileFromZip(t, zipBytes, "actiance_export.xml")
-
- exportedChannels = actiance_export.GetChannelExports(t, strings.NewReader(xmlContents))
- assert.Len(t, exportedChannels, 1)
- messages = exportedChannels[0].Messages
- require.Len(t, messages, 3) //filler post, deleted post, and updated posts ONLY
-
- // post created
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[2].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[2].CreateAt,
- Message: posts[2].Message,
- }, messages[0])
-
- // post deleted ONLY (not its created post, because that was in the previous job)
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[0].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[0].CreateAt,
- Message: "delete " + posts[0].Message,
- UpdateAt: message0DeleteAt,
- UpdatedType: shared.Deleted,
- }, messages[1])
-
- // post updated ONLY (not its created post, because that was in the previous job)
- require.Equal(t, &actiance_export.PostExport{
- XMLName: xml.Name{Local: "Message"},
- MessageId: posts[1].Id,
- UserEmail: th.BasicUser.Email,
- UserType: "user",
- CreateAt: posts[1].CreateAt,
- Message: posts[1].Message,
- UpdateAt: updatedPost1.UpdateAt,
- UpdatedType: shared.UpdatedNoMsgChange,
- }, messages[2])
- })
-
- t.Run("GlobalRelay e2e 5 - test delete and update semantics", func(t *testing.T) {
- // regMsg has strings in pos: 1: post_id, 2: sent_time, 3: username, 4: userId, 5: email, 6: message
- regMsgTmpl := "\n %[1]s\n %[2]s\n @%[3]s\n %[4]s\n @%[3]s\n user\n (%[5]s)\n %[6]s\n"
- // updatedMsgTmpl has strings in pos: 1: post_id, 2: sent_time, 3: username, 4: userId, 5: email, 6: message, 7: update_type, 8: update_time, 9: edited_new_msg_id
- updatedMsgTmpl := "\n %[1]s\n %[2]s\n @%[3]s\n %[4]s\n @%[3]s\n user\n (%[5]s)\n %[6]s\n %[7]s\n %[8]s\n %[9]s\n\n"
-
- assertContainsAllMsgs := func(msg string, allExpected []string, tag string) {
- for _, expected := range allExpected {
- assert.Contains(t, msg, expected, "%s, expected exported msg to contain: \n%s\n\nExported msg:\n%s\n", tag, expected, msg)
- if !strings.Contains(msg, expected) {
- break
- }
- }
-
- numMessages := 0
- for l := range strings.SplitSeq(msg, "\n") {
- if strings.Contains(l, "") {
- numMessages += 1
- }
- }
-
- assert.Equal(t, len(allExpected), numMessages, tag)
- }
-
- th := setup(t)
- defer th.TearDown()
-
- rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeGlobalrelayZip, attachmentDir, exportDir, attachmentBackend, exportBackend)
- posts := rets[0].posts
- zipBytes := ret5s[0].zipBytes[0]
-
- //
- // Job 1
- //
- data := readFilenumFromZip(t, zipBytes, 0)
- // clean some bad csrf if present
- msg := global_relay_export.CleanTestOutput(data)
- message0DeleteAt := ret5s[0].message0DeleteAt
-
- // post created
- allExpected := []string{
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
- // 1 2 3 4 5 6 7 8 9
- fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
- }
- assertContainsAllMsgs(msg, allExpected, "job 1")
-
- //
- // Job 2
- //
- posts = rets[1].posts
- message0DeleteAt = ret5s[1].message0DeleteAt
- zipBytes = ret5s[1].zipBytes[0]
- zipBytes2 := ret5s[1].zipBytes[1]
-
- data = readFilenumFromZip(t, zipBytes, 0)
- // clean some bad csrf if present
- msg = global_relay_export.CleanTestOutput(data)
-
- // post created
- allExpected = []string{
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message),
- }
- assertContainsAllMsgs(msg, allExpected, "job 2a")
-
- data = readFilenumFromZip(t, zipBytes2, 0)
- // clean some bad csrf if present
- msg = global_relay_export.CleanTestOutput(data)
-
- allExpected = []string{
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
- // 1 2 3 4 5 6 7 8 9
- fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
- }
- assertContainsAllMsgs(msg, allExpected, "job 2b")
-
- //
- // Job 3
- //
- posts = rets[2].posts
- zipBytes = ret5s[2].zipBytes[0]
-
- data = readFilenumFromZip(t, zipBytes, 0)
- // clean some bad csrf if present
- msg = global_relay_export.CleanTestOutput(data)
-
- // post created
- allExpected = []string{
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[0].Message),
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message),
- }
- assertContainsAllMsgs(msg, allExpected, "job 3")
-
- //
- // Job 4
- //
- posts = rets[3].posts
- updatedPost1 := ret5s[3].updatedPost1
- message0DeleteAt = ret5s[3].message0DeleteAt
- zipBytes = ret5s[3].zipBytes[0]
-
- data = readFilenumFromZip(t, zipBytes, 0)
- // clean some bad csrf if present
- msg = global_relay_export.CleanTestOutput(data)
-
- allExpected = []string{
- // post created
- // 1 2 3 4 5 6
- fmt.Sprintf(regMsgTmpl, posts[2].Id, conv(posts[2].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[2].Message),
-
- // post deleted ONLY (not its created post, because that was in the previous job)
- // 1 2 3 4 5 6 7 8 9
- fmt.Sprintf(updatedMsgTmpl, posts[0].Id, conv(posts[0].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, "delete "+posts[0].Message, shared.Deleted, conv(message0DeleteAt), ""),
-
- // post updated ONLY (not its created post, because that was in the previous job)
- // 1 2 3 4 5 6 7 8 9
- fmt.Sprintf(updatedMsgTmpl, posts[1].Id, conv(posts[1].CreateAt), th.BasicUser.Username, th.BasicUser.Id, th.BasicUser.Email, posts[1].Message, shared.UpdatedNoMsgChange, conv(updatedPost1.UpdateAt), ""),
- }
- assertContainsAllMsgs(msg, allExpected, "job 4")
- })
-
- t.Run("CSV e2e 5 - test delete and update semantics", func(t *testing.T) {
- type msgDetailsToCheck struct {
- createAt int64
- updateAt int64
- updatedType shared.PostUpdatedType
- postId string
- message string
- }
-
- assertContainsAllMsgs := func(msg string, allExpected []msgDetailsToCheck, tag string) {
- allLines := strings.Split(strings.Trim(msg, " \n"), "\n")
- allLines = allLines[1:] // remove header
- var msgLines []string
- for _, l := range allLines {
- if !strings.Contains(l, "previously-joined") {
- msgLines = append(msgLines, l)
- }
- }
-
- assert.Equal(t, len(allExpected), len(msgLines), tag)
-
- for _, expected := range allExpected {
- found := false
- for _, msg := range msgLines {
- if strings.HasPrefix(msg, fmt.Sprintf("%d,%d,%s,",
- expected.createAt, expected.updateAt, expected.updatedType)) &&
- strings.Contains(msg, expected.postId+",,,"+expected.message) {
- found = true
- break
- }
- }
- assert.True(t, found, "%s actual msg did not contain expected msg. msg: \n%s\nexpected msg details: %v\n", tag, msg, expected)
- }
- }
-
- th := setup(t)
- defer th.TearDown()
-
- rets, ret5s := generateE2ETestType5Results(t, th, model.ComplianceExportTypeCsv, attachmentDir, exportDir, attachmentBackend, exportBackend)
- posts := rets[0].posts
- message0DeleteAt := ret5s[0].message0DeleteAt
- zipBytes := ret5s[0].zipBytes[0]
-
- // NOTE: we know csv outputs correctly from above, so just test that the right post IDs are being exported
-
- //
- // Job 1
- //
- export := readFilenumFromZip(t, zipBytes, 0)
-
- // post created, post deleted
- assertContainsAllMsgs(export, []msgDetailsToCheck{
- {
- createAt: posts[0].CreateAt,
- updateAt: message0DeleteAt,
- postId: posts[0].Id,
- message: posts[0].Message,
- },
- {
- createAt: posts[0].CreateAt,
- updateAt: message0DeleteAt,
- updatedType: shared.Deleted,
- postId: posts[0].Id,
- message: "delete " + posts[0].Message,
- },
- }, "Job 1")
-
- //
- // Job 2
- //
- posts = rets[1].posts
- message0DeleteAt = ret5s[1].message0DeleteAt
- zipBytes = ret5s[1].zipBytes[0]
- zipBytes2 := ret5s[1].zipBytes[1]
-
- export = readFilenumFromZip(t, zipBytes, 0)
-
- // post created
- assertContainsAllMsgs(export, []msgDetailsToCheck{
- {
- createAt: posts[1].CreateAt,
- updateAt: posts[1].UpdateAt,
- postId: posts[1].Id,
- message: posts[1].Message,
- },
- }, "Job 2 batch 1")
-
- export = readFilenumFromZip(t, zipBytes2, 0)
-
- // post created
- assertContainsAllMsgs(export, []msgDetailsToCheck{
- {
- createAt: posts[0].CreateAt,
- updateAt: message0DeleteAt,
- postId: posts[0].Id,
- message: posts[0].Message,
- },
- {
- createAt: posts[0].CreateAt,
- updateAt: message0DeleteAt,
- updatedType: shared.Deleted,
- postId: posts[0].Id,
- message: "delete " + posts[0].Message,
- },
- }, "Job 2 batch 2")
-
- // Job 3
- //
- posts = rets[2].posts
- zipBytes = ret5s[2].zipBytes[0]
-
- export = readFilenumFromZip(t, zipBytes, 0)
-
- // 2 posts created
- assertContainsAllMsgs(export, []msgDetailsToCheck{
- {
- createAt: posts[0].CreateAt,
- updateAt: posts[0].UpdateAt,
- postId: posts[0].Id,
- message: posts[0].Message,
- },
- {
- createAt: posts[1].CreateAt,
- updateAt: posts[1].UpdateAt,
- postId: posts[1].Id,
- message: posts[1].Message,
- },
- }, "Job 3")
-
- //
- // Job 4
- //
- posts = rets[3].posts
- updatedPost1 := ret5s[3].updatedPost1
- message0DeleteAt = ret5s[3].message0DeleteAt
- zipBytes = ret5s[3].zipBytes[0]
-
- export = readFilenumFromZip(t, zipBytes, 0)
-
- // post created
- assertContainsAllMsgs(export, []msgDetailsToCheck{
- {
- createAt: posts[2].CreateAt,
- updateAt: posts[2].UpdateAt,
- postId: posts[2].Id,
- message: posts[2].Message,
- },
- // post deleted ONLY (not its created post, because that was in the previous job)
- {
- createAt: posts[0].CreateAt,
- updateAt: message0DeleteAt,
- updatedType: shared.Deleted,
- postId: posts[0].Id,
- message: "delete " + posts[0].Message,
- },
- // post updated ONLY (not its created post, because that was in the previous job)
- {
- createAt: posts[1].CreateAt,
- updateAt: updatedPost1.UpdateAt,
- updatedType: shared.UpdatedNoMsgChange,
- postId: posts[1].Id,
- message: posts[1].Message,
- },
- }, "Job 4")
- })
-
- t.Run("csv -- multiple batches, 1 zip per batch, output to a single directory", func(t *testing.T) {
- th := setup(t)
- defer th.TearDown()
- defer func() {
- err := os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- }()
-
- time.Sleep(1 * time.Millisecond)
- now := model.GetMillis()
-
- jobStart := now - 1
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = jobStart
- *cfg.MessageExportSettings.BatchSize = 5
- *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeCsv
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- attachmentContent := "Hello there"
- attachmentPath001 := "path/to/attachments/one.txt"
- _, _ = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath001)
- post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now,
- UpdateAt: now,
- FileIds: []string{"test1"},
- })
- require.NoError(t, err)
-
- attachment, err := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
- Id: st.NewTestID(),
- CreatorId: post.UserId,
- PostId: post.Id,
- CreateAt: now,
- UpdateAt: now,
- Path: attachmentPath001,
- })
- require.NoError(t, err)
-
- for i := range 10 {
- _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now + int64(i),
- UpdateAt: now + int64(i),
- })
- require.NoError(t, e)
- }
-
- job := runJobForTest(t, th, nil)
-
- warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
- require.NoError(t, err)
- require.Equal(t, 0, warnings)
-
- numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
- require.NoError(t, err)
- require.Equal(t, int64(11), numExported)
- jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
- require.NoError(t, err)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- batch001 := shared.GetBatchPath(jobExportDir, jobStart, now+3, 1)
- batch002 := shared.GetBatchPath(jobExportDir, now+3, now+8, 2)
- batch003 := shared.GetBatchPath(jobExportDir, now+8, jobEnd, 3)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- require.ElementsMatch(t, []string{batch001, batch002, batch003}, files)
-
- zipBytes, err := exportBackend.ReadFile(batch001)
- require.NoError(t, err)
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- csvZipFilePath := path.Join("files", post.Id, fmt.Sprintf("%s-%s", attachment.Id, path.Base(attachment.Path)))
-
- attachmentInZip, err := zipReader.Open(csvZipFilePath)
- require.NoError(t, err)
- attachmentInZipContents, err := io.ReadAll(attachmentInZip)
- require.NoError(t, err)
- err = attachmentInZip.Close()
- require.NoError(t, err)
-
- require.EqualValuesf(t, attachmentContent, string(attachmentInZipContents), "file contents not equal")
- })
-}
-
-func openZipAndReadFile(t *testing.T, backend filestore.FileBackend, path string, filename string) string {
- zipBytes, err := backend.ReadFile(path)
- require.NoError(t, err)
- return readFileFromZip(t, zipBytes, filename)
-}
-
-func readFileFromZip(t *testing.T, zipBytes []byte, filename string) string {
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- file, err := zipReader.Open(filename)
- require.NoError(t, err)
- contents, err := io.ReadAll(file)
- require.NoError(t, err)
- err = file.Close()
- require.NoError(t, err)
-
- return string(contents)
-}
-
-func openZipAndReadFileNum(t *testing.T, backend filestore.FileBackend, path string, fileNum int) string {
- zipBytes, err := backend.ReadFile(path)
- require.NoError(t, err)
- return readFilenumFromZip(t, zipBytes, fileNum)
-}
-
-func openZipAndReadFileStartingWith(t *testing.T, backend filestore.FileBackend, path string, startsWith string) string {
- zipBytes, err := backend.ReadFile(path)
- require.NoError(t, err)
-
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- var names []string
- for _, f := range zipReader.File {
- if strings.HasPrefix(f.Name, startsWith) {
- file, err := f.Open()
- require.NoError(t, err)
- contents, err := io.ReadAll(file)
- require.NoError(t, err)
- err = file.Close()
- require.NoError(t, err)
-
- return string(contents)
- }
- names = append(names, f.Name)
- }
-
- require.True(t, false, "called openZipAndReadFileStartingWith but didn't file file starting with: %s. Found: %v", startsWith, names)
- return ""
-}
-
-func readFilenumFromZip(t *testing.T, zipBytes []byte, fileNum int) string {
- zipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
- require.NoError(t, err)
-
- file, err := zipReader.File[fileNum].Open()
- require.NoError(t, err)
- contents, err := io.ReadAll(file)
- require.NoError(t, err)
- err = file.Close()
- require.NoError(t, err)
-
- return string(contents)
-}
-
-func dataContainsOneOfExpected(data string, expected []string) bool {
- for _, perm := range expected {
- if strings.Contains(data, perm) {
- return true
- }
- }
- return false
-}
diff --git a/server/enterprise/message_export/message_export_test_e2e_generators.go b/server/enterprise/message_export/message_export_test_e2e_generators.go
deleted file mode 100644
index c9781bd27c9..00000000000
--- a/server/enterprise/message_export/message_export_test_e2e_generators.go
+++ /dev/null
@@ -1,1551 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "bytes"
- "context"
- "fmt"
- "os"
- "strconv"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/v8/channels/api4"
- "github.com/mattermost/mattermost/server/v8/channels/jobs"
- st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
-)
-
-type MessageExport struct {
- TeamId string
- TeamName string
- TeamDisplayName string
-
- ChannelId string
- ChannelName string
- ChannelDisplayName string
- ChannelType model.ChannelType
-
- UserId string
- UserEmail string
- Username string
- IsBot bool
-
- PostId string
- PostCreateAt int64
- PostUpdateAt int64
- PostDeleteAt int64
- PostEditAt int64
- PostMessage string
- PostType string
- PostRootId string
- PostProps string
- PostOriginalId string
- PostFileIds []string
-}
-
-func removePointers(p *model.MessageExport) MessageExport {
- return MessageExport{
- TeamId: model.SafeDereference(p.TeamId),
- TeamName: model.SafeDereference(p.TeamName),
- TeamDisplayName: model.SafeDereference(p.TeamDisplayName),
- ChannelId: model.SafeDereference(p.ChannelId),
- ChannelName: model.SafeDereference(p.ChannelName),
- ChannelDisplayName: model.SafeDereference(p.ChannelDisplayName),
- ChannelType: model.SafeDereference(p.ChannelType),
- UserId: model.SafeDereference(p.UserId),
- UserEmail: model.SafeDereference(p.UserEmail),
- Username: model.SafeDereference(p.Username),
- IsBot: p.IsBot,
- PostId: model.SafeDereference(p.PostId),
- PostCreateAt: model.SafeDereference(p.PostCreateAt),
- PostUpdateAt: model.SafeDereference(p.PostUpdateAt),
- PostDeleteAt: model.SafeDereference(p.PostDeleteAt),
- PostEditAt: model.SafeDereference(p.PostEditAt),
- PostMessage: model.SafeDereference(p.PostMessage),
- PostType: model.SafeDereference(p.PostType),
- PostRootId: model.SafeDereference(p.PostRootId),
- PostProps: model.SafeDereference(p.PostProps),
- PostOriginalId: model.SafeDereference(p.PostOriginalId),
- PostFileIds: p.PostFileIds,
- }
-}
-
-// assertNumPostsToExport checks both the MessageExport and the AnalyticsPostCount -- they were sometimes giving
-// different numbers and this helps debug.
-func assertNumPostsToExport(t *testing.T, th *api4.TestHelper, num int, since, until int64) {
- exports, _, err := th.App.Srv().Store().Compliance().MessageExport(th.Context, model.MessageExportCursor{
- LastPostUpdateAt: since, UntilUpdateAt: until,
- }, 100)
- assert.NoError(t, err)
- assert.Len(t, exports, num)
- assert.Lenf(t, exports, num, "MessageExport posts found, since %d, th.BasicChannel.Id: %s\n", since, th.BasicChannel.Id)
- if len(exports) != num {
- for _, p := range exports {
- t.Logf("%#+v\n", removePointers(p))
- }
- }
-
- count, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeSystemPosts: true, UsersPostsOnly: true, SincePostID: "", SinceUpdateAt: since})
- require.NoError(t, err)
- assert.Equal(t, num, int(count))
-}
-
-func assertNumExported(t *testing.T, expectedNum int, data map[string]string) {
- numExported, err := strconv.Atoi(data[shared.JobDataMessagesExported])
- require.NoError(t, err)
- assert.Equalf(t, numExported, expectedNum, "\njobData: %v\n", data)
- require.Equal(t, expectedNum, numExported)
-}
-
-func getMostRecentJobWithId(t *testing.T, th *api4.TestHelper, id string) *model.Job {
- list, _, err := th.SystemAdminClient.GetJobsByType(context.Background(), "message_export", 0, 1)
- require.NoError(t, err)
- require.Len(t, list, 1)
- require.Equal(t, id, list[0].Id)
- return list[0]
-}
-
-func checkJobForStatus(t *testing.T, th *api4.TestHelper, id string, status string) {
- doneChan := make(chan bool)
- var job *model.Job
- go func() {
- defer close(doneChan)
- for {
- job = getMostRecentJobWithId(t, th, id)
- if job.Status == status {
- break
- }
- time.Sleep(100 * time.Millisecond)
- }
- require.Equal(t, status, job.Status)
- }()
- select {
- case <-doneChan:
- case <-time.After(15 * time.Second):
- require.True(t, false, "expected job's status to be %s, got %s", status, job.Status)
- }
-}
-
-func runJobForTest(t *testing.T, th *api4.TestHelper, jobData map[string]string) *model.Job {
- job, _, err := th.SystemAdminClient.CreateJob(context.Background(),
- &model.Job{Type: "message_export", Data: jobData})
- require.NoError(t, err)
- // poll until completion
- checkJobForStatus(t, th, job.Id, "success")
- job = getMostRecentJobWithId(t, th, job.Id)
- return job
-}
-
-func setup(t *testing.T) *api4.TestHelper {
- jobs.DefaultWatcherPollingInterval = 100
- th := api4.SetupEnterprise(t).InitBasic()
- th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
- messageExportImpl := MessageExportJobInterfaceImpl{th.App.Srv()}
- th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
-
- err := th.App.Srv().Jobs.StartWorkers()
- require.NoError(t, err)
-
- err = th.App.Srv().Jobs.StartSchedulers()
- require.NoError(t, err)
-
- return th
-}
-
-// jobDataInvariantsShouldBeEqual tests that the parts of the job.Data that shouldn't change, don't change.
-func jobDataInvariantsShouldBeEqual(t *testing.T, expected map[string]string, received map[string]string) {
- assert.Equal(t, expected[shared.JobDataExportType], received[shared.JobDataExportType])
- assert.Equal(t, expected[shared.JobDataBatchSize], received[shared.JobDataBatchSize])
- assert.Equal(t, expected[shared.JobDataChannelBatchSize], received[shared.JobDataChannelBatchSize])
- assert.Equal(t, expected[shared.JobDataChannelHistoryBatchSize], received[shared.JobDataChannelHistoryBatchSize])
- assert.Equal(t, expected[shared.JobDataExportDir], received[shared.JobDataExportDir])
- assert.Equal(t, expected[shared.JobDataJobEndTime], received[shared.JobDataJobEndTime])
- assert.Equal(t, expected[shared.JobDataJobStartTime], received[shared.JobDataJobStartTime])
-}
-
-type joinLeave struct {
- join int64
- leave int64
-}
-type batchStartEndTimes struct {
- start int64
- end int64
-}
-
-type JobResults struct {
- start int64
- joinLeaves []joinLeave
- users []*model.User
- createUpdateTimes []int64
- attachments []*model.FileInfo
- contents []string
- jobEndTime int64
- batches []string
- posts []*model.Post
- channels []*model.Channel
- teams []*model.Team
- batchTimes []batchStartEndTimes
- jobExportDir string
-}
-
-func generateActianceBatchTest1(t *testing.T, th *api4.TestHelper, attachmentDir, exportDir string,
- attachmentBackend filestore.FileBackend) JobResults {
- now := model.GetMillis()
- jobStart := now - 1
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = jobStart
- *cfg.MessageExportSettings.BatchSize = 5
- *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeActiance
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- contents := []string{"Hello there"}
- attachmentPath001 := "path/to/attachments/one.txt"
- _, _ = attachmentBackend.WriteFile(bytes.NewBufferString(contents[0]), attachmentPath001)
- post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now,
- UpdateAt: now,
- FileIds: []string{"test1"},
- })
- require.NoError(t, err)
-
- attachment, err := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
- Id: st.NewTestID(),
- CreatorId: post.UserId,
- PostId: post.Id,
- CreateAt: now,
- UpdateAt: now,
- Path: attachmentPath001,
- })
- require.NoError(t, err)
- attachments := []*model.FileInfo{attachment}
-
- for i := range 10 {
- _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now + int64(i),
- UpdateAt: now + int64(i),
- })
- require.NoError(t, e)
- }
-
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 11, jobStart, until)
-
- job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
- require.NoError(t, err)
- require.Equal(t, 0, warnings)
-
- numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
- require.NoError(t, err)
- require.Equal(t, int64(11), numExported)
-
- jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
- require.NoError(t, err)
- jobExportDir := job.Data[shared.JobDataExportDir]
- batch001 := shared.GetBatchPath(jobExportDir, jobStart, now+3, 1)
- batch002 := shared.GetBatchPath(jobExportDir, now+3, now+8, 2)
- batch003 := shared.GetBatchPath(jobExportDir, now+8, jobEnd, 3)
- batches := []string{batch001, batch002, batch003}
-
- return JobResults{
- attachments: attachments,
- contents: contents,
- batches: batches,
- jobExportDir: jobExportDir,
- }
-}
-
-func generateActianceBatchTest2(t *testing.T, th *api4.TestHelper, attachmentDir, exportDir string) JobResults {
- now := model.GetMillis()
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.BatchSize = 3
- *cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeActiance
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- for i := range 10 {
- _, e := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: th.BasicChannel.Id,
- UserId: st.NewTestID(),
- Message: st.NewTestID(),
- CreateAt: now + int64(i),
- UpdateAt: now + int64(i),
- })
- require.NoError(t, e)
- }
-
- // start at the 2nd post and get till the 7th post (inclusive) = 6 posts
- job := runJobForTest(t, th, map[string]string{
- shared.JobDataBatchStartTime: strconv.Itoa(int(now) + 1),
- shared.JobDataJobEndTime: strconv.Itoa(int(now) + 6),
- })
- numExported, err := strconv.ParseInt(job.Data[shared.JobDataMessagesExported], 0, 64)
- require.NoError(t, err)
- numExpected, err := strconv.ParseInt(job.Data[shared.JobDataTotalPostsExpected], 0, 64)
- require.NoError(t, err)
- // test that we only exported 6 (because the JobDataJobEndTime was translated to the cursor's UntilUpdateAt)
- require.Equal(t, 6, int(numExported))
- // test that we were reporting that correctly in the UI
- require.Equal(t, 6, int(numExpected))
-
- jobEnd, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 0, 64)
- require.NoError(t, err)
- require.Equal(t, now+6, jobEnd)
- jobExportDir := job.Data[shared.JobDataExportDir]
- batch001 := shared.GetBatchPath(jobExportDir, now+1, now+3, 1)
- // lastPostUpdateAt will be post#4 (now+3), even though we exported it above, because LastPostId will exclude it
- batch002 := shared.GetBatchPath(jobExportDir, now+3, now+6, 2)
- batches := []string{batch001, batch002}
-
- return JobResults{
- batches: batches,
- jobExportDir: jobExportDir,
- }
-}
-
-func generateE2ETestType1Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
- attachmentBackend, exportBackend filestore.FileBackend, testStopping bool) JobResults {
- // This tests (reading the files exported and testing the actual exported data):
- // - job system exports the complete time from beginning to end; i.e., it doesn't use the post updateAt values as the bounds, it uses the start time and end time of the job.
- // - job system uses previous job's end time as the start for the next batch
- // - user joins and leaves before the first post in the first batch (but after the job start time)
- // - user joins and leaves before the first post in the second batch
- // - user joins and leaves before the first post in the last batch
- // - user joins and leaves after the last post in the last batch (but before the job end time)
- // - channel with no posts but user activity (one user joins and leaves after start of batch period but before first post)
- // - channel with no posts but user activity (one user leaves after last post but before end of batch period)
- // - worked, but making sure we test for it specifically in e2e:
- // - exports with multiple channels (this wasn't tested before)
- // - user joins before job start time and stays (should record user's original join time, not the start of the job)
- // - attachments are recorded with correct names in the xml and content in the files
- // - a post from a user who wasn't a member in the channel creates a record for the user entering the channel at the start of the batch and leaving at the end of the batch (to be discussed with end user, this seems wrong)
-
- // Also tests the `BatchSize+1` logic in the worker, because we have 9 posts and batch size of 3.
-
- start := model.GetMillis()
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = 0
- *cfg.MessageExportSettings.BatchSize = 3
- *cfg.MessageExportSettings.ExportFormat = exportType
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- // Users:
- users := make([]*model.User, 0)
- user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user1",
- Email: "user1@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user2",
- Email: "user2@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user3",
- Email: "user3@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user4",
- Email: "user4@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user5",
- Email: "user5@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user6",
- Email: "user6@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user7",
- Email: "user7@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user8",
- Email: "user8@email",
- })
- require.NoError(t, err)
- users = append(users, user)
-
- channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Two",
- Name: "channel_two_name",
- Type: model.ChannelTypePrivate,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // channel3 will have only one user leaving during export time
- channel3, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Three",
- Name: "channel_three_name",
- Type: model.ChannelTypeOpen,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // channel4 will have only one user joining during export time
- channel4, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Four",
- Name: "channel_four_name",
- Type: model.ChannelTypeOpen,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // Save 9 posts so that we have the following batches:
- createUpdateTimes := []int64{
- start + 10, start + 14, start + 20, // batch 1: start-20, posts start at 10
- start + 23, start + 25, start + 30, // batch 2: 20-30, posts start at 23
- start + 36, start + 37, start + 40, // batch 3: 30-40, posts start at 36
- }
-
- jl := []joinLeave{
- {start - 5, 0}, // user 1 never leaves
- {start + 7, start + 8}, // user 2 joins and leaves before first post (but after startTime)
- {start + 11, start + 15}, // user 3 joins and leaves during first batch
- {start + 21, start + 22}, // user 4 joins and leaves during second batch but before second batch's post
- {start + 32, start + 35}, // user 5 joins and leaves during third batch but before third batch's post
- {start + 55, start + 57}, // user 6 joins and leaves after the last batch's last post but before when export is run
- {start - 100, start + 5}, // user 7 joins channel3 before start time and leaves before batch 1
- {start + 59, 0}, // user 8 joins channel4 after batch 3 (but before end)
- }
-
- // user 1 joins before start time and stays (and posts)
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, jl[0].join)
- require.NoError(t, err)
- // user 2 joins and leaves before first post (but after startTime)
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[1].Id, channel2.Id, jl[1].join)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[1].Id, channel2.Id, jl[1].leave)
- require.NoError(t, err)
- // user 3 joins and leaves during first batch
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[2].Id, channel2.Id, jl[2].join)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[2].Id, channel2.Id, jl[2].leave)
- require.NoError(t, err)
- // user 4 joins and leaves during second batch but before second batch's post
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[3].Id, channel2.Id, jl[3].join)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[3].Id, channel2.Id, jl[3].leave)
- // user 5 joins and leaves during third batch but before third batch's post
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[4].Id, channel2.Id, jl[4].join)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[4].Id, channel2.Id, jl[4].leave)
- require.NoError(t, err)
- // user 6 joins and leaves after the last batch's last post
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[5].Id, channel2.Id, jl[5].join)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[5].Id, channel2.Id, jl[5].leave)
- require.NoError(t, err)
-
- // user 7 joins channel3 before start time and leaves before batch 1
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[6].Id, channel3.Id, start-100)
- require.NoError(t, err)
- err = th.App.Srv().Store().ChannelMemberHistory().LogLeaveEvent(users[6].Id, channel3.Id, start+5)
- require.NoError(t, err)
- // user 8 joins channel4 after batch 3 (but before end)
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[7].Id, channel4.Id, start+59)
- require.NoError(t, err)
-
- assertNumPostsToExport(t, th, 0, start, model.GetMillis())
-
- var attachments []*model.FileInfo
- var contents []string
- var posts []*model.Post
- for i, updateAt := range createUpdateTimes {
- post, err2 := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: fmt.Sprintf("message %d", i),
- CreateAt: updateAt,
- UpdateAt: updateAt,
- FileIds: []string{fmt.Sprintf("test%d", i)},
- })
- require.NoError(t, err2)
- posts = append(posts, post)
- time.Sleep(time.Millisecond)
-
- attachmentContent := fmt.Sprintf("Hello there %d", i)
- attachmentPath := fmt.Sprintf("path/to/attachments/file_%d.txt", i)
- _, err = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath)
- require.NoError(t, err)
-
- info, err2 := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
- Id: st.NewTestID(),
- CreatorId: post.UserId,
- PostId: post.Id,
- CreateAt: updateAt,
- UpdateAt: updateAt,
- Path: attachmentPath,
- })
- require.NoError(t, err2)
- attachments = append(attachments, info)
- contents = append(contents, attachmentContent)
- }
-
- // Test that it's picking up a previous successful job
- var previousJob *model.Job
- previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
- require.Error(t, err)
- require.Nil(t, previousJob)
-
- _, err = th.App.Srv().Store().Job().Save(&model.Job{
- Id: "blah",
- Type: model.JobTypeMessageExport,
- Priority: 0,
- CreateAt: 0,
- StartAt: 0,
- LastActivityAt: 0,
- Status: model.JobStatusSuccess,
- Progress: 100,
- Data: map[string]string{shared.JobDataBatchStartTime: strconv.Itoa(int(start))},
- })
- require.NoError(t, err)
-
- previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
- require.NoError(t, err)
- require.NotNilf(t, previousJob, "prevJob")
-
- var prevUpdatedAt int64
- if timestamp, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; prevExists {
- prevUpdatedAt, err = strconv.ParseInt(timestamp, 10, 64)
- require.NoError(t, err)
- }
- require.Equal(t, prevUpdatedAt, start)
-
- // move past the last post time
- time.Sleep(100 * time.Millisecond)
-
- // check number of messages to be exported
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 9, start, until)
-
- // Now run the exports
- var job *model.Job
- if testStopping {
- var jobData map[string]string
- // manually create the job (which will start right away, so we need to wait for it below, after we use its id.
- job, _, err = th.SystemAdminClient.CreateJob(context.Background(), &model.Job{
- Type: "message_export",
- Data: map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)}})
- require.NoError(t, err)
-
- // Stop the export after the second batch by stopping the Worker.
- batchCount := 0
- testEndOfBatchCb = func(worker *MessageExportWorker) {
- batchCount++
- if batchCount == 2 {
- job = getMostRecentJobWithId(t, th, job.Id)
- jobData = job.Data
-
- // let the job continue, but stop Worker, check we went back to Pending, then start the Worker.
- go func() {
- worker.Stop()
- checkJobForStatus(t, th, job.Id, model.JobStatusPending)
- worker.Run()
- checkJobForStatus(t, th, job.Id, model.JobStatusInProgress)
- }()
- }
- }
-
- // Wait for the rest of the exports to finish
- checkJobForStatus(t, th, job.Id, model.JobStatusSuccess)
- job = getMostRecentJobWithId(t, th, job.Id)
- testEndOfBatchCb = nil
- jobDataInvariantsShouldBeEqual(t, jobData, job.Data)
- } else {
- job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
- }
-
- warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
- require.NoError(t, err)
- require.Equal(t, 0, warnings)
-
- assertNumExported(t, 9, job.Data)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batchTimes := []batchStartEndTimes{
- {start: prevUpdatedAt, end: createUpdateTimes[2]},
- {start: createUpdateTimes[2], end: createUpdateTimes[5]},
- {start: createUpdateTimes[5], end: jobEndTime},
- }
-
- batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
- batch002 := shared.GetBatchPath(jobExportDir, batchTimes[1].start, batchTimes[1].end, 2)
- batch003 := shared.GetBatchPath(jobExportDir, batchTimes[2].start, batchTimes[2].end, 3)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches := []string{batch001, batch002, batch003}
- require.ElementsMatch(t, batches, files)
-
- return JobResults{
- start: start,
- joinLeaves: jl,
- users: users,
- createUpdateTimes: createUpdateTimes,
- attachments: attachments,
- contents: contents,
- jobEndTime: jobEndTime,
- batches: batches,
- posts: posts,
- channels: []*model.Channel{channel2, channel3, channel4},
- teams: []*model.Team{th.BasicTeam},
- batchTimes: batchTimes,
- }
-}
-
-func generateE2ETestType2Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
- attachmentBackend, exportBackend filestore.FileBackend) JobResults {
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = 0
- *cfg.MessageExportSettings.BatchSize = 3
- *cfg.MessageExportSettings.ExportFormat = exportType
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- start := model.GetMillis()
-
- // Users:
- users := make([]*model.User, 0)
- user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user1",
- Email: "user1@email",
- })
- require.NoError(t, err)
- users = append(users, user)
- user, err = th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user2",
- Email: "user2@email",
- })
- require.NoError(t, err)
- users = append(users, user)
-
- channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Two",
- Name: "channel_two_name",
- Type: model.ChannelTypePrivate,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- createUpdateTimes := []int64{start + 10, start + 14}
-
- jl := []joinLeave{
- {start - 5, 0}, // user 1 never leaves
- }
-
- // user 1 joins before start time and stays (and posts)
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, jl[0].join)
- require.NoError(t, err)
-
- assertNumPostsToExport(t, th, 0, start, model.GetMillis())
-
- var posts []*model.Post
-
- // first post from user 1 (member)
- post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 1",
- CreateAt: createUpdateTimes[0],
- UpdateAt: createUpdateTimes[0],
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[1].Id,
- Message: "message 2",
- CreateAt: createUpdateTimes[1],
- UpdateAt: createUpdateTimes[1],
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // Test that it's picking up a previous successful job
- var previousJob *model.Job
- previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
- require.Error(t, err)
- require.Nil(t, previousJob)
-
- _, err = th.App.Srv().Store().Job().Save(&model.Job{
- Id: "blah",
- Type: model.JobTypeMessageExport,
- Priority: 0,
- CreateAt: 0,
- StartAt: 0,
- LastActivityAt: 0,
- Status: model.JobStatusSuccess,
- Progress: 100,
- Data: map[string]string{shared.JobDataBatchStartTime: strconv.Itoa(int(start))},
- })
- require.NoError(t, err)
-
- previousJob, err = th.App.Srv().Store().Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
- require.NoError(t, err)
- require.NotNilf(t, previousJob, "prevJob")
-
- var prevUpdatedAt int64
- if timestamp, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; prevExists {
- prevUpdatedAt, err = strconv.ParseInt(timestamp, 10, 64)
- require.NoError(t, err)
- }
- require.Equal(t, prevUpdatedAt, start)
-
- // move past the last post time
- time.Sleep(30 * time.Millisecond)
-
- // check number of messages to be exported
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 2, start, until)
-
- // Now run the exports
- job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- warnings, err := strconv.Atoi(job.Data[shared.JobDataWarningCount])
- require.NoError(t, err)
- require.Equal(t, 0, warnings)
-
- assertNumExported(t, 2, job.Data)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batch001 := shared.GetBatchPath(jobExportDir, prevUpdatedAt, jobEndTime, 1)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches := []string{batch001}
- require.ElementsMatch(t, batches, files)
-
- return JobResults{
- start: start,
- joinLeaves: jl,
- users: users,
- createUpdateTimes: createUpdateTimes,
- jobEndTime: jobEndTime,
- batches: batches,
- posts: posts,
- channels: []*model.Channel{channel2},
- teams: []*model.Team{th.BasicTeam},
- batchTimes: []batchStartEndTimes{{start, jobEndTime}},
- }
-}
-
-// Type3Results specific data needed to be returned by this test only
-type Type3Results struct {
- message1DeleteAt int64
- updatedPost2 *model.Post
- message3AndFileInfoDeleteAt int64
- deletedPost3 *model.Post
-}
-
-func generateE2ETestType3Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
- attachmentBackend, exportBackend filestore.FileBackend) (JobResults, Type3Results) {
- // This tests (reading the files exported and testing the exported xml):
- // - post create at field is set
- // - post deleted fields are set
- // - post updated (not edited)
- // - post deleted with a deleted file
- // - post edited (new message created with original message, old message updated)
- // - post edited with 3 simultaneous posts in-between - forward
- // - post edited but falls on the batch boundary (originalId is in batch 1, newId is batch 2)
-
- start := model.GetMillis()
-
- // Users:
- users := make([]*model.User, 0)
- user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user1",
- Email: "user1@email",
- })
- require.NoError(t, err)
- users = append(users, user)
-
- // only testing one channel
- channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Two",
- Name: "channel_two_name",
- Type: model.ChannelTypePrivate,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // user 1 joins before start time and stays (and posts)
- user1JoinTime := start - 100
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, user1JoinTime)
- require.NoError(t, err)
-
- jl := []joinLeave{
- {user1JoinTime, 0}, // user 1 never leaves
- }
-
- assertNumPostsToExport(t, th, 0, start, model.GetMillis())
-
- var attachments []*model.FileInfo
- var contents []string
- var posts []*model.Post
-
- // 0 - post create
- post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 0",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 1 - post deleted
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 1",
- })
- require.NoError(t, err)
- message1DeleteAt := model.GetMillis()
- err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message1DeleteAt, users[0].Id)
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 2 - post updated not edited (e.g., reaction)
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 2",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
- _, err = th.App.Srv().Store().Reaction().Save(&model.Reaction{
- UserId: users[0].Id,
- PostId: post.Id,
- EmojiName: "smile",
- ChannelId: channel2.Id,
- })
- require.NoError(t, err)
- updatedPost2, err := th.App.Srv().Store().Post().GetSingle(th.Context, post.Id, false)
- require.NoError(t, err)
-
- // 3 - post deleted with a deleted file
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 3",
- FileIds: []string{"test3"},
- })
- require.NoError(t, err)
- time.Sleep(100 * time.Millisecond)
- message3AndFileInfoDeleteAt := model.GetMillis()
- err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message3AndFileInfoDeleteAt, users[0].Id)
- require.NoError(t, err)
- deletedPost3, err := th.App.Srv().Store().Post().GetSingle(th.Context, post.Id, true)
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // Message for deleted file -- NOT INCLUDED IN THE BATCH SIZE
- attachmentContent := "Hello there message 3"
- attachmentPath := "path/to/attachments/file_3.txt"
- _, err = attachmentBackend.WriteFile(bytes.NewBufferString(attachmentContent), attachmentPath)
- require.NoError(t, err)
- info, err2 := th.App.Srv().Store().FileInfo().Save(th.Context, &model.FileInfo{
- Id: st.NewTestID(),
- CreatorId: post.UserId,
- PostId: post.Id,
- CreateAt: post.CreateAt,
- UpdateAt: message3AndFileInfoDeleteAt,
- Path: attachmentPath,
- DeleteAt: message3AndFileInfoDeleteAt,
- })
- require.NoError(t, err2)
- attachments = append(attachments, info)
- time.Sleep(100 * time.Millisecond)
- contents = append(contents, attachmentContent)
-
- // 4 - original post
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 4",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
- // 5 - post edited
- post, err = th.App.Srv().Store().Post().Update(th.Context, &model.Post{
- Id: post.Id,
- CreateAt: post.CreateAt,
- EditAt: model.GetMillis(),
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "edited message 4",
- }, post)
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 6 - post edited but falls on the batch boundary
- // original post, but gets modified by the next edit
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 6",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 7 - new post with original message
- // update returns "new" post, which is the old post modified
- post, err = th.App.Srv().Store().Post().Update(th.Context, &model.Post{
- Id: post.Id,
- CreateAt: post.CreateAt,
- EditAt: model.GetMillis(),
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "edited message 6",
- }, post)
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- require.Len(t, posts, 8)
- // therefore, need a batch size of 7
-
- // use the config fallback
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- *cfg.MessageExportSettings.BatchSize = 7
- *cfg.MessageExportSettings.ExportFormat = exportType
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- // check number of messages to be exported
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 8, start, until)
-
- // Now run the exports
- job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- assertNumExported(t, 8, job.Data)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batchTimes := []batchStartEndTimes{
- {start, posts[7].UpdateAt},
- {posts[7].UpdateAt, jobEndTime},
- }
-
- // using posts[7] because it's updateAt is what posts[6] is changed to (after the edit)
- batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
- batch002 := shared.GetBatchPath(jobExportDir, batchTimes[1].start, batchTimes[1].end, 2)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches := []string{batch001, batch002}
- require.ElementsMatch(t, batches, files)
-
- return JobResults{
- start: start,
- users: users,
- joinLeaves: jl,
- jobEndTime: jobEndTime,
- batches: batches,
- posts: posts,
- attachments: attachments,
- contents: contents,
- channels: []*model.Channel{channel2},
- teams: []*model.Team{th.BasicTeam},
- batchTimes: batchTimes,
- }, Type3Results{
- message1DeleteAt: message1DeleteAt,
- updatedPost2: updatedPost2,
- message3AndFileInfoDeleteAt: message3AndFileInfoDeleteAt,
- deletedPost3: deletedPost3,
- }
-}
-
-func generateE2ETestType4Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
- attachmentBackend, exportBackend filestore.FileBackend) JobResults {
- start := model.GetMillis()
-
- // Users:
- users := make([]*model.User, 0)
- user, err := th.App.Srv().Store().User().Save(th.Context, &model.User{
- Username: "user1",
- Email: "user1@email",
- })
- require.NoError(t, err)
- users = append(users, user)
-
- // only testing one channel
- channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Two",
- Name: "channel_two_name",
- Type: model.ChannelTypePrivate,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // user 1 joins before start time and stays (and posts)
- user1JoinTime := model.GetMillis() - 200
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(users[0].Id, channel2.Id, user1JoinTime)
- require.NoError(t, err)
-
- jl := []joinLeave{
- {user1JoinTime, 0}, // user 1 never leaves
- }
-
- // This tests (reading the files exported and testing the exported xml):
- // - post edited with 3 simultaneous posts in-between
- assertNumPostsToExport(t, th, 0, start, model.GetMillis())
-
- var posts []*model.Post
-
- // 0 - post edited with 3 simultaneous posts in-between - forward
- // original post with edited message
- originalPost, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 0",
- })
- require.NoError(t, err)
- posts = append(posts, originalPost)
- time.Sleep(100 * time.Millisecond)
-
- // 1 - edited post
- post, err := th.App.Srv().Store().Post().Update(th.Context, &model.Post{
- Id: originalPost.Id,
- CreateAt: originalPost.CreateAt,
- EditAt: model.GetMillis(),
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "edited message 0",
- }, originalPost)
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- simultaneous := post.UpdateAt
-
- // 2 - post 1 at same updateAt
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 2",
- CreateAt: simultaneous,
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 3 - post 2 at same updateAt
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 3",
- CreateAt: simultaneous,
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // 4 - post 3 in-between
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: users[0].Id,
- Message: "message 4",
- CreateAt: simultaneous,
- })
- require.NoError(t, err)
- posts = append(posts, post)
- // Use the config fallback for simplicity
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- *cfg.MessageExportSettings.BatchSize = 10
- *cfg.MessageExportSettings.ExportFormat = exportType
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- // check number of messages to be exported
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 5, start, until)
-
- // Now run the exports
- job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
- // cleanup for next run
- _, err = th.App.Srv().Store().Job().Delete(job.Id)
- require.NoError(t, err)
-
- assertNumExported(t, 5, job.Data)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batchTimes := []batchStartEndTimes{{start, jobEndTime}}
-
- batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches := []string{batch001}
- require.ElementsMatch(t, batches, files)
-
- return JobResults{
- start: start,
- users: users,
- joinLeaves: jl,
- jobEndTime: jobEndTime,
- batches: batches,
- posts: posts,
- channels: []*model.Channel{channel2},
- teams: []*model.Team{th.BasicTeam},
- batchTimes: batchTimes,
- }
-}
-
-// Type5Results specific data needed to be returned by this test only
-type Type5Results struct {
- message0DeleteAt int64
- updatedPost1 *model.Post
- zipBytes [][]byte
-}
-
-// generateE2ETestType5Results does 4 jobs, returns an array of each job's data to use in testing
-func generateE2ETestType5Results(t *testing.T, th *api4.TestHelper, exportType, attachmentDir, exportDir string,
- attachmentBackend, exportBackend filestore.FileBackend) ([]JobResults, []Type5Results) {
- // This tests (reading the files exported and testing the exported file):
- // - post deleted in current job: shows created post, then deleted post
- // - post deleted in current job but different batch: shows created post (in second batch), then deleted post
- // - post created in previous job, deleted in current job: shows only deleted post in current job
- // (and same for updated post)
-
- start := model.GetMillis()
-
- channel2, err := th.App.Srv().Store().Channel().Save(th.Context, &model.Channel{
- DisplayName: "the Channel Two",
- Name: "channel_two_name",
- Type: model.ChannelTypePrivate,
- TeamId: th.BasicTeam.Id,
- CreatorId: th.BasicUser.Id,
- }, 999)
- require.NoError(t, err)
-
- // user 1 joins before start time and stays (and posts)
- user1JoinTime := model.GetMillis() - 100
- err = th.App.Srv().Store().ChannelMemberHistory().LogJoinEvent(th.BasicUser.Id, channel2.Id, user1JoinTime)
- require.NoError(t, err)
-
- // Job 1: post deleted in current job: shows created post, then deleted post
- assertNumPostsToExport(t, th, 0, start, model.GetMillis())
-
- var posts []*model.Post
-
- // post create
- post, err := th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 0",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // post deleted
- message0DeleteAt := model.GetMillis()
- err = th.App.Srv().Store().Post().Delete(th.Context, post.Id, message0DeleteAt, th.BasicUser.Id)
- require.NoError(t, err)
-
- // use the config fallback
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- *cfg.MessageExportSettings.BatchSize = 2
- *cfg.MessageExportSettings.ExportFormat = exportType
- *cfg.FileSettings.DriverName = model.ImageDriverLocal
- *cfg.FileSettings.Directory = attachmentDir
-
- if exportDir != attachmentDir {
- *cfg.FileSettings.DedicatedExportStore = true
- *cfg.FileSettings.ExportDriverName = model.ImageDriverLocal
- *cfg.FileSettings.ExportDirectory = exportDir
- }
- })
-
- // check number of messages to be exported -- will be 1 (because one message deleted)
- until := model.GetMillis()
- assertNumPostsToExport(t, th, 1, start, until)
-
- // Now run the exports
- job := runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- assertNumExported(t, 1, job.Data)
-
- jobExportDir := job.Data[shared.JobDataExportDir]
- jobEndTime, err := strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batchTimes := []batchStartEndTimes{
- {start: start, end: jobEndTime},
- }
-
- batch001 := shared.GetBatchPath(jobExportDir, batchTimes[0].start, batchTimes[0].end, 1)
- files, err := exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches := []string{batch001}
- require.ElementsMatch(t, batches, files)
-
- zipBytes, err := exportBackend.ReadFile(batches[0])
- require.NoError(t, err)
-
- var setupReturns []JobResults
- var setupType5Returns []Type5Results
-
- setupReturns = append(setupReturns, JobResults{
- posts: posts,
- })
- setupType5Returns = append(setupType5Returns, Type5Results{
- message0DeleteAt: message0DeleteAt,
- zipBytes: [][]byte{zipBytes},
- })
-
- // Cleanup for next job
- err = os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- _, err = th.App.Srv().Store().Job().Delete(job.Id)
- assert.NoError(t, err)
-
- //
- // Job 2
- //
-
- // Job 2: post deleted in current job, shows up in second batch because it was deleted after the "second" post
- start = model.GetMillis()
-
- posts = make([]*model.Post, 0)
-
- // post create -- this will be the one deleted
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 0",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // post create -- this is the "second" post, but it will show up first because first post is deleted after
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 1",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // post deleted -- first post deleted
- message0DeleteAt = model.GetMillis()
- err = th.App.Srv().Store().Post().Delete(th.Context, posts[0].Id, message0DeleteAt, th.BasicUser.Id)
- require.NoError(t, err)
-
- // use the config fallback
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- *cfg.MessageExportSettings.BatchSize = 1
- })
-
- // check number of messages to be exported
- until = model.GetMillis()
- assertNumPostsToExport(t, th, 2, start, until)
-
- // Now run the exports
- job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- assertNumExported(t, 2, job.Data)
-
- jobExportDir = job.Data[shared.JobDataExportDir]
- jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- // use the message1 updateAt, because the message0's updateAt is now after
- batch001 = shared.GetBatchPath(jobExportDir, start, posts[1].UpdateAt, 1)
- batch002 := shared.GetBatchPath(jobExportDir, posts[1].UpdateAt, jobEndTime, 2)
- files, err = exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches = []string{batch001, batch002}
- require.ElementsMatch(t, batches, files)
-
- zipBytes, err = exportBackend.ReadFile(batches[0])
- require.NoError(t, err)
- zipBytes2, err := exportBackend.ReadFile(batches[1])
- require.NoError(t, err)
-
- setupReturns = append(setupReturns, JobResults{
- posts: posts,
- })
- setupType5Returns = append(setupType5Returns, Type5Results{
- message0DeleteAt: message0DeleteAt,
- zipBytes: [][]byte{zipBytes, zipBytes2},
- })
-
- // Cleanup for next job
- err = os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- _, err = th.App.Srv().Store().Job().Delete(job.Id)
- assert.NoError(t, err)
-
- //
- // Job 3
- //
-
- // Job 3: post created in previous job, deleted in current job: shows only deleted post in current job
- start = model.GetMillis()
-
- posts = make([]*model.Post, 0)
-
- // post create -- this will be the one deleted in second job
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 0",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // post create -- this will be the one updated in second job
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 1",
- })
- require.NoError(t, err)
- posts = append(posts, post)
-
- // use the config fallback
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- *cfg.MessageExportSettings.BatchSize = 10
- })
-
- until = model.GetMillis()
- assertNumPostsToExport(t, th, 2, start, until)
-
- // Now run the exports
- job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- assertNumExported(t, 2, job.Data)
-
- jobExportDir = job.Data[shared.JobDataExportDir]
- jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batch001 = shared.GetBatchPath(jobExportDir, start, jobEndTime, 1)
- files, err = exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches = []string{batch001}
- require.ElementsMatch(t, batches, files)
-
- zipBytes, err = exportBackend.ReadFile(batches[0])
- require.NoError(t, err)
-
- setupReturns = append(setupReturns, JobResults{
- posts: posts,
- })
- setupType5Returns = append(setupType5Returns, Type5Results{
- zipBytes: [][]byte{zipBytes},
- })
-
- // Now, clean up outputs for next job
- err = os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- _, err = th.App.Srv().Store().Job().Delete(job.Id)
- assert.NoError(t, err)
-
- //
- // Job 4
- //
- start = model.GetMillis()
-
- // make a copy of posts so the one we sent earlier doesn't get modified and cause difficult to detect bugs
- posts = append([]*model.Post{}, posts...)
-
- // post create -- filler
- post, err = th.App.Srv().Store().Post().Save(th.Context, &model.Post{
- ChannelId: channel2.Id,
- UserId: th.BasicUser.Id,
- Message: "message 1",
- })
- require.NoError(t, err)
- posts = append(posts, post)
- time.Sleep(100 * time.Millisecond)
-
- // post deleted -- first post deleted (the first one exported earlier)
- message0DeleteAt = model.GetMillis()
- err = th.App.Srv().Store().Post().Delete(th.Context, posts[0].Id, message0DeleteAt, th.BasicUser.Id)
- require.NoError(t, err)
-
- // post updated -- second post updated (the second one exported earlier)
- _, err = th.App.Srv().Store().Reaction().Save(&model.Reaction{
- UserId: th.BasicUser.Id,
- PostId: posts[1].Id,
- EmojiName: "smile",
- ChannelId: channel2.Id,
- })
- require.NoError(t, err)
- updatedPost1, err := th.App.Srv().Store().Post().GetSingle(th.Context, posts[1].Id, false)
- require.NoError(t, err)
-
- // use the config fallback
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.ExportFromTimestamp = start
- })
-
- // check number of messages to be exported
- // filler post, deleted post, and updated post
- until = model.GetMillis()
- assertNumPostsToExport(t, th, 3, start, until)
-
- // Now run the exports
- job = runJobForTest(t, th, map[string]string{shared.JobDataJobEndTime: strconv.FormatInt(until, 10)})
-
- assertNumExported(t, 3, job.Data)
-
- jobExportDir = job.Data[shared.JobDataExportDir]
- jobEndTime, err = strconv.ParseInt(job.Data[shared.JobDataJobEndTime], 10, 64)
- require.NoError(t, err)
-
- batch001 = shared.GetBatchPath(jobExportDir, start, jobEndTime, 1)
- files, err = exportBackend.ListDirectory(jobExportDir)
- require.NoError(t, err)
- batches = []string{batch001}
- require.ElementsMatch(t, batches, files)
-
- zipBytes, err = exportBackend.ReadFile(batches[0])
- require.NoError(t, err)
-
- setupReturns = append(setupReturns, JobResults{
- posts: posts,
- })
- setupType5Returns = append(setupType5Returns, Type5Results{
- message0DeleteAt: message0DeleteAt,
- updatedPost1: updatedPost1,
- zipBytes: [][]byte{zipBytes},
- })
-
- // Cleanup
- err = os.RemoveAll(exportDir)
- assert.NoError(t, err)
- err = os.RemoveAll(attachmentDir)
- assert.NoError(t, err)
- _, err = th.App.Srv().Store().Job().Delete(job.Id)
- assert.NoError(t, err)
-
- return setupReturns, setupType5Returns
-}
diff --git a/server/enterprise/message_export/scheduler.go b/server/enterprise/message_export/scheduler.go
deleted file mode 100644
index 99aa440d7e2..00000000000
--- a/server/enterprise/message_export/scheduler.go
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "net/http"
- "time"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/jobs"
- ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
-)
-
-type MessageExportScheduler struct {
- jobServer *jobs.JobServer
- enabledFunc func(cfg *model.Config) bool
-}
-
-var _ jobs.Scheduler = (*MessageExportScheduler)(nil)
-
-func NewMessageExportScheduler(jobServer *jobs.JobServer, enabledFunc func(cfg *model.Config) bool) *MessageExportScheduler {
- return &MessageExportScheduler{
- enabledFunc: enabledFunc,
- jobServer: jobServer,
- }
-}
-
-func (s *MessageExportScheduler) Enabled(cfg *model.Config) bool {
- return s.enabledFunc(cfg)
-}
-
-func (s *MessageExportScheduler) NextScheduleTime(cfg *model.Config, now time.Time, _ bool, _ *model.Job) *time.Time {
- // We set the next scheduled time regardless of whether there is a running or pending job
- // In ScheduleJob we check pending or running jobs, before actually scheduling a job
- parsedTime, err := time.Parse("15:04", *cfg.MessageExportSettings.DailyRunTime)
- if err != nil {
- s.jobServer.Logger().Error(
- "Cannot determine next schedule time for message export. DailyRunTime config value is invalid.",
- mlog.String("daily_run_time", *cfg.MessageExportSettings.DailyRunTime),
- )
- return nil
- }
- return jobs.GenerateNextStartDateTime(now, parsedTime)
-}
-
-func (s *MessageExportScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, havePendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) {
- // Don't schedule a job if we already have a pending job
- if havePendingJobs {
- return nil, nil
- }
- // Don't schedule a job if we already have a running job
- count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeMessageExport)
- if err != nil {
- return nil, model.NewAppError(
- "ScheduleJob",
- "app.job.get_count_by_status_and_type.app_error",
- map[string]any{"jobtype": model.JobTypeMessageExport, "status": model.JobStatusInProgress},
- "",
- http.StatusInternalServerError).Wrap(err)
- }
- if count > 0 {
- return nil, nil
- }
- return s.jobServer.CreateJob(rctx, model.JobTypeMessageExport, nil)
-}
-
-func (dr *MessageExportJobInterfaceImpl) MakeScheduler() ejobs.Scheduler {
- enabled := func(cfg *model.Config) bool {
- license := dr.Server.License()
- return license != nil && *license.Features.MessageExport && *cfg.MessageExportSettings.EnableExport
- }
- return NewMessageExportScheduler(dr.Server.Jobs, enabled)
-}
diff --git a/server/enterprise/message_export/scheduler_test.go b/server/enterprise/message_export/scheduler_test.go
deleted file mode 100644
index 5371a12c778..00000000000
--- a/server/enterprise/message_export/scheduler_test.go
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/v8/channels/api4"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
-)
-
-func TestMessageExportJobEnabled(t *testing.T) {
- t.Run("MessageExport job is enabled only if feature is enabled", func(t *testing.T) {
- th := api4.SetupEnterpriseWithStoreMock(t)
- defer th.TearDown()
-
- th.Server.SetLicense(model.NewTestLicense("message_export"))
-
- messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
-
- config := &model.Config{
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- },
- }
- scheduler := messageExport.MakeScheduler()
- result := scheduler.Enabled(config)
- assert.True(t, result)
- })
-
- t.Run("MessageExport job is disabled if there is no license", func(t *testing.T) {
- th := api4.SetupEnterpriseWithStoreMock(t)
- defer th.TearDown()
-
- th.Server.SetLicense(nil)
-
- messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
-
- config := &model.Config{
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- },
- }
- scheduler := messageExport.MakeScheduler()
- result := scheduler.Enabled(config)
- assert.False(t, result)
- })
-}
-
-func TestMessageExportJobPending(t *testing.T) {
- th := api4.SetupEnterpriseWithStoreMock(t)
- defer th.TearDown()
-
- mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
- mockUserStore := mocks.UserStore{}
- mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
- mockPostStore := mocks.PostStore{}
- mockPostStore.On("GetMaxPostSize").Return(65535, nil)
- mockSystemStore := mocks.SystemStore{}
- mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
- mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
- mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
- mockStore.On("User").Return(&mockUserStore)
- mockStore.On("Post").Return(&mockPostStore)
- mockStore.On("System").Return(&mockSystemStore)
- mockStore.On("GetDBSchemaVersion").Return(1, nil)
-
- mockJobServerStore := th.App.Srv().Jobs.Store.(*mocks.Store)
- mockJobStore := mocks.JobStore{}
- // Mock that we have an in-progress message export job
- mockJobStore.On("GetCountByStatusAndType", model.JobStatusInProgress, model.JobTypeMessageExport).Return(int64(1), nil)
- mockJobServerStore.On("Job").Return(&mockJobStore)
-
- th.App.UpdateConfig(func(cfg *model.Config) {
- *cfg.MessageExportSettings.EnableExport = true
- *cfg.MessageExportSettings.DailyRunTime = "10:40"
- })
-
- th.App.Srv().SetLicense(model.NewTestLicense("message_export"))
-
- messageExport := &MessageExportJobInterfaceImpl{th.App.Srv()}
- scheduler := messageExport.MakeScheduler()
-
- // Confirm that job is not scheduled if we have pending jobs
- job, err := scheduler.ScheduleJob(th.Context, th.App.Config(), true, nil)
- assert.Nil(t, err)
- assert.Nil(t, job)
-
- // Confirm that job is not scheduled if we have an inprogress job
- job, err = scheduler.ScheduleJob(th.Context, th.App.Config(), false, nil)
- assert.Nil(t, err)
- assert.Nil(t, job)
-}
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl
deleted file mode 100644
index 78c9bbae2fd..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch2.tmpl
+++ /dev/null
@@ -1,101 +0,0 @@
-
- private - channel_two_name - %s
- %d
-
- user1@email
- user
- %d
- user1@email
-
-
- user2@email
- user
- %d
- user2@email
-
-
- user3@email
- user
- %d
- user3@email
-
-
- %s
- user1@email
- user
- %d
- message 0
-
-
- user1@email
- %d
-
- path/to/attachments/file_0.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_0.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 1
-
-
- user1@email
- %d
-
- path/to/attachments/file_1.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_1.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 2
-
-
- user1@email
- %d
-
- path/to/attachments/file_2.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_2.txt
- Completed
-
-
- user2@email
- user
- %d
- user2@email
-
-
- user3@email
- user
- %d
- user3@email
-
-
- user1@email
- user
- %d
- user1@email
-
- %d
-
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl
deleted file mode 100644
index 2a246c4690a..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E1Batch1ch3.tmpl
+++ /dev/null
@@ -1,17 +0,0 @@
-
- public - channel_three_name - %s
- %d
-
- user7@email
- user
- %d
- user7@email
-
-
- user7@email
- user
- %d
- user7@email
-
- %d
-
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl
deleted file mode 100644
index 696344dfc3e..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E1Batch2.tmpl
+++ /dev/null
@@ -1,92 +0,0 @@
-
-
-
- private - channel_two_name - %s
- %d
-
- user1@email
- user
- %d
- user1@email
-
-
- user4@email
- user
- %d
- user4@email
-
-
- %s
- user1@email
- user
- %d
- message 3
-
-
- user1@email
- %d
-
- path/to/attachments/file_3.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_3.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 4
-
-
- user1@email
- %d
-
- path/to/attachments/file_4.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_4.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 5
-
-
- user1@email
- %d
-
- path/to/attachments/file_5.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_5.txt
- Completed
-
-
- user4@email
- user
- %d
- user4@email
-
-
- user1@email
- user
- %d
- user1@email
-
- %d
-
-
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl
deleted file mode 100644
index a3050015e29..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch2.tmpl
+++ /dev/null
@@ -1,101 +0,0 @@
-
- private - channel_two_name - %s
- %d
-
- user1@email
- user
- %d
- user1@email
-
-
- user5@email
- user
- %d
- user5@email
-
-
- user6@email
- user
- %d
- user6@email
-
-
- %s
- user1@email
- user
- %d
- message 6
-
-
- user1@email
- %d
-
- path/to/attachments/file_6.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_6.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 7
-
-
- user1@email
- %d
-
- path/to/attachments/file_7.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_7.txt
- Completed
-
-
- %s
- user1@email
- user
- %d
- message 8
-
-
- user1@email
- %d
-
- path/to/attachments/file_8.txt
-
-
- user1@email
- %d
-
- path/to/attachments/file_8.txt
- Completed
-
-
- user5@email
- user
- %d
- user5@email
-
-
- user6@email
- user
- %d
- user6@email
-
-
- user1@email
- user
- %d
- user1@email
-
- %d
-
diff --git a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl b/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl
deleted file mode 100644
index 28f2a25217f..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E1Batch3ch4.tmpl
+++ /dev/null
@@ -1,17 +0,0 @@
-
- public - channel_four_name - %s
- %d
-
- user8@email
- user
- %d
- user8@email
-
-
- user8@email
- user
- %d
- user8@email
-
- %d
-
diff --git a/server/enterprise/message_export/testdata/actianceE2E2.tmpl b/server/enterprise/message_export/testdata/actianceE2E2.tmpl
deleted file mode 100644
index 6e7e0042a40..00000000000
--- a/server/enterprise/message_export/testdata/actianceE2E2.tmpl
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
- private - channel_two_name - %s
- %d
-
- user1@email
- user
- %d
- user1@email
-
-
- user2@email
- user
- %d
- user2@email
-
-
- %s
- user1@email
- user
- %d
- message 1
-
-
- %s
- user2@email
- user
- %d
- message 2
-
-
- user1@email
- user
- %d
- user1@email
-
-
- user2@email
- user
- %d
- user2@email
-
- %d
-
-
diff --git a/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl b/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl
deleted file mode 100644
index 00082370b95..00000000000
--- a/server/enterprise/message_export/testdata/actianceXMLHeader.tmpl
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-%s%s
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl
deleted file mode 100644
index f2c741c78ed..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E1Batch1.tmpl
+++ /dev/null
@@ -1,14 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[16]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[17]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user2@email,user2,,,,User user2 (user2@email) joined the channel,enter,user,
-%[18]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user2@email,user2,,,,User user2 (user2@email) left the channel,leave,user,
-%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,message 0,message,user,
-%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,(files/%[10]s/%[13]s-file_0.txt),attachment,user,
-%[20]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user3@email,user3,,,,User user3 (user3@email) joined the channel,enter,user,
-%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,message 1,message,user,
-%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,(files/%[11]s/%[14]s-file_1.txt),attachment,user,
-%[22]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user3@email,user3,,,,User user3 (user3@email) left the channel,leave,user,
-%[23]d,0,,%[1]s,,,%[5]s,channel_three_name,the Channel Three,public,%[9]s,user7@email,user7,,,,User user7 (user7@email) was already in the channel,previously-joined,user,
-%[24]d,0,,%[1]s,,,%[5]s,channel_three_name,the Channel Three,public,%[9]s,user7@email,user7,,,,User user7 (user7@email) left the channel,leave,user,
-%[25]d,%[25]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,message 2,message,user,
-%[25]d,%[25]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,(files/%[12]s/%[15]s-file_2.txt),attachment,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl
deleted file mode 100644
index 0d95a21e230..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E1Batch2.tmpl
+++ /dev/null
@@ -1,10 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[13]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user4@email,user4,,,,User user4 (user4@email) joined the channel,enter,user,
-%[15]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user4@email,user4,,,,User user4 (user4@email) left the channel,leave,user,
-%[16]d,%[16]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 3,message,user,
-%[16]d,%[16]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,(files/%[7]s/%[10]s-file_3.txt),attachment,user,
-%[17]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 4,message,user,
-%[17]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,(files/%[8]s/%[11]s-file_4.txt),attachment,user,
-%[18]d,%[18]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 5,message,user,
-%[18]d,%[18]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[12]s-file_5.txt),attachment,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl b/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl
deleted file mode 100644
index 4eb6ab76254..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E1Batch3.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[16]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[17]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user5@email,user5,,,,User user5 (user5@email) joined the channel,enter,user,
-%[18]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[7]s,user5@email,user5,,,,User user5 (user5@email) left the channel,leave,user,
-%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,message 6,message,user,
-%[19]d,%[19]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[10]s,,,(files/%[10]s/%[13]s-file_6.txt),attachment,user,
-%[20]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,message 7,message,user,
-%[20]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[11]s,,,(files/%[11]s/%[14]s-file_7.txt),attachment,user,
-%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,message 8,message,user,
-%[21]d,%[21]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user1@email,user1,%[12]s,,,(files/%[12]s/%[15]s-file_8.txt),attachment,user,
-%[22]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user6@email,user6,,,,User user6 (user6@email) joined the channel,enter,user,
-%[23]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[8]s,user6@email,user6,,,,User user6 (user6@email) left the channel,leave,user,
-%[24]d,0,,%[1]s,,,%[5]s,channel_four_name,the Channel Four,public,%[9]s,user8@email,user8,,,,User user8 (user8@email) joined the channel,enter,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl
deleted file mode 100644
index bd0250d5daa..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E2Batch1.tmpl
+++ /dev/null
@@ -1,5 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[9]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[10]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user2@email,user2,,,,User user2 (user2@email) was already in the channel,previously-joined,user,
-%[11]d,%[11]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
-%[12]d,%[12]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[6]s,user2@email,user2,%[8]s,,,message 2,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl
deleted file mode 100644
index e879f4000d4..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm1.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
-%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
-%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
-%[18]d,%[25]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
-%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
-%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
-%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
-%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
-%[23]d,%[26]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,%[24]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl
deleted file mode 100644
index 15166d1fa33..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm2.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
-%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
-%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
-%[18]d,%[25]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
-%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
-%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
-%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
-%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
-%[23]d,%[26]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,%[24]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl
deleted file mode 100644
index 0ff6103a4ca..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm3.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
-%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
-%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
-%[18]d,%[24]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
-%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
-%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
-%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
-%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
-%[25]d,%[26]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl
deleted file mode 100644
index ba121187d74..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch1Perm4.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[14]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[15]d,%[15]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,message 0,message,user,
-%[16]d,%[17]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,message 1,message,user,
-%[16]d,%[17]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,delete message 1,message,user,
-%[18]d,%[24]d,UpdatedNoMsgChange,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
-%[19]d,%[20]d,Deleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,delete message 3,message,user,
-%[19]d,%[20]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),attachment,user,
-%[19]d,%[20]d,FileDeleted,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,(files/%[9]s/%[13]s-file_3.txt),deleted attachment,user,
-%[21]d,%[22]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[11]s,,,edited message 4,message,user,
-%[21]d,%[22]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,%[11]s,,message 4,message,user,
-%[25]d,%[26]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[12]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl
deleted file mode 100644
index aba0e2a3475..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm1.tmpl
+++ /dev/null
@@ -1,3 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[9]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[8]d,%[10]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,%[7]s,,message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl b/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl
deleted file mode 100644
index cbf6719310b..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E3Batch2Perm2.tmpl
+++ /dev/null
@@ -1,3 +0,0 @@
-Post Creation Time,Post Update Time,Post Update Type,Team Id,Team Name,Team Display Name,Channel Id,Channel Name,Channel Display Name,Channel Type,User Id,User Email,Username,Post Id,Edited By Post Id,Replied to Post Id,Post Message,Post Type,User Type,Previews Post Id
-%[8]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[7]d,%[9]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,,,edited message 6,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl b/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl
deleted file mode 100644
index 3fbf4f31bce..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E4Batch1.tmpl
+++ /dev/null
@@ -1,6 +0,0 @@
-%[11]d,0,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,,,,User user1 (user1@email) was already in the channel,previously-joined,user,
-%[12]d,%[14]d,EditedOriginalMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[6]s,%[7]s,,message 0,message,user,
-%[12]d,%[14]d,EditedNewMsg,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[7]s,,,edited message 0,message,user,
-%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[8]s,,,message 2,message,user,
-%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[9]s,,,message 3,message,user,
-%[13]d,%[14]d,,%[1]s,%[2]s,%[3]s,%[4]s,channel_two_name,the Channel Two,private,%[5]s,user1@email,user1,%[10]s,,,message 4,message,user,
diff --git a/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl b/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl
deleted file mode 100644
index b803ead72b7..00000000000
--- a/server/enterprise/message_export/testdata/csvE2E5Job1.tmpl
+++ /dev/null
@@ -1,2 +0,0 @@
-1732245676086,czoko3u8oidrtcgt58qi3afmrc,faketeambfc64t,dn_qtkwchcc7jng5mo87qw5gzxojo,zao11dtduinninqwtbjn3c6jzw,fakechannelsucatskrmt,dn_pyo5j7yqui865qt69ikoi6sgcr,public,741xw3i71frkmks64of8myh4gc,tfxgyrwfbbymzqz3c74sysbqao@localhost,fakeuser6b4riuzx5b,pf4y8d9s77domftewwfzk4fn5o,,,message 0,message,user,
-1732245676088,czoko3u8oidrtcgt58qi3afmrc,faketeambfc64t,dn_qtkwchcc7jng5mo87qw5gzxojo,zao11dtduinninqwtbjn3c6jzw,fakechannelsucatskrmt,dn_pyo5j7yqui865qt69ikoi6sgcr,public,741xw3i71frkmks64of8myh4gc,tfxgyrwfbbymzqz3c74sysbqao@localhost,fakeuser6b4riuzx5b,pf4y8d9s77domftewwfzk4fn5o,,,delete message 0,message,user,
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl
deleted file mode 100644
index abbac799f95..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch1.tmpl
+++ /dev/null
@@ -1,137 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[22]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[16]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 3 |
-
-
-
- | %[17]s |
- @user2 |
- user |
- user2@email |
- %[5]s |
- %[6]s |
- 0 seconds |
- 0 |
-
-
-
- | %[18]s |
- @user3 |
- user |
- user3@email |
- %[7]s |
- %[8]s |
- 0 seconds |
- 0 |
-
-
-
-
-Messages
-
-
- =20
--
- %[19]s
- %[9]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 0
-
-
--
- %[19]s
- %[9]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[20]s
- %[10]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[20]s
- %[10]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[11]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 2
-
-
--
- %[21]s
- %[11]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
-
-
Exported on %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl
deleted file mode 100644
index 3461fba402c..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch1Ch3.tmpl
+++ /dev/null
@@ -1,47 +0,0 @@
-Conversation Summary
-
-
- - TeamId:%[1]s
- - TeamName:
- - TeamDisplayName:
- - ChannelId:%[2]s
- - ChannelName:channel_three_name
- - ChannelDisplayName:the Channel Three
- - Started:%[3]s
- - Ended:%[4]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[5]s |
- @user7 |
- user |
- user7@email |
- %[6]s |
- %[7]s |
- 0 seconds |
- 0 |
-
-
-
-
-Messages
-
-
-Exported on %[8]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl
deleted file mode 100644
index 9855c22aafb..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch1Summary.tmpl
+++ /dev/null
@@ -1,29 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[14]s
-* TeamName: %[15]s
-* TeamDisplayName: %[16]s
-* ChannelId: %[13]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: %[1]s
-* Started: %[2]s
-* Ended: %[3]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[17]s @user1 user user1@email %[4]s %[5]s 0 seconds 3 %[18]s @user2 user user2@email %[6]s %[7]s 0 seconds 0 %[19]s @user3 user user3@email %[8]s %[9]s 0 seconds 0
-
---------
-Messages
---------
-
-* %[20]s %[10]s @user1 %[17]s @user1 user (user1@email) message 0
-* %[20]s %[10]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[21]s %[11]s @user1 %[17]s @user1 user (user1@email) message 1
-* %[21]s %[11]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[22]s %[12]s @user1 %[17]s @user1 user (user1@email) message 2
-* %[22]s %[12]s @user1 %[17]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl
deleted file mode 100644
index ecc9eba10f2..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch1SummaryCh3.tmpl
+++ /dev/null
@@ -1,22 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[1]s
-* TeamName:
-* TeamDisplayName:
-* ChannelId: %[4]s
-* ChannelName: channel_three_name
-* ChannelDisplayName: the Channel Three
-* Started: %[5]s
-* Ended: %[6]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[7]s @user7 user user7@email %[8]s %[9]s 0 seconds 0
-
---------
-Messages
---------
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl
deleted file mode 100644
index 81cea566453..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch2.tmpl
+++ /dev/null
@@ -1,126 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[11]s
- - TeamName:%[12]s
- - TeamDisplayName:%[13]s
- - ChannelId:%[20]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[14]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 3 |
-
-
-
- | %[16]s |
- @user4 |
- user |
- user4@email |
- %[5]s |
- %[6]s |
- 0 seconds |
- 0 |
-
-
-
-
-Messages
-
-
- =20
--
- %[17]s
- %[7]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- message 3
-
-
--
- %[17]s
- %[7]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[18]s
- %[8]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- message 4
-
-
--
- %[18]s
- %[8]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[19]s
- %[9]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- message 5
-
-
--
- %[19]s
- %[9]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
-
-
Exported on %[10]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl
deleted file mode 100644
index 5197e603df7..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch2Summary.tmpl
+++ /dev/null
@@ -1,29 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[11]s
-* TeamName: %[12]s
-* TeamDisplayName: %[13]s
-* ChannelId: %[10]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[14]s @user1 user user1@email %[3]s %[4]s 0 seconds 3 %[15]s @user4 user user4@email %[5]s %[6]s 0 seconds 0
-
---------
-Messages
---------
-
-* %[16]s %[7]s @user1 %[14]s @user1 user (user1@email) message 3
-* %[16]s %[7]s @user1 %[14]s @user1 user (user1@email) Uploaded file
-* %[17]s %[8]s @user1 %[14]s @user1 user (user1@email) message 4
-* %[17]s %[8]s @user1 %[14]s @user1 user (user1@email) Uploaded file
-* %[18]s %[9]s @user1 %[14]s @user1 user (user1@email) message 5
-* %[18]s %[9]s @user1 %[14]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl
deleted file mode 100644
index 3aa981c8ad6..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch3.tmpl
+++ /dev/null
@@ -1,137 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[22]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[16]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 3 |
-
-
-
- | %[17]s |
- @user5 |
- user |
- user5@email |
- %[5]s |
- %[6]s |
- 0 seconds |
- 0 |
-
-
-
- | %[18]s |
- @user6 |
- user |
- user6@email |
- %[7]s |
- %[8]s |
- 0 seconds |
- 0 |
-
-
-
-
-Messages
-
-
- =20
--
- %[19]s
- %[9]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 6
-
-
--
- %[19]s
- %[9]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[20]s
- %[10]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 7
-
-
--
- %[20]s
- %[10]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[11]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- message 8
-
-
--
- %[21]s
- %[11]s
- @user1
- %[16]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
-
-
Exported on %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl
deleted file mode 100644
index a24fc616f90..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch3Ch4.tmpl
+++ /dev/null
@@ -1,47 +0,0 @@
-Conversation Summary
-
-
- - TeamId:%[1]s
- - TeamName:
- - TeamDisplayName:
- - ChannelId:%[2]s
- - ChannelName:channel_four_name
- - ChannelDisplayName:the Channel Four
- - Started:%[3]s
- - Ended:%[4]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[5]s |
- @user8 |
- user |
- user8@email |
- %[6]s |
- %[4]s |
- 0 seconds |
- 0 |
-
-
-
-
-Messages
-
-
-Exported on %[8]s
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl
deleted file mode 100644
index 2554298f024..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch3Summary.tmpl
+++ /dev/null
@@ -1,29 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[13]s
-* TeamName: %[14]s
-* TeamDisplayName: %[15]s
-* ChannelId: %[12]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[16]s @user1 user user1@email %[3]s %[4]s 0 seconds 3 %[22]s @user5 user user5@email %[5]s %[6]s 0 seconds 0 %[23]s @user6 user user6@email %[7]s %[8]s 0 seconds 0
-
---------
-Messages
---------
-
-* %[19]s %[9]s @user1 %[16]s @user1 user (user1@email) message 6
-* %[19]s %[9]s @user1 %[16]s @user1 user (user1@email) Uploaded file
-* %[20]s %[10]s @user1 %[16]s @user1 user (user1@email) message 7
-* %[20]s %[10]s @user1 %[16]s @user1 user (user1@email) Uploaded file
-* %[21]s %[11]s @user1 %[16]s @user1 user (user1@email) message 8
-* %[21]s %[11]s @user1 %[16]s @user1 user (user1@email) Uploaded file
diff --git a/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl b/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl
deleted file mode 100644
index 910dafdda96..00000000000
--- a/server/enterprise/message_export/testdata/grE2E1Batch3SummaryCh4.tmpl
+++ /dev/null
@@ -1,22 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[1]s
-* TeamName:
-* TeamDisplayName:
-* ChannelId: %[4]s
-* ChannelName: channel_four_name
-* ChannelDisplayName: the Channel Four
-* Started: %[5]s
-* Ended: %[6]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[7]s @user8 user user8@email %[8]s %[6]s 0 seconds 0
-
---------
-Messages
---------
diff --git a/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl b/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl
deleted file mode 100644
index 43c641755c7..00000000000
--- a/server/enterprise/message_export/testdata/grE2E2Batch1.tmpl
+++ /dev/null
@@ -1,82 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[10]s
- - TeamName:%[11]s
- - TeamDisplayName:%[12]s
- - ChannelId:%[13]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[14]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 1 |
-
-
-
- | %[15]s |
- @user2 |
- user |
- user2@email |
- %[5]s |
- %[6]s |
- 0 seconds |
- 1 |
-
-
-
-
-Messages
-
-
- =20
--
- %[16]s
- %[7]s
- @user1
- %[14]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[17]s
- %[8]s
- @user2
- %[15]s
- @user2
- user
- (user2@email)
- message 2
-
-
-
-
Exported on %[9]s
diff --git a/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl b/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl
deleted file mode 100644
index 8dbea6c4ea3..00000000000
--- a/server/enterprise/message_export/testdata/grE2E2Batch1Summary.tmpl
+++ /dev/null
@@ -1,25 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[9]s
-* TeamName: %[10]s
-* TeamDisplayName: %[11]s
-* ChannelId: %[12]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[13]s @user1 user user1@email %[3]s %[4]s 0 seconds 1 %[14]s @user2 user user2@email %[5]s %[6]s 0 seconds 1
-
---------
-Messages
---------
-
-* %[15]s %[7]s @user1 %[13]s @user1 user (user1@email) message 1
-* %[16]s %[8]s @user2 %[14]s @user2 user (user2@email) message 2
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl
deleted file mode 100644
index f3e66c0c1cc..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1Perm1.tmpl
+++ /dev/null
@@ -1,191 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[16]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[17]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 9 |
-
-
-
-
-Messages
-
-
- =20
--
- %[18]s
- %[5]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 0
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 1
- Deleted
- %[26]s
-
-
-
--
- %[20]s
- %[7]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 2
- UpdatedNoMsgChange
- %[27]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 3
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 3
- Deleted
- %[9]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Deleted file
- FileDeleted
- %[9]s
-
-
-
--
- %[22]s
- %[10]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 4
- EditedOriginalMsg
- %[28]s
- %[23]s
-
-
--
- %[23]s
- %[11]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 4
- EditedNewMsg
- %[28]s
-
-
-
--
- %[24]s
- %[12]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 6
- EditedOriginalMsg
- %[29]s
- %[25]s
-
-
-
-
Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl
deleted file mode 100644
index c755f39a459..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1Perm2.tmpl
+++ /dev/null
@@ -1,191 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[16]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[17]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 9 |
-
-
-
-
-Messages
-
-
- =20
--
- %[18]s
- %[5]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 0
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 1
- Deleted
- %[26]s
-
-
-
--
- %[20]s
- %[7]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 2
- UpdatedNoMsgChange
- %[27]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 3
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 3
- Deleted
- %[9]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Deleted file
- FileDeleted
- %[9]s
-
-
-
--
- %[23]s
- %[11]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 4
- EditedNewMsg
- %[28]s
-
-
-
--
- %[22]s
- %[10]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 4
- EditedOriginalMsg
- %[28]s
- %[23]s
-
-
--
- %[24]s
- %[12]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 6
- EditedOriginalMsg
- %[29]s
- %[25]s
-
-
-
-
Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl
deleted file mode 100644
index 70fbaf1602f..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1Perm3.tmpl
+++ /dev/null
@@ -1,191 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[16]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[17]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 9 |
-
-
-
-
-Messages
-
-
- =20
--
- %[18]s
- %[5]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 0
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 1
- Deleted
- %[26]s
-
-
-
--
- %[20]s
- %[7]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 2
- UpdatedNoMsgChange
- %[27]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 3
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 3
- Deleted
- %[9]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Deleted file
- FileDeleted
- %[9]s
-
-
-
--
- %[22]s
- %[10]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 4
- EditedOriginalMsg
- %[28]s
- %[23]s
-
-
--
- %[23]s
- %[11]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 4
- EditedNewMsg
- %[28]s
-
-
-
--
- %[25]s
- %[12]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 6
- EditedNewMsg
- %[29]s
-
-
-
-
-
Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl
deleted file mode 100644
index 118aa327252..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1Perm4.tmpl
+++ /dev/null
@@ -1,191 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[13]s
- - TeamName:%[14]s
- - TeamDisplayName:%[15]s
- - ChannelId:%[16]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[17]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 9 |
-
-
-
-
-Messages
-
-
- =20
--
- %[18]s
- %[5]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 0
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 1
-
-
--
- %[19]s
- %[6]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 1
- Deleted
- %[26]s
-
-
-
--
- %[20]s
- %[7]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 2
- UpdatedNoMsgChange
- %[27]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 3
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Uploaded file
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- delete message 3
- Deleted
- %[9]s
-
-
-
--
- %[21]s
- %[8]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- Deleted file
- FileDeleted
- %[9]s
-
-
-
--
- %[23]s
- %[11]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 4
- EditedNewMsg
- %[28]s
-
-
-
--
- %[22]s
- %[10]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- message 4
- EditedOriginalMsg
- %[28]s
- %[23]s
-
-
--
- %[25]s
- %[12]s
- @user1
- %[17]s
- @user1
- user
- (user1@email)
- edited message 6
- EditedNewMsg
- %[29]s
-
-
-
-
-
Exported on %[1]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl
deleted file mode 100644
index ef64694bf1f..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm1.tmpl
+++ /dev/null
@@ -1,34 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[13]s
-* TeamName: %[14]s
-* TeamDisplayName: %[15]s
-* ChannelId: %[16]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
-
---------
-Messages
---------
-
-* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
-* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
-* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
-* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
-* %[24]s %[12]s @user1 %[17]s @user1 user (user1@email) message 6 EditedOriginalMsg %[29]s %[25]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl
deleted file mode 100644
index 8d63a411856..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm2.tmpl
+++ /dev/null
@@ -1,34 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[13]s
-* TeamName: %[14]s
-* TeamDisplayName: %[15]s
-* ChannelId: %[16]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
-
---------
-Messages
---------
-
-* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
-* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
-* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
-* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
-* %[24]s %[12]s @user1 %[17]s @user1 user (user1@email) message 6 EditedOriginalMsg %[29]s %[25]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl
deleted file mode 100644
index b396e09cfc9..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm3.tmpl
+++ /dev/null
@@ -1,34 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[13]s
-* TeamName: %[14]s
-* TeamDisplayName: %[15]s
-* ChannelId: %[16]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
-
---------
-Messages
---------
-
-* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
-* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
-* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
-* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
-* %[25]s %[12]s @user1 %[17]s @user1 user (user1@email) edited message 6 EditedNewMsg %[29]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl
deleted file mode 100644
index a52c309944f..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch1SummaryPerm4.tmpl
+++ /dev/null
@@ -1,34 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[13]s
-* TeamName: %[14]s
-* TeamDisplayName: %[15]s
-* ChannelId: %[16]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[17]s @user1 user user1@email %[3]s %[4]s 0 seconds 9
-
---------
-Messages
---------
-
-* %[18]s %[5]s @user1 %[17]s @user1 user (user1@email) message 0
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) message 1
-* %[19]s %[6]s @user1 %[17]s @user1 user (user1@email) delete message 1 Deleted %[26]s
-* %[20]s %[7]s @user1 %[17]s @user1 user (user1@email) message 2 UpdatedNoMsgChange %[27]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) message 3
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Uploaded file
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) delete message 3 Deleted %[9]s
-* %[21]s %[8]s @user1 %[17]s @user1 user (user1@email) Deleted file FileDeleted %[9]s
-* %[23]s %[11]s @user1 %[17]s @user1 user (user1@email) edited message 4 EditedNewMsg %[28]s
-* %[22]s %[10]s @user1 %[17]s @user1 user (user1@email) message 4 EditedOriginalMsg %[28]s %[23]s
-* %[25]s %[12]s @user1 %[17]s @user1 user (user1@email) edited message 6 EditedNewMsg %[29]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl
deleted file mode 100644
index 69ae80f23e1..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch2Perm1.tmpl
+++ /dev/null
@@ -1,63 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[6]s
- - TeamName:%[7]s
- - TeamDisplayName:%[8]s
- - ChannelId:%[9]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[10]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 1 |
-
-
-
-
-Messages
-
-
- =20
--
- %[14]s
- %[5]s
- @user1
- %[10]s
- @user1
- user
- (user1@email)
- edited message 6
- EditedNewMsg
- %[12]s
-
-
-
-
-
Exported on %[15]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl
deleted file mode 100644
index 5d263ed5be3..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch2Perm2.tmpl
+++ /dev/null
@@ -1,63 +0,0 @@
-Mattermost Compliance Export
-
-Conversation Summary
-
-
- - TeamId:%[6]s
- - TeamName:%[7]s
- - TeamDisplayName:%[8]s
- - ChannelId:%[9]s
- - ChannelName:channel_two_name
- - ChannelDisplayName:the Channel Two
- - Started:%[1]s
- - Ended:%[2]s
- - Duration:0 seconds
-
-
-
- UserId
|
- Username
|
- UserType
|
- Email |
- Joined |
- Left |
- Duration |
- Messages |
-
- =20
-
- | %[10]s |
- @user1 |
- user |
- user1@email |
- %[3]s |
- %[4]s |
- 0 seconds |
- 1 |
-
-
-
-
-Messages
-
-
- =20
--
- %[13]s
- %[5]s
- @user1
- %[10]s
- @user1
- user
- (user1@email)
- message 6
- EditedOriginalMsg
- %[12]s
- %[14]s
-
-
-
-
Exported on %[15]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl
deleted file mode 100644
index 04b95a29da8..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm1.tmpl
+++ /dev/null
@@ -1,24 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[6]s
-* TeamName: %[7]s
-* TeamDisplayName: %[8]s
-* ChannelId: %[9]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[10]s @user1 user user1@email %[3]s %[4]s 0 seconds 1
-
---------
-Messages
---------
-
-* %[14]s %[5]s @user1 %[10]s @user1 user (user1@email) edited message 6 EditedNewMsg %[12]s
diff --git a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl b/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl
deleted file mode 100644
index 3f5c304d1a7..00000000000
--- a/server/enterprise/message_export/testdata/grE2E3Batch2SummaryPerm2.tmpl
+++ /dev/null
@@ -1,24 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[6]s
-* TeamName: %[7]s
-* TeamDisplayName: %[8]s
-* ChannelId: %[9]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[10]s @user1 user user1@email %[3]s %[4]s 0 seconds 1
-
---------
-Messages
---------
-
-* %[13]s %[5]s @user1 %[10]s @user1 user (user1@email) message 6 EditedOriginalMsg %[12]s %[14]s
diff --git a/server/enterprise/message_export/testdata/grE2E4Summary.tmpl b/server/enterprise/message_export/testdata/grE2E4Summary.tmpl
deleted file mode 100644
index 5d7c87a12cb..00000000000
--- a/server/enterprise/message_export/testdata/grE2E4Summary.tmpl
+++ /dev/null
@@ -1,18 +0,0 @@
---------------------
-Conversation Summary
---------------------
-
-* TeamId: %[5]s
-* TeamName: %[6]s
-* TeamDisplayName: %[7]s
-* ChannelId: %[8]s
-* ChannelName: channel_two_name
-* ChannelDisplayName: the Channel Two
-* Started: %[1]s
-* Ended: %[2]s
-* Duration: 0 seconds
-
-UserId
-Username
-UserType
-Email Joined Left Duration Messages %[9]s @user1 user user1@email %[3]s %[4]s 0 seconds 5
diff --git a/server/enterprise/message_export/worker.go b/server/enterprise/message_export/worker.go
deleted file mode 100644
index b81f410304a..00000000000
--- a/server/enterprise/message_export/worker.go
+++ /dev/null
@@ -1,539 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "context"
- "errors"
- "fmt"
- "net/http"
- "path"
- "strconv"
- "sync"
- "time"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/jobs"
- "github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
- "github.com/mattermost/mattermost/server/v8/platform/shared/templates"
-)
-
-const (
- TimeBetweenBatchesMs = 100
- DefaultPreviousJobPageSize = 5
-)
-
-// testEndOfBatchCb is only used for testing
-var testEndOfBatchCb func(worker *MessageExportWorker)
-
-type MessageExportWorker struct {
- name string
- // stateMut protects stopCh, cancel, and stopped and helps enforce
- // ordering in case subsequent Run or Stop calls are made.
- stateMut sync.Mutex
- stopCh chan struct{}
- stopped bool
- stoppedCh chan struct{}
- jobs chan model.Job
- jobServer *jobs.JobServer
- logger mlog.LoggerIFace
- htmlTemplateWatcher *templates.Container
- license func() *model.License
-
- context context.Context
- cancel func()
-}
-
-func (dr *MessageExportJobInterfaceImpl) MakeWorker() model.Worker {
- const workerName = "MessageExportWorker"
- logger := dr.Server.Jobs.Logger().With(mlog.String("worker_name", workerName))
-
- templatesDir, ok := fileutils.FindDir("templates")
- if !ok {
- logger.Error("Failed to initialize HTMLTemplateWatcher, templates directory not found")
- return nil
- }
- htmlTemplateWatcher, err := templates.New(templatesDir)
- if err != nil {
- logger.Error("Failed to initialize HTMLTemplateWatcher", mlog.Err(err))
- return nil
- }
-
- ctx, cancel := context.WithCancel(context.Background())
-
- return &MessageExportWorker{
- name: workerName,
- stoppedCh: make(chan struct{}, 1),
- jobs: make(chan model.Job),
- jobServer: dr.Server.Jobs,
- logger: logger,
- htmlTemplateWatcher: htmlTemplateWatcher,
- // It is not a best practice to store context inside a struct,
- // however we need to cancel a SQL query during a job execution.
- // There is no other good way.
- context: ctx,
- cancel: cancel,
- license: dr.Server.License,
- stopped: true,
- }
-}
-
-func (w *MessageExportWorker) IsEnabled(cfg *model.Config) bool {
- return w.license() != nil && *w.license().Features.MessageExport && *cfg.MessageExportSettings.EnableExport
-}
-
-func (w *MessageExportWorker) Run() {
- w.stateMut.Lock()
- // We have to re-assign the stop channel again, because
- // it might happen that the job was restarted due to a config change.
- if w.stopped {
- w.stopped = false
- w.stopCh = make(chan struct{})
- w.context, w.cancel = context.WithCancel(context.Background())
- } else {
- w.stateMut.Unlock()
- return
- }
- // Run is called from a separate goroutine and doesn't return.
- // So we cannot Unlock in a defer clause.
- w.stateMut.Unlock()
-
- w.logger.Debug("Worker Started")
-
- defer func() {
- w.logger.Debug("Worker finished")
- w.stoppedCh <- struct{}{}
- }()
-
- for {
- select {
- case <-w.stopCh:
- w.logger.Debug("Worker: Received stop signal")
- return
- case job := <-w.jobs:
- w.DoJob(&job)
- }
- }
-}
-
-func (w *MessageExportWorker) Stop() {
- w.stateMut.Lock()
- defer w.stateMut.Unlock()
-
- // Set to close, and if already closed before, then return.
- if w.stopped {
- return
- }
- w.stopped = true
-
- w.logger.Debug("Worker: Stopping")
- w.cancel()
- close(w.stopCh)
- <-w.stoppedCh
-}
-
-func (w *MessageExportWorker) JobChannel() chan<- model.Job {
- return w.jobs
-}
-
-func (w *MessageExportWorker) DoJob(job *model.Job) {
- logger := w.logger.With(jobs.JobLoggerFields(job)...)
- logger.Debug("Worker: Received a new candidate job.")
- defer w.jobServer.HandleJobPanic(logger, job)
-
- var appErr *model.AppError
- job, appErr = w.jobServer.ClaimJob(job)
- if appErr != nil {
- logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
- return
- } else if job == nil {
- return
- }
-
- var cancelContext request.CTX = request.EmptyContext(w.logger)
- cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
- cancelWatcherChan := make(chan struct{}, 1)
- cancelContext = cancelContext.WithContext(cancelCtx)
- go w.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
- defer cancelCancelWatcher()
-
- rctx := request.EmptyContext(logger).WithContext(w.context)
- // if job data is missing, we'll do our best to recover
- w.initJobData(rctx, logger, job, time.Now())
- data, err := extractJobData(logger, job.Data)
- if err != nil {
- // Error in conversion. Not much we can do about that. But it shouldn't happen, unless someone edited the db.
- w.setJobError(logger, job, model.NewAppError("Job.DoJob", "ent.message_export.job_data_conversion.app_error", nil, "", http.StatusBadRequest).Wrap(err))
- return
- }
-
- reportProgress := func(message string) {
- logger.Debug(message)
- // Don't fail because we couldn't update progress.
- w.setJobProgressMessage(0, message, rctx.Logger(), job)
- }
-
- jobParams := shared.BackendParams{
- Config: w.jobServer.Config(),
- Store: shared.NewMessageExportStore(w.jobServer.Store),
- HtmlTemplates: w.htmlTemplateWatcher,
- }
- jobParams.FileAttachmentBackend, err = shared.GetFileAttachmentBackend(rctx, w.jobServer.Config())
- if err != nil {
- w.setJobError(logger, job, model.NewAppError("GetFileAttachmentBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
- return
- }
- jobParams.ExportBackend, err = shared.GetExportBackend(rctx, w.jobServer.Config())
- if err != nil {
- w.setJobError(logger, job, model.NewAppError("GetExportBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
- return
- }
-
- data, err = shared.GetInitialExportPeriodData(rctx, jobParams.Store, data, reportProgress)
- if err != nil {
- w.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.calculate_channel_exports.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
- return
- }
- job.Data[shared.JobDataTotalPostsExpected] = strconv.Itoa(data.TotalPostsExpected)
-
- for {
- select {
- case <-cancelWatcherChan:
- logger.Debug("Worker: Job has been canceled via CancellationWatcher")
- w.setJobCanceled(logger, job)
- return
-
- case <-w.stopCh:
- logger.Debug("Worker: Job has been canceled via Worker Stop. Setting the job back to pending")
- w.SetJobPending(logger, job)
- return
-
- case <-time.After(TimeBetweenBatchesMs * time.Millisecond):
- logger.Debug("Starting batch export", mlog.Int("last_post_update_at", data.Cursor.LastPostUpdateAt))
-
- _, data, err = RunBatch(rctx, data, jobParams)
- if err != nil {
- // We ignore error if the job was explicitly cancelled
- if errors.Is(w.context.Err(), context.Canceled) {
- logger.Debug("Worker: Job has been canceled via worker's context. Setting the job back to pending")
- w.SetJobPending(logger, job)
- } else {
- w.setJobError(logger, job, model.NewAppError("DoJob", "ent.message_export.run_export.app_error", nil, "", http.StatusInternalServerError).Wrap(err))
- }
- return
- }
-
- setJobDataEndOfBatch(job, data)
-
- if data.Finished {
- w.finishExport(rctx, logger, job, data.WarningCount)
- return
- }
-
- // also saves job.Data
- if err := w.setJobProgress(logger, job, getJobProgress(data.MessagesExported, data.TotalPostsExpected)); err != nil {
- // TODO: MM-59093 handle job errors (robust, recoverable)
- return
- }
-
- // testEndOfBatchCb is only used by tests.
- if testEndOfBatchCb != nil {
- testEndOfBatchCb(w)
- }
- }
- }
-}
-
-func (w *MessageExportWorker) finishExport(rctx request.CTX, logger *mlog.Logger, job *model.Job, totalWarningCount int) {
- job.Data[shared.JobDataWarningCount] = strconv.Itoa(totalWarningCount)
- // we've exported everything up to the current time
- logger.Debug("FormatExport complete")
-
- job.Data[shared.JobDataIsDownloadable] = "true"
-
- if totalWarningCount > 0 {
- w.setJobWarning(logger, job)
- } else {
- w.setJobSuccess(logger, job)
- }
-}
-
-// initializes job data if it's missing, allows us to recover from failed or improperly configured jobs
-func (w *MessageExportWorker) initJobData(rctx request.CTX, logger mlog.LoggerIFace, job *model.Job, now time.Time) {
- if job.Data == nil {
- job.Data = make(map[string]string)
- }
- if _, exists := job.Data[shared.JobDataMessagesExported]; !exists {
- logger.Info("Worker: JobDataMessagesExported does not exist, starting at 0")
- job.Data[shared.JobDataMessagesExported] = "0"
- }
- if _, exists := job.Data[shared.JobDataExportType]; !exists {
- exportFormat := *w.jobServer.Config().MessageExportSettings.ExportFormat
- logger.Info("Worker: Defaulting to configured export format", mlog.String("export_format", exportFormat))
- job.Data[shared.JobDataExportType] = exportFormat
- }
- if _, exists := job.Data[shared.JobDataBatchSize]; !exists {
- batchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.BatchSize)
- logger.Info("Worker: Defaulting to configured batch size", mlog.String("batch_size", batchSize))
- job.Data[shared.JobDataBatchSize] = batchSize
- }
- if _, exists := job.Data[shared.JobDataChannelBatchSize]; !exists {
- channelBatchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.ChannelBatchSize)
- logger.Info("Worker: Defaulting to configured channel batch size", mlog.String("channel_batch_size", channelBatchSize))
- job.Data[shared.JobDataChannelBatchSize] = channelBatchSize
- }
- if _, exists := job.Data[shared.JobDataChannelHistoryBatchSize]; !exists {
- channelHistoryBatchSize := strconv.Itoa(*w.jobServer.Config().MessageExportSettings.ChannelHistoryBatchSize)
- logger.Info("Worker: Defaulting to configured channel history batch size", mlog.String("channel_history_batch_size", channelHistoryBatchSize))
- job.Data[shared.JobDataChannelHistoryBatchSize] = channelHistoryBatchSize
- }
- if _, exists := job.Data[shared.JobDataBatchNumber]; !exists {
- logger.Info("Worker: JobDataBatchNumber does not exist, starting at 0")
- job.Data[shared.JobDataBatchNumber] = "0"
- }
-
- // If this is a new job (JobEndTime doesn't exist), set it to now, because this is when the job has first started.
- // The logic is that a job exports messages up to the moment the job was started. If the job was picked up after
- // gracefully stopping, then run it until that original initial endTime.
- // However, if the job was cancelled or errored out, that job will not be picked up again, so this will be a new job
- // starting from the last successful batchStartTimestamp up until now. This is intentional (for now) because failed
- // jobs do not get rescheduled properly yet, and when they are run again it means that new day's worth of messages
- // need to be exported.
- if _, exists := job.Data[shared.JobDataJobEndTime]; !exists {
- millis := strconv.FormatInt(model.GetMillisForTime(now), 10)
- logger.Info("Worker: JobDataJobEndTime not found in previous job, using now", mlog.String("job_data_job_end_time", millis))
- job.Data[shared.JobDataJobEndTime] = millis
- }
-
- if _, exists := job.Data[shared.JobDataBatchStartTime]; !exists {
- previousJob, err := w.getPreviousNonCliJob(rctx)
-
- if err != nil {
- exportFromTimestamp := strconv.FormatInt(*w.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
- logger.Info("Worker: No previously successful job found, falling back to configured MessageExportSettings.ExportFromTimestamp", mlog.String("export_from_timestamp", exportFromTimestamp))
- job.Data[shared.JobDataBatchStartTime] = exportFromTimestamp
- job.Data[shared.JobDataJobStartTime] = exportFromTimestamp
- job.Data[shared.JobDataBatchStartId] = ""
- job.Data[shared.JobDataJobStartId] = job.Data[shared.JobDataBatchStartId]
- job.Data[shared.JobDataExportDir] = getJobExportDir(logger, job.Data, exportFromTimestamp, job.Data[shared.JobDataJobEndTime])
- return
- }
-
- logger.Info("Worker: Implicitly resuming export from where previously successful job left off")
- if previousJob == nil {
- previousJob = &model.Job{}
- }
- if previousJob.Data == nil {
- previousJob.Data = make(map[string]string)
- }
-
- // Backwards compatibility for <10.5:
- if batchStartTimestamp, prevExists := previousJob.Data["batch_start_timestamp"]; prevExists {
- previousJob.Data[shared.JobDataBatchStartTime] = batchStartTimestamp
- }
-
- if _, prevExists := previousJob.Data[shared.JobDataBatchStartTime]; !prevExists {
- exportFromTimestamp := strconv.FormatInt(*w.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
- logger.Info("Worker: Previously successful job lacks job data, falling back to configured MessageExportSettings.ExportFromTimestamp", mlog.String("export_from_timestamp", exportFromTimestamp))
- job.Data[shared.JobDataBatchStartTime] = exportFromTimestamp
- job.Data[shared.JobDataJobStartTime] = exportFromTimestamp
- } else {
- job.Data[shared.JobDataBatchStartTime] = previousJob.Data[shared.JobDataBatchStartTime]
- }
- if _, prevExists := previousJob.Data[shared.JobDataBatchStartId]; !prevExists {
- logger.Info("Worker: Previously successful job lacks post ID, falling back to empty string")
- job.Data[shared.JobDataBatchStartId] = ""
- } else {
- job.Data[shared.JobDataBatchStartId] = previousJob.Data[shared.JobDataBatchStartId]
- }
- job.Data[shared.JobDataJobStartId] = job.Data[shared.JobDataBatchStartId]
- } else {
- logger.Info("Worker: JobDataBatchStartTime start time was already set",
- mlog.String(shared.JobDataBatchStartTime, job.Data[shared.JobDataBatchStartTime]))
- }
-
- if _, exists := job.Data[shared.JobDataJobStartTime]; !exists {
- // Just in case, if we don't have this (JobDataBatchStartTime was already set, but this wasn't) set it:
- job.Data[shared.JobDataJobStartTime] = job.Data[shared.JobDataBatchStartTime]
- logger.Info("Worker: JobDataJobStartTime start time was not set, using batch startTimestamp",
- mlog.String(shared.JobDataJobStartTime, job.Data[shared.JobDataJobStartTime]))
- }
-
- job.Data[shared.JobDataExportDir] = getJobExportDir(logger, job.Data, job.Data[shared.JobDataJobStartTime], job.Data[shared.JobDataJobEndTime])
-}
-
-// getPreviousNonCliJob returns the most recent job that was not initiated by mmctl
-func (w *MessageExportWorker) getPreviousNonCliJob(rctx request.CTX) (*model.Job, error) {
- offset := 0
-
- for {
- jobs, err := w.jobServer.Store.Job().GetAllByTypesAndStatusesPage(rctx,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- offset, DefaultPreviousJobPageSize)
- if err != nil {
- return nil, err
- }
-
- // Find the first job not initiated by mmctl
- for _, job := range jobs {
- if job.Data == nil || job.Data[shared.JobDataInitiatedBy] != "mmctl" {
- return job, nil
- }
- }
-
- // If we didn't get a full page of jobs, we've reached the end
- if len(jobs) < DefaultPreviousJobPageSize {
- return nil, nil
- }
-
- // If we didn't find a non-mmctl job in this page, continue to the next page
- offset += DefaultPreviousJobPageSize
- }
-}
-
-func extractJobData(logger *mlog.Logger, strmap map[string]string) (shared.JobData, error) {
- data, err := shared.StringMapToJobDataWithZeroValues(strmap)
- if err != nil {
- return data, err
- }
-
- // ExportPeriodStartTime is initialized to BatchStartTime because this is where we will start exporting. But unlike
- // BatchStartTime, it won't change as we process the batches.
- // If this is the first time this job has run, BatchStartTime will be the start of the entire job. If this job has
- // been resumed, then BatchStartTime will be the start of the newest batch. This is expected--the channel activity
- // and total posts will be calculated from ExportPeriodStartTime (anything earlier has already been exported in
- // previous batches).
- // Note: ExportPeriodStartTime is different from JobStartTime because JobStartTime won't change
- // if the job processes some batches, is stopped, and picked up again.
- data.ExportPeriodStartTime = data.BatchStartTime
-
- logger.Info("Worker: initial job variables set",
- mlog.String("export_type", data.ExportType),
- mlog.String("export_dir", data.ExportDir),
- mlog.Int("job_start_time", data.JobStartTime),
- mlog.Int("batch_start_time", data.BatchStartTime),
- mlog.Int("export_period_start_time", data.ExportPeriodStartTime),
- mlog.Int("job_end_time", data.JobEndTime),
- mlog.String("job_start_id", data.JobStartId),
- mlog.Int("batch_size", data.BatchSize),
- mlog.Int("channel_batch_size", data.ChannelBatchSize),
- mlog.Int("channel_history_batch_size", data.ChannelHistoryBatchSize),
- mlog.Int("batch_number", data.BatchNumber),
- mlog.Int("total_posts_exported", data.MessagesExported))
-
- return data, err
-}
-
-func setJobDataEndOfBatch(job *model.Job, data shared.JobData) {
- job.Data[shared.JobDataBatchStartTime] = strconv.FormatInt(data.BatchStartTime, 10)
- job.Data[shared.JobDataBatchStartId] = data.Cursor.LastPostId
- job.Data[shared.JobDataMessagesExported] = strconv.Itoa(data.MessagesExported)
- job.Data[shared.JobDataBatchNumber] = strconv.Itoa(data.BatchNumber)
-}
-
-// getJobExportDir will use the existing JobDataExportDir if available. If it's not available, this is the first run
-// for the job, so we use the startTime and endTime passed in.
-func getJobExportDir(logger mlog.LoggerIFace, data model.StringMap, startTime string, endTime string) string {
- exportDir, exists := data[shared.JobDataExportDir]
- if !exists {
- // If we don't have a jobDataExportDir, this is the first run for the job, so we use the batch startTime
- exportDir = path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%s-%s", time.Now().Format(model.ComplianceExportDirectoryFormat), startTime, endTime))
- logger.Info("Worker: JobDataExportDir does not exist, using current datetime", mlog.String("job_data_export_dir", exportDir))
- }
-
- return exportDir
-}
-
-func getJobProgress(totalExportedPosts, totalPostsExpected int) int {
- return totalExportedPosts * 100 / totalPostsExpected
-}
-
-func (w *MessageExportWorker) setJobProgressMessage(progress int64, message string, logger mlog.LoggerIFace, job *model.Job) {
- job.Status = model.JobStatusInProgress
- job.Progress = progress
- if job.Data == nil {
- job.Data = make(map[string]string)
- }
- job.Data["progress_message"] = message
-
- if _, err := w.jobServer.Store.Job().UpdateOptimistically(job, model.JobStatusInProgress); err != nil {
- logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- }
-}
-
-func (w *MessageExportWorker) setJobProgress(logger mlog.LoggerIFace, job *model.Job, progress int) error {
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
-
- if err := w.jobServer.SetJobProgress(job, int64(progress)); err != nil {
- logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- w.setJobError(logger, job, err)
- return err
- }
-
- return nil
-}
-
-func (w *MessageExportWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
- // setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
- if err := w.jobServer.SetJobProgress(job, 100); err != nil {
- logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- w.setJobError(logger, job, err)
- }
- if err := w.jobServer.SetJobSuccess(job); err != nil {
- logger.Error("Worker: Failed to set success for job", mlog.Err(err))
- w.setJobError(logger, job, err)
- }
-}
-
-func (w *MessageExportWorker) setJobWarning(logger mlog.LoggerIFace, job *model.Job) {
- // setting progress causes the job data to be saved, which is necessary if we want the next job to pick up where this one left off
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
- if err := w.jobServer.SetJobProgress(job, 100); err != nil {
- logger.Error("Worker: Failed to update progress for job", mlog.Err(err))
- w.setJobError(logger, job, err)
- }
- if err := w.jobServer.SetJobWarning(job); err != nil {
- logger.Error("Worker: Failed to set warning for job", mlog.Err(err))
- w.setJobError(logger, job, err)
- }
-}
-
-func (w *MessageExportWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
- logger.Error("Worker: Job error", mlog.Err(appError))
- if err := w.jobServer.SetJobError(job, appError); err != nil {
- logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
- }
-}
-
-func (w *MessageExportWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
- if err := w.jobServer.SetJobCanceled(job); err != nil {
- logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
- }
-}
-
-func (w *MessageExportWorker) SetJobPending(logger mlog.LoggerIFace, job *model.Job) {
- if job.Data != nil {
- job.Data["progress_message"] = ""
- }
- if err := w.jobServer.SetJobPending(job); err != nil {
- logger.Error("Worker: Failed to mark job as pending", mlog.Err(err))
- }
-}
diff --git a/server/enterprise/message_export/worker_test.go b/server/enterprise/message_export/worker_test.go
deleted file mode 100644
index f64ec82b2a7..00000000000
--- a/server/enterprise/message_export/worker_test.go
+++ /dev/null
@@ -1,750 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.enterprise for license information.
-
-package message_export
-
-import (
- "context"
- "errors"
- "fmt"
- "os"
- "path"
- "strconv"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
- tmock "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-
- "github.com/mattermost/mattermost/server/public/model"
- "github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
- "github.com/mattermost/mattermost/server/public/shared/mlog"
- "github.com/mattermost/mattermost/server/public/shared/request"
- "github.com/mattermost/mattermost/server/v8/channels/app"
- "github.com/mattermost/mattermost/server/v8/channels/jobs"
- "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
- "github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
- "github.com/mattermost/mattermost/server/v8/einterfaces/mocks"
- "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
-)
-
-func TestInitJobDataNoJobData(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- }
-
- // mock job store doesn't return a previously successful job, forcing fallback to config
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return(nil, errors.New("test"))
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- ConfigService: &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- },
- logger: logger,
- }
-
- now := time.Now()
- worker.initJobData(request.EmptyContext(logger), logger, job, now)
-
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
- assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[shared.JobDataBatchStartTime])
- expectedDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%d-%d", now.Format(model.ComplianceExportDirectoryFormat), 0, now.UnixMilli()))
- assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
-}
-
-func TestInitJobDataPreviousJobNoJobData(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- previousJob := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- StartAt: model.GetMillis() - 1000,
- LastActivityAt: model.GetMillis() - 1000,
- }
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- }
-
- // mock job store returns a previously successful job, but it doesn't have job data either, so we still fall back to config
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{previousJob}, nil)
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- ConfigService: &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- },
- logger: logger,
- }
-
- now := time.Now()
- worker.initJobData(request.EmptyContext(logger), logger, job, now)
-
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
- assert.Equal(t, strconv.FormatInt(*worker.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10), job.Data[shared.JobDataBatchStartTime])
- expectedDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%d-%d", now.Format(model.ComplianceExportDirectoryFormat), 0, now.UnixMilli()))
- assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
-}
-
-func TestInitJobDataPreviousJobWithJobData(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- previousJob := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- StartAt: model.GetMillis() - 1000,
- LastActivityAt: model.GetMillis() - 1000,
- Data: map[string]string{shared.JobDataBatchStartTime: "123"},
- }
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{shared.JobDataExportDir: "this-is-the-export-dir"},
- }
-
- // mock job store returns a previously successful job that has the config that we're looking for, so we use it
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{previousJob}, nil)
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- ConfigService: &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- },
- logger: logger,
- }
-
- now := time.Now()
- worker.initJobData(request.EmptyContext(logger), logger, job, now)
-
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
- assert.Equal(t, previousJob.Data[shared.JobDataBatchStartTime], job.Data[shared.JobDataBatchStartTime])
- expectedDir := "this-is-the-export-dir"
- assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
-}
-
-func TestInitJobDataPreviousJobWithJobDataPre105(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- previousJob := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- StartAt: model.GetMillis() - 1000,
- LastActivityAt: model.GetMillis() - 1000,
- Data: map[string]string{"batch_start_timestamp": "123"},
- }
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{shared.JobDataExportDir: "this-is-the-export-dir"},
- }
-
- // mock job store returns a previously successful job that has the config that we're looking for, so we use it
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{previousJob}, nil)
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- ConfigService: &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- },
- logger: logger,
- }
-
- now := time.Now()
- worker.initJobData(request.EmptyContext(logger), logger, job, now)
-
- assert.Equal(t, model.ComplianceExportTypeActiance, job.Data[shared.JobDataExportType])
- assert.Equal(t, strconv.Itoa(*worker.jobServer.Config().MessageExportSettings.BatchSize), job.Data[shared.JobDataBatchSize])
-
- // Assert the new job picks up the <10.5 job start time:
- assert.Equal(t, previousJob.Data[shared.JobDataBatchStartTime], job.Data[shared.JobDataBatchStartTime])
-
- expectedDir := "this-is-the-export-dir"
- assert.Equal(t, expectedDir, job.Data[shared.JobDataExportDir])
-}
-
-func TestDoJobNoPostsToExport(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- mockMetrics := &mocks.MetricsInterface{}
- defer mockMetrics.AssertExpectations(t)
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- }
- retJob := *job
- retJob.Status = model.JobStatusInProgress
-
- // claim job succeeds
- mockStore.JobStore.
- On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).
- Return(&retJob, nil)
- mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
-
- // no previous job, data will be loaded from config
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return(nil, errors.New("test"))
-
- // no channels with activity
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return(make([]string, 0), nil)
-
- // no posts found to export
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
- make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
- )
-
- mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(shared.EstimatedPostCount), nil,
- )
-
- // job completed successfully
- mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil)
- mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusSuccess).Return(job, nil)
- mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
-
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() {
- err = os.RemoveAll(tempDir)
- assert.NoError(t, err)
- })
-
- worker := &MessageExportWorker{
- jobServer: jobs.NewJobServer(
- &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- FileSettings: model.FileSettings{
- DriverName: model.NewPointer(model.ImageDriverLocal),
- Directory: model.NewPointer(tempDir),
- },
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- mockStore,
- mockMetrics,
- logger,
- ),
- logger: logger,
- }
-
- // actually execute the code under test
- worker.DoJob(job)
-}
-
-func TestDoJobWithDedicatedExportBackend(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
-
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- mockMetrics := &mocks.MetricsInterface{}
- defer mockMetrics.AssertExpectations(t)
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- }
- retJob := *job
- retJob.Status = model.JobStatusInProgress
-
- // claim job succeeds
- mockStore.JobStore.
- On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).
- Return(&retJob, nil)
- mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
-
- // no previous job, data will be loaded from config
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return(nil, errors.New("test"))
-
- channelId := st.NewTestID()
- channelName := st.NewTestID()
- channelDisplayName := st.NewTestID()
- channelType := model.ChannelTypeOpen
- messages := []*model.MessageExport{
- {
- TeamId: model.NewPointer(st.NewTestID()),
- ChannelId: model.NewPointer(channelId),
- ChannelName: model.NewPointer(channelName),
- UserId: model.NewPointer(st.NewTestID()),
- UserEmail: model.NewPointer(st.NewTestID()),
- Username: model.NewPointer(st.NewTestID()),
- PostId: model.NewPointer(st.NewTestID()),
- PostCreateAt: model.NewPointer[int64](123),
- PostUpdateAt: model.NewPointer[int64](123),
- PostDeleteAt: model.NewPointer[int64](123),
- PostMessage: model.NewPointer(st.NewTestID()),
- },
- }
-
- // need to export at least one post to make an export directory and file
-
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return([]string{*messages[0].ChannelId}, nil)
- mockStore.ChannelStore.On("GetMany", []string{channelId}, true).
- Return(model.ChannelList{{
- Id: channelId,
- DisplayName: channelDisplayName,
- Name: channelName,
- Type: channelType,
- }}, nil)
-
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
- messages, model.MessageExportCursor{}, nil,
- ).Once()
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Return(
- make([]*model.MessageExport, 0), model.MessageExportCursor{}, nil,
- ).Once()
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil)
-
- mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(1), nil,
- )
-
- // job completed successfully
- mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil)
- mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusSuccess).Return(job, nil)
- mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
-
- // create primary filestore directory
- tempPrimaryDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- defer os.RemoveAll(tempPrimaryDir)
-
- // create dedicated filestore directory
- tempDedicatedDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- defer os.RemoveAll(tempDedicatedDir)
-
- // setup worker with primary and dedicated filestores.
- worker := &MessageExportWorker{
- jobServer: jobs.NewJobServer(
- &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- FileSettings: model.FileSettings{
- DriverName: model.NewPointer(model.ImageDriverLocal),
- Directory: model.NewPointer(tempPrimaryDir),
- DedicatedExportStore: model.NewPointer(true),
- ExportDriverName: model.NewPointer(model.ImageDriverLocal),
- ExportDirectory: model.NewPointer(tempDedicatedDir),
- },
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- mockStore,
- mockMetrics,
- logger,
- ),
- logger: logger,
- }
-
- // actually execute the code under test
- worker.DoJob(job)
-
- // ensure no primary filestore files exist
- files, err := os.ReadDir(tempPrimaryDir)
- require.NoError(t, err)
- assert.Zero(t, len(files))
-
- // ensure some dedicated filestore files exist
- files, err = os.ReadDir(tempDedicatedDir)
- require.NoError(t, err)
- assert.NotZero(t, len(files))
-}
-
-func TestDoJobCancel(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
-
- mockStore := &storetest.Store{}
- t.Cleanup(func() { mockStore.AssertExpectations(t) })
- mockMetrics := &mocks.MetricsInterface{}
- t.Cleanup(func() { mockMetrics.AssertExpectations(t) })
-
- job := &model.Job{
- Id: st.NewTestID(),
- CreateAt: model.GetMillis(),
- Status: model.JobStatusPending,
- Type: model.JobTypeMessageExport,
- }
-
- tempDir, err := os.MkdirTemp("", "")
- require.NoError(t, err)
- t.Cleanup(func() { os.RemoveAll(tempDir) })
-
- impl := MessageExportJobInterfaceImpl{
- Server: &app.Server{
- Jobs: jobs.NewJobServer(
- &testutils.StaticConfigService{
- Cfg: &model.Config{
- // mock config
- FileSettings: model.FileSettings{
- DriverName: model.NewPointer(model.ImageDriverLocal),
- Directory: model.NewPointer(tempDir),
- },
- MessageExportSettings: model.MessageExportSettings{
- EnableExport: model.NewPointer(true),
- ExportFormat: model.NewPointer(model.ComplianceExportTypeActiance),
- DailyRunTime: model.NewPointer("01:00"),
- ExportFromTimestamp: model.NewPointer(int64(0)),
- BatchSize: model.NewPointer(10000),
- ChannelBatchSize: model.NewPointer(100),
- ChannelHistoryBatchSize: model.NewPointer(100),
- },
- },
- },
- mockStore,
- mockMetrics,
- logger,
- ),
- },
- }
- worker, ok := impl.MakeWorker().(*MessageExportWorker)
- require.True(t, ok)
-
- retJob := *job
- retJob.Status = model.JobStatusInProgress
-
- // Claim job succeeds
- mockStore.JobStore.
- On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).
- Return(&retJob, nil)
- mockMetrics.On("IncrementJobActive", model.JobTypeMessageExport)
-
- // No previous job, data will be loaded from config
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return(nil, errors.New("test"))
-
- // Job updates the system console UI, once for getting channels, once for getting activity
- mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil).Times(2)
-
- // a few calls pass
- mockStore.ChannelMemberHistoryStore.On("GetChannelsWithActivityDuring", mock.Anything, mock.Anything).
- Return([]string{"channel-id"}, nil)
- mockStore.ChannelStore.On("GetMany", []string{"channel-id"}, true).
- Return(model.ChannelList{{
- Id: "channel-id",
- DisplayName: "channel-display-name",
- Name: "channel-name",
- Type: model.ChannelTypeDirect,
- }}, nil)
- mockStore.ChannelMemberHistoryStore.On("GetUsersInChannelDuring", mock.Anything, mock.Anything, []string{"channel-id"}).Return([]*model.ChannelMemberHistoryResult{}, nil)
-
- cancelled := make(chan struct{})
- // Cancel the worker and return an error
- mockStore.ComplianceStore.On("MessageExport", mock.Anything, mock.AnythingOfType("model.MessageExportCursor"), 10001).Run(func(args tmock.Arguments) {
- worker.cancel()
-
- rctx, ok := args.Get(0).(request.CTX)
- require.True(t, ok)
- assert.Error(t, rctx.Context().Err())
- assert.ErrorIs(t, rctx.Context().Err(), context.Canceled)
-
- cancelled <- struct{}{}
- }).Return(
- nil, model.MessageExportCursor{}, context.Canceled,
- )
-
- mockStore.PostStore.On("AnalyticsPostCount", mock.Anything).Return(
- int64(shared.EstimatedPostCount), nil,
- )
-
- // Job marked as pending
- mockStore.JobStore.On("UpdateStatus", job.Id, model.JobStatusPending).Return(job, nil)
- mockMetrics.On("DecrementJobActive", model.JobTypeMessageExport)
-
- go worker.Run()
-
- worker.JobChannel() <- *job
-
- // Wait for the cancelation
- <-cancelled
-
- // Cleanup
- worker.Stop()
-}
-
-func TestGetPreviousJobNoJobs(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- // Mock the job store to return empty jobs list
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{}, nil).Once()
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- },
- logger: logger,
- }
-
- rctx := request.EmptyContext(logger)
- job, err := worker.getPreviousNonCliJob(rctx)
-
- require.NoError(t, err)
- assert.Nil(t, job, "Expected nil job when no jobs are returned")
-}
-
-func TestGetPreviousJobOneRegularJob(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- regularJob := &model.Job{
- Id: st.NewTestID(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{},
- }
-
- // Mock the job store to return one regular job
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{regularJob}, nil).Once()
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- },
- logger: logger,
- }
-
- rctx := request.EmptyContext(logger)
- job, err := worker.getPreviousNonCliJob(rctx)
-
- require.NoError(t, err)
- assert.Equal(t, regularJob.Id, job.Id, "Expected to get the regular job")
-}
-
-func TestGetPreviousJobOneMmctlJob(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- mmctlJob := &model.Job{
- Id: st.NewTestID(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{shared.JobDataInitiatedBy: "mmctl"},
- }
-
- // Mock the job store to return only mmctl jobs (4 jobs, not a full page)
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return([]*model.Job{mmctlJob, mmctlJob, mmctlJob, mmctlJob}, nil).Once()
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- },
- logger: logger,
- }
-
- rctx := request.EmptyContext(logger)
- job, err := worker.getPreviousNonCliJob(rctx)
-
- require.NoError(t, err)
- assert.Nil(t, job, "Expected nil job when only mmctl jobs are found")
-}
-
-func TestGetPreviousJobManyJobs(t *testing.T) {
- logger := mlog.CreateConsoleTestLogger(t)
- mockStore := &storetest.Store{}
- defer mockStore.AssertExpectations(t)
-
- // Create DefaultPageSize mmctl jobs for first page
- firstPageJobs := make([]*model.Job, DefaultPreviousJobPageSize)
- for i := range DefaultPreviousJobPageSize {
- firstPageJobs[i] = &model.Job{
- Id: st.NewTestID(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{shared.JobDataInitiatedBy: "mmctl"},
- }
- }
-
- // Create DefaultPageSize mmctl jobs for second page
- secondPageJobs := make([]*model.Job, DefaultPreviousJobPageSize)
- for i := range DefaultPreviousJobPageSize {
- secondPageJobs[i] = &model.Job{
- Id: st.NewTestID(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{shared.JobDataInitiatedBy: "mmctl"},
- }
- }
-
- // Create 1 regular job for the third page (last job)
- regularJob := &model.Job{
- Id: st.NewTestID(),
- Status: model.JobStatusSuccess,
- Type: model.JobTypeMessageExport,
- Data: map[string]string{},
- }
- thirdPageJobs := []*model.Job{regularJob}
-
- // Mock the job store to return the jobs in pages
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 0, DefaultPreviousJobPageSize).Return(firstPageJobs, nil).Once()
-
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 1*DefaultPreviousJobPageSize, DefaultPreviousJobPageSize).Return(secondPageJobs, nil).Once()
-
- mockStore.JobStore.On("GetAllByTypesAndStatusesPage", mock.Anything,
- []string{model.JobTypeMessageExport},
- []string{model.JobStatusWarning, model.JobStatusSuccess},
- 2*DefaultPreviousJobPageSize, DefaultPreviousJobPageSize).Return(thirdPageJobs, nil).Once()
-
- worker := &MessageExportWorker{
- jobServer: &jobs.JobServer{
- Store: mockStore,
- },
- logger: logger,
- }
-
- rctx := request.EmptyContext(logger)
- job, err := worker.getPreviousNonCliJob(rctx)
-
- require.NoError(t, err)
- assert.NotNil(t, job)
- assert.Equal(t, regularJob.Id, job.Id, "Expected to find the regular job at the end")
-}