Unverified Commit d9390244 authored by Jesús Espino's avatar Jesús Espino Committed by GitHub

MM-8810: Add CSV Compliance export (#8966)

* MM-8810: Add CSV Compliance export

* Only allowing to schedule actiances export throught the cli

* De-duplicating some code

* Fixes on texts

* Fixes on translations
parent 85267390
......@@ -15,21 +15,48 @@ import (
)
var MessageExportCmd = &cobra.Command{
Use: "export",
Short: "Export data from Mattermost",
Long: "Export data from Mattermost in a format suitable for import into a third-party application",
Example: "export --format=actiance --exportFrom=12345",
RunE: messageExportCmdF,
Use: "export",
Short: "Export data from Mattermost",
Long: "Export data from Mattermost in a format suitable for import into a third-party application",
}
var ScheduleExportCmd = &cobra.Command{
Use: "schedule",
Short: "Schedule an export data job in Mattermost",
Long: "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)",
Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345",
RunE: scheduleExportCmdF,
}
var CsvExportCmd = &cobra.Command{
Use: "csv",
Short: "Export data from Mattermost in CSV format",
Long: "Export data from Mattermost in CSV format",
Example: "export csv --exportFrom=12345",
RunE: buildExportCmdF("csv"),
}
var ActianceExportCmd = &cobra.Command{
Use: "actiance",
Short: "Export data from Mattermost in Actiance format",
Long: "Export data from Mattermost in Actiance format",
Example: "export actiance --exportFrom=12345",
RunE: buildExportCmdF("actiance"),
}
func init() {
MessageExportCmd.Flags().String("format", "actiance", "The format to export data in")
MessageExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
MessageExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data")
ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
MessageExportCmd.AddCommand(ScheduleExportCmd)
MessageExportCmd.AddCommand(CsvExportCmd)
MessageExportCmd.AddCommand(ActianceExportCmd)
RootCmd.AddCommand(MessageExportCmd)
}
func messageExportCmdF(command *cobra.Command, args []string) error {
func scheduleExportCmdF(command *cobra.Command, args []string) error {
a, err := InitDBCommandContextCobra(command)
if err != nil {
return err
......@@ -79,3 +106,32 @@ func messageExportCmdF(command *cobra.Command, args []string) error {
return nil
}
func buildExportCmdF(format string) func(command *cobra.Command, args []string) error {
return func(command *cobra.Command, args []string) error {
a, err := InitDBCommandContextCobra(command)
if err != nil {
return err
}
defer a.Shutdown()
startTime, err := command.Flags().GetInt64("exportFrom")
if err != nil {
return errors.New("exportFrom flag error")
} else if startTime < 0 {
return errors.New("exportFrom must be a positive integer")
}
if a.MessageExport == nil {
CommandPrettyPrintln("MessageExport feature not available")
}
err2 := a.MessageExport.RunExport(format, startTime)
if err2 != nil {
return err2
}
CommandPrettyPrintln("SUCCESS: Your data was exported.")
return nil
}
}
......@@ -24,7 +24,7 @@ func TestMessageExportNotEnabled(t *testing.T) {
defer os.RemoveAll(filepath.Dir(configPath))
// should fail fast because the feature isn't enabled
require.Error(t, RunCommand(t, "--config", configPath, "export"))
require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule"))
}
func TestMessageExportInvalidFormat(t *testing.T) {
......@@ -32,7 +32,7 @@ func TestMessageExportInvalidFormat(t *testing.T) {
defer os.RemoveAll(filepath.Dir(configPath))
// should fail fast because format isn't supported
require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export"))
require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule"))
}
func TestMessageExportNegativeExportFrom(t *testing.T) {
......@@ -40,7 +40,7 @@ func TestMessageExportNegativeExportFrom(t *testing.T) {
defer os.RemoveAll(filepath.Dir(configPath))
// should fail fast because export from must be a valid timestamp
require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export"))
require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule"))
}
func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
......@@ -48,7 +48,7 @@ func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
defer os.RemoveAll(filepath.Dir(configPath))
// should fail fast because timeout seconds must be a positive int
require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export"))
require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule"))
}
func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string {
......
......@@ -56,7 +56,7 @@ func sliceIncludes(vs []string, t string) bool {
func randomPastTime(seconds int) int64 {
now := time.Now()
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0))
return today.Unix() - int64(rand.Intn(seconds*1000))
return (today.Unix() * 1000) - int64(rand.Intn(seconds*1000))
}
func randomEmoji() string {
......
......@@ -11,4 +11,5 @@ import (
type MessageExportInterface interface {
StartSynchronizeJob(ctx context.Context, exportFromTimestamp int64) (*model.Job, *model.AppError)
RunExport(format string, since int64) *model.AppError
}
......@@ -1046,11 +1046,19 @@
},
{
"id": "api.file.read_file.reading_local.app_error",
"translation": "Encountered an error reading from local server storage"
"translation": "Encountered an error reading from local server file storage"
},
{
"id": "api.file.read_file.s3.app_error",
"translation": ""
"translation": "Encountered an error reading from S3 storage"
},
{
"id": "api.file.reader.reading_local.app_error",
"translation": "Encountered an error opening a reader from local server file storage"
},
{
"id": "api.file.reader.s3.app_error",
"translation": "Encountered an error opening a reader from S3 storage"
},
{
"id": "api.file.test_connection.local.connection.app_error",
......@@ -3580,27 +3588,27 @@
},
{
"id": "ent.compliance.bad_export_type.appError",
"translation": ""
"translation": "Unknown output format {{.ExportType}}"
},
{
"id": "ent.compliance.csv.attachment.copy.appError",
"translation": ""
"translation": "Unable to copy the attachment into the zip file."
},
{
"id": "ent.compliance.csv.attachment.export.appError",
"translation": ""
"translation": "Unable to add attachment to the CSV export."
},
{
"id": "ent.compliance.csv.file.creation.appError",
"translation": ""
"translation": "Unable to create temporary CSV export file."
},
{
"id": "ent.compliance.csv.header.export.appError",
"translation": ""
"translation": "Unable to add header to the CSV export."
},
{
"id": "ent.compliance.csv.metadata.export.appError",
"translation": ""
"translation": "Unable to add metadata file to the zip file."
},
{
"id": "ent.compliance.csv.metadata.json.marshalling.appError",
......@@ -4444,7 +4452,7 @@
},
{
"id": "model.config.is_valid.message_export.export_type.app_error",
"translation": "Message export job ExportFormat must be one of either 'actiance' or 'globalrelay'"
"translation": "Message export job ExportFormat must be one of 'actiance', 'csv' or 'globalrelay'"
},
{
"id": "model.config.is_valid.message_export.global_relay.config_missing.app_error",
......
......@@ -156,6 +156,7 @@ const (
TIMEZONE_SETTINGS_DEFAULT_SUPPORTED_TIMEZONES_PATH = "timezones.json"
COMPLIANCE_EXPORT_TYPE_CSV = "csv"
COMPLIANCE_EXPORT_TYPE_ACTIANCE = "actiance"
COMPLIANCE_EXPORT_TYPE_GLOBALRELAY = "globalrelay"
GLOBALRELAY_CUSTOMER_TYPE_A9 = "A9"
......@@ -2366,7 +2367,7 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY) {
} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_CSV) {
return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest)
}
......
......@@ -4,7 +4,12 @@
package model
type MessageExport struct {
TeamId *string
TeamName *string
TeamDisplayName *string
ChannelId *string
ChannelName *string
ChannelDisplayName *string
ChannelType *string
......@@ -12,9 +17,10 @@ type MessageExport struct {
UserEmail *string
Username *string
PostId *string
PostCreateAt *int64
PostMessage *string
PostType *string
PostFileIds StringArray
PostId *string
PostCreateAt *int64
PostMessage *string
PostType *string
PostOriginalId *string
PostFileIds StringArray
}
......@@ -223,13 +223,18 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha
Posts.CreateAt AS PostCreateAt,
Posts.Message AS PostMessage,
Posts.Type AS PostType,
Posts.OriginalId AS PostOriginalId,
Posts.FileIds AS PostFileIds,
Teams.Id AS TeamId,
Teams.Name AS TeamName,
Teams.DisplayName AS TeamDisplayName,
Channels.Id AS ChannelId,
CASE
CASE
WHEN Channels.Type = 'D' THEN 'Direct Message'
WHEN Channels.Type = 'G' THEN 'Group Message'
ELSE Channels.DisplayName
END AS ChannelDisplayName,
Channels.Name AS ChannelName,
Channels.Type AS ChannelType,
Users.Id AS UserId,
Users.Email AS UserEmail,
......@@ -237,6 +242,7 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha
FROM
Posts
LEFT OUTER JOIN Channels ON Posts.ChannelId = Channels.Id
LEFT OUTER JOIN Teams ON Channels.TeamId = Teams.Id
LEFT OUTER JOIN Users ON Posts.UserId = Users.Id
WHERE
Posts.CreateAt > :StartTime AND
......
......@@ -13,6 +13,7 @@ import (
type FileBackend interface {
TestConnection() *model.AppError
Reader(path string) (io.ReadCloser, *model.AppError)
ReadFile(path string) ([]byte, *model.AppError)
CopyFile(oldPath, newPath string) *model.AppError
MoveFile(oldPath, newPath string) *model.AppError
......
......@@ -33,6 +33,14 @@ func (b *LocalFileBackend) TestConnection() *model.AppError {
return nil
}
func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
if f, err := os.Open(filepath.Join(b.directory, path)); err != nil {
return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
return f, nil
}
}
func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) {
if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil {
return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
......
......@@ -82,6 +82,18 @@ func (b *S3FileBackend) TestConnection() *model.AppError {
return nil
}
func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
s3Clnt, err := b.s3New()
if err != nil {
return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
if err != nil {
return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return minioObject, nil
}
func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
s3Clnt, err := b.s3New()
if err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment