Unverified Commit ab99f065 authored by George Goldberg's avatar George Goldberg Committed by GitHub

MM-11781: Basic Data Export Command Line. (#9296)

* MM-11781: Basic Data Export Command Line.

* ChannelStore new unit tests.

* TeamStore new unit tests.

* Unit test for new UserStore function.

* Unit tests for post store new methods.

* Review fixes.

* Fix duplicate command name.
parent 5786b0d6
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"encoding/json"
"io"
"net/http"
"strings"
"github.com/mattermost/mattermost-server/model"
)
func (a *App) BulkExport(writer io.Writer) *model.AppError {
if err := a.ExportVersion(writer); err != nil {
return err
}
if err := a.ExportAllTeams(writer); err != nil {
return err
}
if err := a.ExportAllChannels(writer); err != nil {
return err
}
if err := a.ExportAllUsers(writer); err != nil {
return err
}
if err := a.ExportAllPosts(writer); err != nil {
return err
}
return nil
}
func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError {
b, err := json.Marshal(line)
if err != nil {
return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest)
}
if _, err := writer.Write(append(b, '\n')); err != nil {
return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest)
}
return nil
}
func (a *App) ExportVersion(writer io.Writer) *model.AppError {
version := 1
versionLine := &LineImportData{
Type: "version",
Version: &version,
}
return a.ExportWriteLine(writer, versionLine)
}
func (a *App) ExportAllTeams(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
teams := result.Data.([]*model.TeamForExport)
if len(teams) == 0 {
break
}
for _, team := range teams {
afterId = team.Id
// Skip deleted.
if team.DeleteAt != 0 {
continue
}
teamLine := ImportLineFromTeam(team)
if err := a.ExportWriteLine(writer, teamLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) ExportAllChannels(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
channels := result.Data.([]*model.ChannelForExport)
if len(channels) == 0 {
break
}
for _, channel := range channels {
afterId = channel.Id
// Skip deleted.
if channel.DeleteAt != 0 {
continue
}
channelLine := ImportLineFromChannel(channel)
if err := a.ExportWriteLine(writer, channelLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) ExportAllUsers(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.User().GetAllAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
users := result.Data.([]*model.User)
if len(users) == 0 {
break
}
for _, user := range users {
afterId = user.Id
// Skip deleted.
if user.DeleteAt != 0 {
continue
}
userLine := ImportLineFromUser(user)
// Do the Team Memberships.
members, err := a.buildUserTeamAndChannelMemberships(user.Id)
if err != nil {
return err
}
userLine.User.Teams = members
if err := a.ExportWriteLine(writer, userLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) {
var memberships []UserTeamImportData
result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId)
if result.Err != nil {
return nil, result.Err
}
members := result.Data.([]*model.TeamMemberForExport)
for _, member := range members {
// Skip deleted.
if member.DeleteAt != 0 {
continue
}
memberData := ImportUserTeamDataFromTeamMember(member)
// Do the Channel Memberships.
channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId)
if err != nil {
return nil, err
}
memberData.Channels = channelMembers
memberships = append(memberships, *memberData)
}
return &memberships, nil
}
func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) {
var memberships []UserChannelImportData
result := <-a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId)
if result.Err != nil {
return nil, result.Err
}
members := result.Data.([]*model.ChannelMemberForExport)
for _, member := range members {
memberships = append(memberships, *ImportUserChannelDataFromChannelMember(member))
}
return &memberships, nil
}
func (a *App) ExportAllPosts(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
posts := result.Data.([]*model.PostForExport)
if len(posts) == 0 {
break
}
for _, post := range posts {
afterId = post.Id
// Skip deleted.
if post.DeleteAt != 0 {
continue
}
postLine := ImportLineForPost(post)
// Do the Replies.
replies, err := a.buildPostReplies(post.Id)
if err != nil {
return err
}
postLine.Post.Replies = replies
if err := a.ExportWriteLine(writer, postLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) {
var replies []ReplyImportData
result := <-a.Srv.Store.Post().GetRepliesForExport(postId)
if result.Err != nil {
return nil, result.Err
}
replyPosts := result.Data.([]*model.ReplyForExport)
for _, reply := range replyPosts {
replies = append(replies, *ImportReplyFromPost(reply))
}
return &replies, nil
}
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"github.com/mattermost/mattermost-server/model"
"strings"
)
func ImportLineFromTeam(team *model.TeamForExport) *LineImportData {
return &LineImportData{
Type: "team",
Team: &TeamImportData{
Name: &team.Name,
DisplayName: &team.DisplayName,
Type: &team.Type,
Description: &team.Description,
AllowOpenInvite: &team.AllowOpenInvite,
Scheme: team.SchemeName,
},
}
}
func ImportLineFromChannel(channel *model.ChannelForExport) *LineImportData {
return &LineImportData{
Type: "channel",
Channel: &ChannelImportData{
Team: &channel.TeamName,
Name: &channel.Name,
DisplayName: &channel.DisplayName,
Type: &channel.Type,
Header: &channel.Header,
Purpose: &channel.Purpose,
Scheme: channel.SchemeName,
},
}
}
func ImportLineFromUser(user *model.User) *LineImportData {
// Bulk Importer doesn't accept "empty string" for AuthService.
var authService *string
if user.AuthService != "" {
authService = &user.AuthService
}
return &LineImportData{
Type: "user",
User: &UserImportData{
Username: &user.Username,
Email: &user.Email,
AuthService: authService,
AuthData: user.AuthData,
Nickname: &user.Nickname,
FirstName: &user.FirstName,
LastName: &user.LastName,
Position: &user.Position,
Roles: &user.Roles,
Locale: &user.Locale,
},
}
}
func ImportUserTeamDataFromTeamMember(member *model.TeamMemberForExport) *UserTeamImportData {
rolesList := strings.Fields(member.Roles)
if member.SchemeAdmin {
rolesList = append(rolesList, model.TEAM_ADMIN_ROLE_ID)
}
if member.SchemeUser {
rolesList = append(rolesList, model.TEAM_USER_ROLE_ID)
}
roles := strings.Join(rolesList, " ")
return &UserTeamImportData{
Name: &member.TeamName,
Roles: &roles,
}
}
func ImportUserChannelDataFromChannelMember(member *model.ChannelMemberForExport) *UserChannelImportData {
rolesList := strings.Fields(member.Roles)
if member.SchemeAdmin {
rolesList = append(rolesList, model.CHANNEL_ADMIN_ROLE_ID)
}
if member.SchemeUser {
rolesList = append(rolesList, model.CHANNEL_USER_ROLE_ID)
}
roles := strings.Join(rolesList, " ")
return &UserChannelImportData{
Name: &member.ChannelName,
Roles: &roles,
}
}
func ImportLineForPost(post *model.PostForExport) *LineImportData {
return &LineImportData{
Type: "post",
Post: &PostImportData{
Team: &post.TeamName,
Channel: &post.ChannelName,
User: &post.Username,
Message: &post.Message,
CreateAt: &post.CreateAt,
},
}
}
func ImportReplyFromPost(post *model.ReplyForExport) *ReplyImportData {
return &ReplyImportData{
User: &post.Username,
Message: &post.Message,
CreateAt: &post.CreateAt,
}
}
......@@ -9,24 +9,24 @@ import "github.com/mattermost/mattermost-server/model"
type LineImportData struct {
Type string `json:"type"`
Scheme *SchemeImportData `json:"scheme"`
Team *TeamImportData `json:"team"`
Channel *ChannelImportData `json:"channel"`
User *UserImportData `json:"user"`
Post *PostImportData `json:"post"`
DirectChannel *DirectChannelImportData `json:"direct_channel"`
DirectPost *DirectPostImportData `json:"direct_post"`
Emoji *EmojiImportData `json:"emoji"`
Version *int `json:"version"`
Scheme *SchemeImportData `json:"scheme,omitempty"`
Team *TeamImportData `json:"team,omitempty"`
Channel *ChannelImportData `json:"channel,omitempty"`
User *UserImportData `json:"user,omitempty"`
Post *PostImportData `json:"post,omitempty"`
DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"`
DirectPost *DirectPostImportData `json:"direct_post,omitempty"`
Emoji *EmojiImportData `json:"emoji,omitempty"`
Version *int `json:"version,omitempty"`
}
type TeamImportData struct {
Name *string `json:"name"`
DisplayName *string `json:"display_name"`
Type *string `json:"type"`
Description *string `json:"description"`
AllowOpenInvite *bool `json:"allow_open_invite"`
Scheme *string `json:"scheme"`
Description *string `json:"description,omitempty"`
AllowOpenInvite *bool `json:"allow_open_invite,omitempty"`
Scheme *string `json:"scheme,omitempty"`
}
type ChannelImportData struct {
......@@ -34,38 +34,38 @@ type ChannelImportData struct {
Name *string `json:"name"`
DisplayName *string `json:"display_name"`
Type *string `json:"type"`
Header *string `json:"header"`
Purpose *string `json:"purpose"`
Scheme *string `json:"scheme"`
Header *string `json:"header,omitempty"`
Purpose *string `json:"purpose,omitempty"`
Scheme *string `json:"scheme,omitempty"`
}
type UserImportData struct {
ProfileImage *string `json:"profile_image"`
ProfileImage *string `json:"profile_image,omitempty"`
Username *string `json:"username"`
Email *string `json:"email"`
AuthService *string `json:"auth_service"`
AuthData *string `json:"auth_data"`
Password *string `json:"password"`
AuthData *string `json:"auth_data,omitempty"`
Password *string `json:"password,omitempty"`
Nickname *string `json:"nickname"`
FirstName *string `json:"first_name"`
LastName *string `json:"last_name"`
Position *string `json:"position"`
Roles *string `json:"roles"`
Locale *string `json:"locale"`
UseMarkdownPreview *string `json:"feature_enabled_markdown_preview"`
UseFormatting *string `json:"formatting"`
ShowUnreadSection *string `json:"show_unread_section"`
UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"`
UseFormatting *string `json:"formatting,omitempty"`
ShowUnreadSection *string `json:"show_unread_section,omitempty"`
Teams *[]UserTeamImportData `json:"teams"`
Teams *[]UserTeamImportData `json:"teams,omitempty"`
Theme *string `json:"theme"`
UseMilitaryTime *string `json:"military_time"`
CollapsePreviews *string `json:"link_previews"`
MessageDisplay *string `json:"message_display"`
ChannelDisplayMode *string `json:"channel_display_mode"`
TutorialStep *string `json:"tutorial_step"`
Theme *string `json:"theme,omitempty"`
UseMilitaryTime *string `json:"military_time,omitempty"`
CollapsePreviews *string `json:"link_previews,omitempty"`
MessageDisplay *string `json:"message_display,omitempty"`
ChannelDisplayMode *string `json:"channel_display_mode,omitempty"`
TutorialStep *string `json:"tutorial_step,omitempty"`
NotifyProps *UserNotifyPropsImportData `json:"notify_props"`
NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"`
}
type UserNotifyPropsImportData struct {
......@@ -85,15 +85,15 @@ type UserNotifyPropsImportData struct {
type UserTeamImportData struct {
Name *string `json:"name"`
Roles *string `json:"roles"`
Theme *string `json:"theme"`
Channels *[]UserChannelImportData `json:"channels"`
Theme *string `json:"theme,omitempty"`
Channels *[]UserChannelImportData `json:"channels,omitempty"`
}
type UserChannelImportData struct {
Name *string `json:"name"`
Roles *string `json:"roles"`
NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props"`
Favorite *bool `json:"favorite"`
NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"`
Favorite *bool `json:"favorite,omitempty"`
}
type UserChannelNotifyPropsImportData struct {
......@@ -119,9 +119,9 @@ type ReplyImportData struct {
Message *string `json:"message"`
CreateAt *int64 `json:"create_at"`
FlaggedBy *[]string `json:"flagged_by"`
Reactions *[]ReactionImportData `json:"reactions"`
Attachments *[]AttachmentImportData `json:"attachments"`
FlaggedBy *[]string `json:"flagged_by,omitempty"`
Reactions *[]ReactionImportData `json:"reactions,omitempty"`
Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
}
type PostImportData struct {
......@@ -132,10 +132,10 @@ type PostImportData struct {
Message *string `json:"message"`
CreateAt *int64 `json:"create_at"`
FlaggedBy *[]string `json:"flagged_by"`
Reactions *[]ReactionImportData `json:"reactions"`
Replies *[]ReplyImportData `json:"replies"`
Attachments *[]AttachmentImportData `json:"attachments"`
FlaggedBy *[]string `json:"flagged_by,omitempty"`
Reactions *[]ReactionImportData `json:"reactions,omitempty"`
Replies *[]ReplyImportData `json:"replies,omitempty"`
Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
}
type DirectChannelImportData struct {
......
......@@ -5,6 +5,7 @@ package commands
import (
"errors"
"os"
"context"
......@@ -14,10 +15,10 @@ import (
"github.com/spf13/cobra"
)
var MessageExportCmd = &cobra.Command{
var ExportCmd = &cobra.Command{
Use: "export",
Short: "Export data from Mattermost",
Long: "Export data from Mattermost in a format suitable for import into a third-party application",
Long: "Export data from Mattermost in a format suitable for import into a third-party application or another Mattermost instance",
}
var ScheduleExportCmd = &cobra.Command{
......@@ -44,16 +45,31 @@ var ActianceExportCmd = &cobra.Command{
RunE: buildExportCmdF("actiance"),
}
var BulkExportCmd = &cobra.Command{
Use: "bulk [file]",
Short: "Export bulk data.",
Long: "Export data to a file compatible with the Mattermost Bulk Import format.",
Example: " export bulk bulk_data.json",
RunE: bulkExportCmdF,
}
func init() {
ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data")
ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
MessageExportCmd.AddCommand(ScheduleExportCmd)
MessageExportCmd.AddCommand(CsvExportCmd)
MessageExportCmd.AddCommand(ActianceExportCmd)
RootCmd.AddCommand(MessageExportCmd)
BulkExportCmd.Flags().Bool("all-teams", false, "Export all teams from the server.")
ExportCmd.AddCommand(ScheduleExportCmd)
ExportCmd.AddCommand(CsvExportCmd)
ExportCmd.AddCommand(ActianceExportCmd)
ExportCmd.AddCommand(BulkExportCmd)
RootCmd.AddCommand(ExportCmd)
}
func scheduleExportCmdF(command *cobra.Command, args []string) error {
......@@ -140,3 +156,33 @@ func buildExportCmdF(format string) func(command *cobra.Command, args []string)
return nil
}
}
func bulkExportCmdF(command *cobra.Command, args []string) error {
a, err := InitDBCommandContextCobra(command)
if err != nil {
return err
}
defer a.Shutdown()
allTeams, err := command.Flags().GetBool("all-teams")
if err != nil {
return errors.New("Apply flag error")
}
if !allTeams {
return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.")
}
fileWriter, err := os.Create(args[0])
if err != nil {
return err
}
defer fileWriter.Close()
if err := a.BulkExport(fileWriter); err != nil {
CommandPrettyPrintln(err.Error())
return err
}
return nil
}
......@@ -179,6 +179,14 @@
"id": "api.channel.leave.last_member.app_error",
"translation": "You're the only member left, try removing the Private Channel instead of leaving."
},
{
"id": "app.export.export_write_line.json_marshall.error",
"translation": "An error occured marshalling the JSON data for export."
},
{
"id": "app.export.export_write_line.io_writer.error",
"translation": "An error occurred writing the export data."
},
{
"id": "api.channel.leave.left",
"translation": "%v left the channel."
......
......@@ -57,6 +57,12 @@ type ChannelPatch struct {
Purpose *string `json:"purpose"`
}
type ChannelForExport struct {
Channel
TeamName string
SchemeName *string
}
func (o *Channel) DeepCopy() *Channel {
copy := *o
if copy.SchemeId != nil {
......
......@@ -43,6 +43,11 @@ type ChannelMember struct {
type ChannelMembers []ChannelMember
type ChannelMemberForExport struct {
ChannelMember
ChannelName string
}
func (o *ChannelMembers) ToJson() string {
if b, err := json.Marshal(o); err != nil {
return "[]"
......
......@@ -110,6 +110,19 @@ func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch {
return &copy
}
type PostForExport struct {
Post
TeamName string
ChannelName string
Username string
ReplyCount int
}
type ReplyForExport struct {
Post
Username string
}
type PostForIndexing struct {
Post
TeamId string `json:"team_id"`
......
......@@ -52,6 +52,11 @@ type TeamPatch struct {
AllowOpenInvite *bool `json:"allow_open_invite"`
}
type TeamForExport struct {
Team
SchemeName *string
}
type Invites struct {
Invites []map[string]string `json:"invites"`
}
......
......@@ -26,6 +26,11 @@ type TeamUnread struct {
MentionCount int64 `json:"mention_count"`
}
type TeamMemberForExport struct {
TeamMember
TeamName string
}
func (o *TeamMember) ToJson() string {
b, _ := json.Marshal(o)
return string(b)
......
......@@ -2016,3 +2016,57 @@ func (s SqlChannelStore) IsExperimentalPublicChannelsMaterializationEnabled() bo
// See SqlChannelStoreExperimental
return false
}
func (s SqlChannelStore) GetAllChannelsForExportAfter(limit int, afterId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var data []*model.ChannelForExport
if _, err := s.GetReplica().Select(&data, `
SELECT
Channels.*,
Teams.Name as TeamName,
Schemes.Name as SchemeName
FROM Channels
INNER JOIN
Teams ON Channels.TeamId = Teams.Id
LEFT JOIN
Schemes ON Channels.SchemeId = Schemes.Id
WHERE
Channels.Id > :AfterId
AND Channels.Type IN ('O', 'P')
ORDER BY
Id
LIMIT :Limit`,
map[string]interface{}{"AfterId": afterId, "Limit": limit}); err != nil {
result.Err