Commit 2de6c539 authored by George Goldberg's avatar George Goldberg Committed by Christopher Speller

Move Slack Import to App Layer. (#5135)

parent 6097f937
......@@ -5,49 +5,20 @@ package api
import (
"bytes"
"image"
"image/color"
"image/draw"
_ "image/gif"
"image/jpeg"
"io"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
l4g "github.com/alecthomas/log4go"
"github.com/disintegration/imaging"
"github.com/gorilla/mux"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
)
const (
/*
EXIF Image Orientations
1 2 3 4 5 6 7 8
888888 888888 88 88 8888888888 88 88 8888888888
88 88 88 88 88 88 88 88 88 88 88 88
8888 8888 8888 8888 88 8888888888 8888888888 88
88 88 88 88
88 88 888888 888888
*/
Upright = 1
UprightMirrored = 2
UpsideDown = 3
UpsideDownMirrored = 4
RotatedCWMirrored = 5
RotatedCCW = 6
RotatedCCWMirrored = 7
RotatedCW = 8
)
func InitFile() {
l4g.Debug(utils.T("api.file.init.debug"))
......@@ -119,7 +90,7 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
io.Copy(buf, file)
data := buf.Bytes()
info, err := doUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
info, err := app.DoUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
if err != nil {
c.Err = err
return
......@@ -138,169 +109,11 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
}
}
handleImages(previewPathList, thumbnailPathList, imageDataList)
app.HandleImages(previewPathList, thumbnailPathList, imageDataList)
w.Write([]byte(resStruct.ToJson()))
}
func doUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
if err != nil {
err.StatusCode = http.StatusBadRequest
return nil, err
}
info.Id = model.NewId()
info.CreatorId = userId
pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
// Check dimensions before loading the whole thing into memory later on
if info.Width*info.Height > model.MaxImageSize {
err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
err.StatusCode = http.StatusBadRequest
return nil, err
}
nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := app.WriteFile(data, info.Path); err != nil {
return nil, err
}
if result := <-app.Srv.Store.FileInfo().Save(info); result.Err != nil {
return nil, result.Err
}
return info, nil
}
func handleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
for i, data := range fileData {
go func(i int, data []byte) {
img, width, height := prepareImage(fileData[i])
if img != nil {
go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
go generatePreviewImage(*img, previewPathList[i], width)
}
}(i, data)
}
}
func prepareImage(fileData []byte) (*image.Image, int, int) {
// Decode image bytes into Image object
img, imgType, err := image.Decode(bytes.NewReader(fileData))
if err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
return nil, 0, 0
}
width := img.Bounds().Dx()
height := img.Bounds().Dy()
// Fill in the background of a potentially-transparent png file as white
if imgType == "png" {
dst := image.NewRGBA(img.Bounds())
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
img = dst
}
// Flip the image to be upright
orientation, _ := getImageOrientation(fileData)
switch orientation {
case UprightMirrored:
img = imaging.FlipH(img)
case UpsideDown:
img = imaging.Rotate180(img)
case UpsideDownMirrored:
img = imaging.FlipV(img)
case RotatedCWMirrored:
img = imaging.Transpose(img)
case RotatedCCW:
img = imaging.Rotate270(img)
case RotatedCCWMirrored:
img = imaging.Transverse(img)
case RotatedCW:
img = imaging.Rotate90(img)
}
return &img, width, height
}
func getImageOrientation(imageData []byte) (int, error) {
if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
return Upright, err
} else {
if tag, err := exifData.Get("Orientation"); err != nil {
return Upright, err
} else {
orientation, err := tag.Int(0)
if err != nil {
return Upright, err
} else {
return orientation, nil
}
}
}
}
func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
imgWidth := float64(width)
imgHeight := float64(height)
var thumbnail image.Image
if imgHeight < thumbHeight && imgWidth < thumbWidth {
thumbnail = img
} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
} else {
thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
return
}
if err := app.WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
}
func generatePreviewImage(img image.Image, previewPath string, width int) {
var preview image.Image
if width > int(utils.Cfg.FileSettings.PreviewWidth) {
preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
} else {
preview = img
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
return
}
if err := app.WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
info, err := getFileInfoForRequest(c, r, true)
if err != nil {
......
......@@ -516,7 +516,7 @@ func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
switch importFrom {
case "slack":
var err *model.AppError
if err, log = SlackImport(fileData, fileSize, c.TeamId); err != nil {
if err, log = app.SlackImport(fileData, fileSize, c.TeamId); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusBadRequest
}
......
......@@ -5,11 +5,17 @@ package app
import (
"bytes"
_ "image/gif"
"crypto/sha256"
"encoding/base64"
"fmt"
"image"
"image/color"
"image/draw"
"image/jpeg"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
......@@ -20,8 +26,33 @@ import (
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"github.com/disintegration/imaging"
s3 "github.com/minio/minio-go"
"github.com/rwcarlsen/goexif/exif"
_ "golang.org/x/image/bmp"
)
const (
/*
EXIF Image Orientations
1 2 3 4 5 6 7 8
888888 888888 88 88 8888888888 88 88 8888888888
88 88 88 88 88 88 88 88 88 88 88 88
8888 8888 8888 8888 88 8888888888 8888888888 88
88 88 88 88
88 88 888888 888888
*/
Upright = 1
UprightMirrored = 2
UpsideDown = 3
UpsideDownMirrored = 4
RotatedCWMirrored = 5
RotatedCCW = 6
RotatedCCWMirrored = 7
RotatedCW = 8
MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image
)
func ReadFile(path string) ([]byte, *model.AppError) {
......@@ -338,3 +369,161 @@ func GeneratePublicLinkHash(fileId, salt string) string {
return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
}
func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
info, err := model.GetInfoForBytes(filename, data)
if err != nil {
err.StatusCode = http.StatusBadRequest
return nil, err
}
info.Id = model.NewId()
info.CreatorId = userId
pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
info.Path = pathPrefix + filename
if info.IsImage() {
// Check dimensions before loading the whole thing into memory later on
if info.Width*info.Height > MaxImageSize {
err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
err.StatusCode = http.StatusBadRequest
return nil, err
}
nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
}
if err := WriteFile(data, info.Path); err != nil {
return nil, err
}
if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil {
return nil, result.Err
}
return info, nil
}
func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
for i, data := range fileData {
go func(i int, data []byte) {
img, width, height := prepareImage(fileData[i])
if img != nil {
go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
go generatePreviewImage(*img, previewPathList[i], width)
}
}(i, data)
}
}
func prepareImage(fileData []byte) (*image.Image, int, int) {
// Decode image bytes into Image object
img, imgType, err := image.Decode(bytes.NewReader(fileData))
if err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
return nil, 0, 0
}
width := img.Bounds().Dx()
height := img.Bounds().Dy()
// Fill in the background of a potentially-transparent png file as white
if imgType == "png" {
dst := image.NewRGBA(img.Bounds())
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
img = dst
}
// Flip the image to be upright
orientation, _ := getImageOrientation(fileData)
switch orientation {
case UprightMirrored:
img = imaging.FlipH(img)
case UpsideDown:
img = imaging.Rotate180(img)
case UpsideDownMirrored:
img = imaging.FlipV(img)
case RotatedCWMirrored:
img = imaging.Transpose(img)
case RotatedCCW:
img = imaging.Rotate270(img)
case RotatedCCWMirrored:
img = imaging.Transverse(img)
case RotatedCW:
img = imaging.Rotate90(img)
}
return &img, width, height
}
func getImageOrientation(imageData []byte) (int, error) {
if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
return Upright, err
} else {
if tag, err := exifData.Get("Orientation"); err != nil {
return Upright, err
} else {
orientation, err := tag.Int(0)
if err != nil {
return Upright, err
} else {
return orientation, nil
}
}
}
}
func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
imgWidth := float64(width)
imgHeight := float64(height)
var thumbnail image.Image
if imgHeight < thumbHeight && imgWidth < thumbWidth {
thumbnail = img
} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
} else {
thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
return
}
if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
return
}
}
func generatePreviewImage(img image.Image, previewPath string, width int) {
var preview image.Image
if width > int(utils.Cfg.FileSettings.PreviewWidth) {
preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
} else {
preview = img
}
buf := new(bytes.Buffer)
if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
return
}
if err := WriteFile(buf.Bytes(), previewPath); err != nil {
l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
return
}
}
// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"bytes"
......@@ -10,7 +10,6 @@ import (
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
......@@ -35,12 +34,12 @@ func ImportPost(post *model.Post) {
post.Hashtags, _ = model.ParseHashtags(post.Message)
if result := <-app.Srv.Store.Post().Save(post); result.Err != nil {
if result := <-Srv.Store.Post().Save(post); result.Err != nil {
l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message)
}
for _, fileId := range post.FileIds {
if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err)
}
}
......@@ -56,17 +55,17 @@ func ImportUser(team *model.Team, user *model.User) *model.User {
user.Roles = model.ROLE_SYSTEM_USER.Id
if result := <-app.Srv.Store.User().Save(user); result.Err != nil {
if result := <-Srv.Store.User().Save(user); result.Err != nil {
l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err)
return nil
} else {
ruser := result.Data.(*model.User)
if cresult := <-app.Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err)
}
if err := app.JoinUserToTeam(team, user); err != nil {
if err := JoinUserToTeam(team, user); err != nil {
l4g.Error(utils.T("api.import.import_user.join_team.error"), err)
}
......@@ -75,7 +74,7 @@ func ImportUser(team *model.Team, user *model.User) *model.User {
}
func ImportChannel(channel *model.Channel) *model.Channel {
if result := <-app.Srv.Store.Channel().Save(channel); result.Err != nil {
if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
return nil
} else {
sc := result.Data.(*model.Channel)
......@@ -89,7 +88,7 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string,
io.Copy(buf, file)
data := buf.Bytes()
fileInfo, err := doUploadFile(teamId, channelId, userId, fileName, data)
fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data)
if err != nil {
return nil, err
}
......
// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"archive/zip"
......@@ -16,7 +16,6 @@ import (
"unicode/utf8"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
......@@ -138,7 +137,7 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
// Need the team
var team *model.Team
if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
return addedUsers
} else {
......@@ -160,10 +159,10 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
password := model.NewId()
// Check for email conflict and use existing user if found
if result := <-app.Srv.Store.User().GetByEmail(email); result.Err == nil {
if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil {
existingUser := result.Data.(*model.User)
addedUsers[sUser.Id] = existingUser
if err := app.JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
if err := JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
} else {
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
......@@ -192,7 +191,7 @@ func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map
func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
var team *model.Team
if result := <-app.Srv.Store.Team().Get(teamId); result.Err != nil {
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
return nil
} else {
......@@ -245,7 +244,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
}
ImportPost(&newPost)
for _, fileId := range newPost.FileIds {
if result := <-app.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
}
}
......@@ -413,7 +412,7 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
}
func deactivateSlackBotUser(user *model.User) {
_, err := app.UpdateActive(user, false)
_, err := UpdateActive(user, false)
if err != nil {
l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
}
......@@ -424,7 +423,7 @@ func addSlackUsersToChannel(members []string, users map[string]*model.User, chan
if user, ok := users[member]; !ok {
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
} else {
if _, err := app.AddUserToChannel(user, channel); err != nil {
if _, err := AddUserToChannel(user, channel); err != nil {
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
}
}
......@@ -474,7 +473,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str
mChannel := ImportChannel(&newChannel)
if mChannel == nil {
// Maybe it already exists?
if result := <-app.Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
continue
......@@ -609,7 +608,7 @@ func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model
deactivateSlackBotUser(botUser)
}
app.InvalidateAllCaches()
InvalidateAllCaches()
log.WriteString(utils.T("api.slackimport.slack_import.notes"))
log.WriteString("=======\r\n\r\n")
......
// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api
package app
import (
"github.com/mattermost/platform/model"
......
......@@ -6,7 +6,7 @@ import (
"errors"
"os"
"github.com/mattermost/platform/api"
"github.com/mattermost/platform/app"
"github.com/spf13/cobra"
)
......@@ -54,7 +54,7 @@ func slackImportCmdF(cmd *cobra.Command, args []string) error {
CommandPrettyPrintln("Running Slack Import. This may take a long time for large teams or teams with many messages.")
api.SlackImport(fileReader, fileInfo.Size(), team.Id)
app.SlackImport(fileReader, fileInfo.Size(), team.Id)
CommandPrettyPrintln("Finished Slack Import.")
......
......@@ -1075,7 +1075,7 @@ func cmdSlackImport() {
fmt.Fprintln(os.Stdout, "Running Slack Import. This may take a long time for large teams or teams with many messages.")
api.SlackImport(fileReader, fileInfo.Size(), team.Id)
app.SlackImport(fileReader, fileInfo.Size(), team.Id)
flushLogAndExit(0)
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment