Commit 91fe8bb2 authored by Joram Wilander's avatar Joram Wilander Committed by GitHub

Implement upload and get file endpoints for APIv4 (#5396)

* Implement POST /files endpoint for APIv4

* Implement GET /files/{file_id} endpoint for APIv4
parent 4e7dbc3b
......@@ -71,6 +71,7 @@ web/sass-files/sass/.sass-cache/
data/*
webapp/data/*
api/data/*
api4/data/*
enterprise
......
......@@ -4,9 +4,6 @@
package api
import (
"bytes"
_ "image/gif"
"io"
"net/http"
"net/url"
"strconv"
......@@ -16,7 +13,6 @@ import (
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
_ "golang.org/x/image/bmp"
)
func InitFile() {
......@@ -35,12 +31,6 @@ func InitFile() {
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
if len(utils.Cfg.FileSettings.DriverName) == 0 {
c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "")
c.Err.StatusCode = http.StatusNotImplemented
return
}
if r.ContentLength > *utils.Cfg.FileSettings.MaxFileSize {
c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.too_large.app_error", nil, "")
c.Err.StatusCode = http.StatusRequestEntityTooLarge
......@@ -70,48 +60,12 @@ func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
resStruct := &model.FileUploadResponse{
FileInfos: []*model.FileInfo{},
ClientIds: []string{},
}
previewPathList := []string{}
thumbnailPathList := []string{}
imageDataList := [][]byte{}
for i, fileHeader := range m.File["files"] {
file, fileErr := fileHeader.Open()
defer file.Close()
if fileErr != nil {
http.Error(w, fileErr.Error(), http.StatusInternalServerError)
return
}
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
data := buf.Bytes()
info, err := app.DoUploadFile(c.TeamId, channelId, c.Session.UserId, fileHeader.Filename, data)
if err != nil {
c.Err = err
return
}
if info.PreviewPath != "" || info.ThumbnailPath != "" {
previewPathList = append(previewPathList, info.PreviewPath)
thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath)
imageDataList = append(imageDataList, data)
}
resStruct.FileInfos = append(resStruct.FileInfos, info)
if len(m.Value["client_ids"]) > 0 {
resStruct.ClientIds = append(resStruct.ClientIds, m.Value["client_ids"][i])
}
resStruct, err := app.UploadFiles(c.TeamId, channelId, c.Session.UserId, m.File["files"], m.Value["client_ids"])
if err != nil {
c.Err = err
return
}
app.HandleImages(previewPathList, thumbnailPathList, imageDataList)
w.Write([]byte(resStruct.ToJson()))
}
......@@ -239,11 +193,9 @@ func getFileInfoForRequest(c *Context, r *http.Request, requireFileVisible bool)
return nil, NewInvalidParamError("getFileInfoForRequest", "file_id")
}
var info *model.FileInfo
if result := <-app.Srv.Store.FileInfo().Get(fileId); result.Err != nil {
return nil, result.Err
} else {
info = result.Data.(*model.FileInfo)
info, err := app.GetFileInfo(fileId)
if err != nil {
return nil, err
}
// only let users access files visible in a channel, unless they're the one who uploaded the file
......
......@@ -143,6 +143,7 @@ func InitApi(full bool) {
InitTeam()
InitChannel()
InitPost()
InitFile()
InitSystem()
app.Srv.Router.Handle("/api/v4/{anything:.*}", http.HandlerFunc(Handle404))
......
......@@ -4,7 +4,10 @@
package api4
import (
"bytes"
"io"
"net/http"
"os"
"reflect"
"runtime/debug"
"strconv"
......@@ -17,6 +20,8 @@ import (
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/store"
"github.com/mattermost/platform/utils"
s3 "github.com/minio/minio-go"
)
type TestHelper struct {
......@@ -398,3 +403,66 @@ func CheckErrorMessage(t *testing.T, resp *model.Response, errorId string) {
t.Fatal("incorrect error message")
}
}
func readTestFile(name string) ([]byte, error) {
path := utils.FindDir("tests")
file, err := os.Open(path + "/" + name)
if err != nil {
return nil, err
}
defer file.Close()
data := &bytes.Buffer{}
if _, err := io.Copy(data, file); err != nil {
return nil, err
} else {
return data.Bytes(), nil
}
}
func cleanupTestFile(info *model.FileInfo) error {
if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_S3 {
endpoint := utils.Cfg.FileSettings.AmazonS3Endpoint
accessKey := utils.Cfg.FileSettings.AmazonS3AccessKeyId
secretKey := utils.Cfg.FileSettings.AmazonS3SecretAccessKey
secure := *utils.Cfg.FileSettings.AmazonS3SSL
s3Clnt, err := s3.New(endpoint, accessKey, secretKey, secure)
if err != nil {
return err
}
bucket := utils.Cfg.FileSettings.AmazonS3Bucket
if err := s3Clnt.RemoveObject(bucket, info.Path); err != nil {
return err
}
if info.ThumbnailPath != "" {
if err := s3Clnt.RemoveObject(bucket, info.ThumbnailPath); err != nil {
return err
}
}
if info.PreviewPath != "" {
if err := s3Clnt.RemoveObject(bucket, info.PreviewPath); err != nil {
return err
}
}
} else if utils.Cfg.FileSettings.DriverName == model.IMAGE_DRIVER_LOCAL {
if err := os.Remove(utils.Cfg.FileSettings.Directory + info.Path); err != nil {
return err
}
if info.ThumbnailPath != "" {
if err := os.Remove(utils.Cfg.FileSettings.Directory + info.ThumbnailPath); err != nil {
return err
}
}
if info.PreviewPath != "" {
if err := os.Remove(utils.Cfg.FileSettings.Directory + info.PreviewPath); err != nil {
return err
}
}
}
return nil
}
......@@ -382,12 +382,24 @@ func (c *Context) RequirePostId() *Context {
return c
}
func (c *Context) RequireFileId() *Context {
if c.Err != nil {
return c
}
if len(c.Params.FileId) != 26 {
c.SetInvalidUrlParam("file_id")
}
return c
}
func (c *Context) RequireTeamName() *Context {
if c.Err != nil {
return c
}
if !model.IsValidTeamName(c.Params.TeamName){
if !model.IsValidTeamName(c.Params.TeamName) {
c.SetInvalidUrlParam("team_name")
}
......@@ -401,7 +413,7 @@ func (c *Context) RequireChannelName() *Context {
if !model.IsValidChannelIdentifier(c.Params.ChannelName) {
c.SetInvalidUrlParam("channel_name")
}
}
return c
}
......@@ -417,5 +429,3 @@ func (c *Context) RequireEmail() *Context {
return c
}
// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api4
import (
"net/http"
"net/url"
"strconv"
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
const (
FILE_TEAM_ID = "noteam"
)
func InitFile() {
l4g.Debug(utils.T("api.file.init.debug"))
BaseRoutes.Files.Handle("", ApiSessionRequired(uploadFile)).Methods("POST")
BaseRoutes.File.Handle("", ApiSessionRequired(getFile)).Methods("GET")
}
func uploadFile(c *Context, w http.ResponseWriter, r *http.Request) {
if r.ContentLength > *utils.Cfg.FileSettings.MaxFileSize {
c.Err = model.NewLocAppError("uploadFile", "api.file.upload_file.too_large.app_error", nil, "")
c.Err.StatusCode = http.StatusRequestEntityTooLarge
return
}
if err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
m := r.MultipartForm
props := m.Value
if len(props["channel_id"]) == 0 {
c.SetInvalidParam("channel_id")
return
}
channelId := props["channel_id"][0]
if len(channelId) == 0 {
c.SetInvalidParam("channel_id")
return
}
if !app.SessionHasPermissionToChannel(c.Session, channelId, model.PERMISSION_UPLOAD_FILE) {
c.SetPermissionError(model.PERMISSION_UPLOAD_FILE)
return
}
resStruct, err := app.UploadFiles(FILE_TEAM_ID, channelId, c.Session.UserId, m.File["files"], m.Value["client_ids"])
if err != nil {
c.Err = err
return
}
w.WriteHeader(http.StatusCreated)
w.Write([]byte(resStruct.ToJson()))
}
func getFile(c *Context, w http.ResponseWriter, r *http.Request) {
c.RequireFileId()
if c.Err != nil {
return
}
info, err := app.GetFileInfo(c.Params.FileId)
if err != nil {
c.Err = err
return
}
if info.CreatorId != c.Session.UserId && !app.SessionHasPermissionToChannelByPost(c.Session, info.PostId, model.PERMISSION_READ_CHANNEL) {
c.SetPermissionError(model.PERMISSION_READ_CHANNEL)
return
}
if data, err := app.ReadFile(info.Path); err != nil {
c.Err = err
c.Err.StatusCode = http.StatusNotFound
} else if err := writeFileResponse(info.Name, info.MimeType, data, w, r); err != nil {
c.Err = err
return
}
}
func writeFileResponse(filename string, contentType string, bytes []byte, w http.ResponseWriter, r *http.Request) *model.AppError {
w.Header().Set("Cache-Control", "max-age=2592000, public")
w.Header().Set("Content-Length", strconv.Itoa(len(bytes)))
if contentType != "" {
w.Header().Set("Content-Type", contentType)
} else {
w.Header().Del("Content-Type") // Content-Type will be set automatically by the http writer
}
w.Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"; filename*=UTF-8''"+url.QueryEscape(filename))
// prevent file links from being embedded in iframes
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Content-Security-Policy", "Frame-ancestors 'none'")
w.Write(bytes)
return nil
}
// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package api4
import (
"fmt"
"testing"
"time"
"github.com/mattermost/platform/app"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
)
func TestUploadFile(t *testing.T) {
th := Setup().InitBasic().InitSystemAdmin()
defer TearDown()
Client := th.Client
user := th.BasicUser
channel := th.BasicChannel
var uploadInfo *model.FileInfo
var data []byte
var err error
if data, err = readTestFile("test.png"); err != nil {
t.Fatal(err)
} else if fileResp, resp := Client.UploadFile(data, channel.Id, "test.png"); resp.Error != nil {
t.Fatal(resp.Error)
} else if len(fileResp.FileInfos) != 1 {
t.Fatal("should've returned a single file infos")
} else {
uploadInfo = fileResp.FileInfos[0]
}
// The returned file info from the upload call will be missing some fields that will be stored in the database
if uploadInfo.CreatorId != user.Id {
t.Fatal("file should be assigned to user")
} else if uploadInfo.PostId != "" {
t.Fatal("file shouldn't have a post")
} else if uploadInfo.Path != "" {
t.Fatal("file path should not be set on returned info")
} else if uploadInfo.ThumbnailPath != "" {
t.Fatal("file thumbnail path should not be set on returned info")
} else if uploadInfo.PreviewPath != "" {
t.Fatal("file preview path should not be set on returned info")
}
var info *model.FileInfo
if result := <-app.Srv.Store.FileInfo().Get(uploadInfo.Id); result.Err != nil {
t.Fatal(result.Err)
} else {
info = result.Data.(*model.FileInfo)
}
if info.Id != uploadInfo.Id {
t.Fatal("file id from response should match one stored in database")
} else if info.CreatorId != user.Id {
t.Fatal("file should be assigned to user")
} else if info.PostId != "" {
t.Fatal("file shouldn't have a post")
} else if info.Path == "" {
t.Fatal("file path should be set in database")
} else if info.ThumbnailPath == "" {
t.Fatal("file thumbnail path should be set in database")
} else if info.PreviewPath == "" {
t.Fatal("file preview path should be set in database")
}
// This also makes sure that the relative path provided above is sanitized out
expectedPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test.png", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.Path != expectedPath {
t.Logf("file is saved in %v", info.Path)
t.Fatalf("file should've been saved in %v", expectedPath)
}
expectedThumbnailPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_thumb.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.ThumbnailPath != expectedThumbnailPath {
t.Logf("file thumbnail is saved in %v", info.ThumbnailPath)
t.Fatalf("file thumbnail should've been saved in %v", expectedThumbnailPath)
}
expectedPreviewPath := fmt.Sprintf("teams/%v/channels/%v/users/%v/%v/test_preview.jpg", FILE_TEAM_ID, channel.Id, user.Id, info.Id)
if info.PreviewPath != expectedPreviewPath {
t.Logf("file preview is saved in %v", info.PreviewPath)
t.Fatalf("file preview should've been saved in %v", expectedPreviewPath)
}
// Wait a bit for files to ready
time.Sleep(2 * time.Second)
if err := cleanupTestFile(info); err != nil {
t.Fatal(err)
}
_, resp := Client.UploadFile(data, model.NewId(), "test.png")
CheckForbiddenStatus(t, resp)
_, resp = th.SystemAdminClient.UploadFile(data, channel.Id, "test.png")
CheckNoError(t, resp)
}
func TestGetFile(t *testing.T) {
th := Setup().InitBasic().InitSystemAdmin()
defer TearDown()
Client := th.Client
channel := th.BasicChannel
if utils.Cfg.FileSettings.DriverName == "" {
t.Skip("skipping because no file driver is enabled")
}
fileId := ""
var sent []byte
var err error
if sent, err = readTestFile("test.png"); err != nil {
t.Fatal(err)
} else {
fileResp, resp := Client.UploadFile(sent, channel.Id, "test.png")
CheckNoError(t, resp)
fileId = fileResp.FileInfos[0].Id
}
data, resp := Client.GetFile(fileId)
CheckNoError(t, resp)
if data == nil || len(data) == 0 {
t.Fatal("should not be empty")
}
for i := range data {
if data[i] != sent[i] {
t.Fatal("received file didn't match sent one")
}
}
_, resp = Client.GetFile("junk")
CheckBadRequestStatus(t, resp)
_, resp = Client.GetFile(model.NewId())
CheckNotFoundStatus(t, resp)
Client.Logout()
_, resp = Client.GetFile(fileId)
CheckUnauthorizedStatus(t, resp)
_, resp = th.SystemAdminClient.GetFile(fileId)
CheckNoError(t, resp)
}
......@@ -15,6 +15,7 @@ import (
"image/jpeg"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/url"
"os"
......@@ -370,6 +371,54 @@ func GeneratePublicLinkHash(fileId, salt string) string {
return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
}
func UploadFiles(teamId string, channelId string, userId string, fileHeaders []*multipart.FileHeader, clientIds []string) (*model.FileUploadResponse, *model.AppError) {
if len(utils.Cfg.FileSettings.DriverName) == 0 {
return nil, model.NewAppError("uploadFile", "api.file.upload_file.storage.app_error", nil, "", http.StatusNotImplemented)
}
resStruct := &model.FileUploadResponse{
FileInfos: []*model.FileInfo{},
ClientIds: []string{},
}
previewPathList := []string{}
thumbnailPathList := []string{}
imageDataList := [][]byte{}
for i, fileHeader := range fileHeaders {
file, fileErr := fileHeader.Open()
defer file.Close()
if fileErr != nil {
return nil, model.NewAppError("UploadFiles", "api.file.upload_file.bad_parse.app_error", nil, fileErr.Error(), http.StatusBadRequest)
}
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
data := buf.Bytes()
info, err := DoUploadFile(teamId, channelId, userId, fileHeader.Filename, data)
if err != nil {
return nil, err
}
if info.PreviewPath != "" || info.ThumbnailPath != "" {
previewPathList = append(previewPathList, info.PreviewPath)
thumbnailPathList = append(thumbnailPathList, info.ThumbnailPath)
imageDataList = append(imageDataList, data)
}
resStruct.FileInfos = append(resStruct.FileInfos, info)
if len(clientIds) > 0 {
resStruct.ClientIds = append(resStruct.ClientIds, clientIds[i])
}
}
HandleImages(previewPathList, thumbnailPathList, imageDataList)
return resStruct, nil
}
func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
filename := filepath.Base(rawFilename)
......@@ -527,3 +576,11 @@ func generatePreviewImage(img image.Image, previewPath string, width int) {
return
}
}
func GetFileInfo(fileId string) (*model.FileInfo, *model.AppError) {
if result := <-Srv.Store.FileInfo().Get(fileId); result.Err != nil {
return nil, result.Err
} else {
return result.Data.(*model.FileInfo), nil
}
}
......@@ -1165,6 +1165,10 @@
"id": "api.file.upload_file.too_large.app_error",
"translation": "Unable to upload file. File is too large."
},
{
"id": "api.file.upload_file.bad_parse.app_error",
"translation": "Unable to upload file. Header cannot be parsed."
},
{
"id": "api.file.write_file.configured.app_error",
"translation": "File storage not configured properly. Please configure for either S3 or local server file storage."
......@@ -3519,6 +3523,10 @@
"id": "model.channel_member.is_valid.user_id.app_error",
"translation": "Invalid user id"
},
{
"id": "model.client.read_file.app_error",
"translation": "We encountered an error while reading the file"
},
{
"id": "model.client.connecting.app_error",
"translation": "We encountered an error while connecting to the server"
......
......@@ -4,7 +4,11 @@
package model
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"strings"
)
......@@ -112,6 +116,14 @@ func (c *Client4) GetPostRoute(postId string) string {
return fmt.Sprintf(c.GetPostsRoute()+"/%v", postId)
}
func (c *Client4) GetFilesRoute() string {
return fmt.Sprintf("/files")
}
func (c *Client4) GetFileRoute(fileId string) string {
return fmt.Sprintf(c.GetFilesRoute()+"/%v", fileId)
}
func (c *Client4) GetSystemRoute() string {
return fmt.Sprintf("/system")
}
......@@ -156,6 +168,25 @@ func (c *Client4) DoApiRequest(method, url, data, etag string) (*http.Response,
}
}
func (c *Client4) DoUploadFile(url string, data []byte, contentType string) (*FileUploadResponse, *Response) {
rq, _ := http.NewRequest("POST", c.ApiUrl+url, bytes.NewReader(data))
rq.Header.Set("Content-Type", contentType)
rq.Close = true
if len(c.AuthToken) > 0 {
rq.Header.Set(HEADER_AUTH, c.AuthType+" "+c.AuthToken)
}
if rp, err := c.HttpClient.Do(rq); err != nil {
return nil, &Response{Error: NewAppError(url, "model.client.connecting.app_error", nil, err.Error(), 0)}
} else if rp.StatusCode >= 300 {
return nil, &Response{StatusCode: rp.StatusCode, Error: AppErrorFromJson(rp.Body)}
} else {
defer closeBody(rp)
return FileUploadResponseFromJson(rp.Body), BuildResponse(rp)
}
}
// CheckStatusOK is a convenience function for checking the standard OK response
// from the web service.
func CheckStatusOK(r *http.Response) bool {
......@@ -595,6 +626,46 @@ func (c *Client4) GetPostsForChannel(channelId string, page, perPage int, etag s
}
}
// File Section
// UploadFile will upload a file to a channel, to be later attached to a post.
func (c *Client4) UploadFile(data []byte, channelId string, filename string) (*FileUploadResponse, *Response) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
if part, err := writer.CreateFormFile("files", filename); err != nil {
return nil, &Response{Error: NewAppError("UploadPostAttachment", "model.client.upload_post_attachment.file.app_error", nil, err.Error(), http.StatusBadRequest)}
} else if _, err = io.Copy(part, bytes.NewBuffer(data)); err != nil {
return nil, &Response{Error: NewAppError("UploadPostAttachment", "model.client.upload_post_attachment.file.app_error", nil, err.Error(), http.StatusBadRequest)}
}
if part, err := writer.CreateFormField("channel_id"); err != nil {
return nil, &Response{Error: NewAppError("UploadPostAttachment", "model.client.upload_post_attachment.channel_id.app_error", nil, err.Error(), http.StatusBadRequest)}
} else if _, err = io.Copy(part, strings.NewReader(channelId)); err != nil {
return nil, &Response{Error: NewAppError("UploadPostAttachment", "model.client.upload_post_attachment.channel_id.app_error", nil, err.Error(), http.StatusBadRequest)}
}
if err := writer.Close(); err != nil {
return nil, &Response{Error: NewAppError("UploadPostAttachment", "model.client.upload_post_attachment.writer.app_error", nil, err.Error(), http.StatusBadRequest)}
}
return c.DoUploadFile(c.GetFilesRoute(), body.Bytes(), writer.FormDataContentType())
}
// GetFile gets the bytes for a file by id.
func (c *Client4) GetFile(fileId string) ([]byte, *Response) {