Commit 48f16b64 authored by Harrison Healey's avatar Harrison Healey

MM-11272 Added initial post metadata (#9175)

* MM-11272 Added app.PreparePostForClient

* MM-11272 Added app.PreparePostListForClient

* MM-11272 Added EmojiStore.GetMultipleByName

* MM-11272 Added emojis to PreparePostForClient

* MM-11272 Added unit tests for getting reaction counts

* MM-11272 Added unit tests for TestPreparePostForClient

* MM-11272 Added emojis from reactions to Post.Emojis

* MM-11272 Always update post.UpdateAt when reactions change to bust cache

* Fixed merge conflicts

* Moved post metadata-related code into its own file

* Update store mocks

* Fixed typo

* Add missing license headers

* Updated post metadata tests when custom emojis are disabled

* Fix unreliable unit tests

* Fix inconsistent casing in SQL statements

* Fix blank line

* Invalidate store cache after making changes

* Clear post cache synchronously with reactions
parent 2e945e28
......@@ -488,8 +488,13 @@ func getPinnedPosts(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set(model.HEADER_ETAG_SERVER, posts.Etag())
w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson()))
clientPostList, err := c.App.PreparePostListForClient(posts)
if err != nil {
mlog.Error("Failed to prepare posts for getFlaggedPostsForUser response", mlog.Any("err", err))
}
w.Header().Set(model.HEADER_ETAG_SERVER, clientPostList.Etag())
w.Write([]byte(clientPostList.ToJson()))
}
func getPublicChannelsForTeam(c *Context, w http.ResponseWriter, r *http.Request) {
......
......@@ -9,6 +9,7 @@ import (
"strconv"
"time"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
......@@ -67,8 +68,13 @@ func createPost(c *Context, w http.ResponseWriter, r *http.Request) {
c.App.SetStatusOnline(c.Session.UserId, false)
c.App.UpdateLastActivityAtIfNeeded(c.Session)
clientPost, err := c.App.PreparePostForClient(rp)
if err != nil {
mlog.Error("Failed to prepare post for createPost response", mlog.Any("err", err))
}
w.WriteHeader(http.StatusCreated)
w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rp).ToJson()))
w.Write([]byte(clientPost.ToJson()))
}
func createEphemeralPost(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -95,8 +101,13 @@ func createEphemeralPost(c *Context, w http.ResponseWriter, r *http.Request) {
rp := c.App.SendEphemeralPost(ephRequest.UserID, c.App.PostWithProxyRemovedFromImageURLs(ephRequest.Post))
clientPost, err := c.App.PreparePostForClient(rp)
if err != nil {
mlog.Error("Failed to prepare post for createEphemeralPost response", mlog.Any("err", err))
}
w.WriteHeader(http.StatusCreated)
w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rp).ToJson()))
w.Write([]byte(clientPost.ToJson()))
}
func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -165,7 +176,13 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) {
if len(etag) > 0 {
w.Header().Set(model.HEADER_ETAG_SERVER, etag)
}
w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson()))
clientPostList, err := c.App.PreparePostListForClient(list)
if err != nil {
mlog.Error("Failed to prepare posts for getPostsForChannel response", mlog.Any("err", err))
}
w.Write([]byte(clientPostList.ToJson()))
}
func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -198,7 +215,12 @@ func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request)
return
}
w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson()))
clientPostList, err := c.App.PreparePostListForClient(posts)
if err != nil {
mlog.Error("Failed to prepare posts for getFlaggedPostsForUser response", mlog.Any("err", err))
}
w.Write([]byte(clientPostList.ToJson()))
}
func getPost(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -232,12 +254,17 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) {
}
}
post, err = c.App.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare post for getPost response", mlog.Any("err", err))
}
if c.HandleEtag(post.Etag(), "Get Post", w, r) {
return
}
w.Header().Set(model.HEADER_ETAG_SERVER, post.Etag())
w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(post).ToJson()))
w.Write([]byte(post.ToJson()))
}
func deletePost(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -315,8 +342,14 @@ func getPostThread(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
w.Header().Set(model.HEADER_ETAG_SERVER, list.Etag())
w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson()))
clientPostList, err := c.App.PreparePostListForClient(list)
if err != nil {
mlog.Error("Failed to prepare posts for getFlaggedPostsForUser response", mlog.Any("err", err))
}
w.Header().Set(model.HEADER_ETAG_SERVER, clientPostList.Etag())
w.Write([]byte(clientPostList.ToJson()))
}
func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -379,7 +412,12 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
results = model.MakePostSearchResults(c.App.PostListWithProxyAddedToImageURLs(results.PostList), results.Matches)
clientPostList, err := c.App.PreparePostListForClient(results.PostList)
if err != nil {
mlog.Error("Failed to prepare posts for searchPosts response", mlog.Any("err", err))
}
results = model.MakePostSearchResults(clientPostList, results.Matches)
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
w.Write([]byte(results.ToJson()))
......@@ -430,7 +468,12 @@ func updatePost(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rpost).ToJson()))
rpost, err = c.App.PreparePostForClient(rpost)
if err != nil {
mlog.Error("Failed to prepare post for updatePost response", mlog.Any("err", err))
}
w.Write([]byte(rpost.ToJson()))
}
func patchPost(c *Context, w http.ResponseWriter, r *http.Request) {
......@@ -470,7 +513,12 @@ func patchPost(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(patchedPost).ToJson()))
patchedPost, err = c.App.PreparePostForClient(patchedPost)
if err != nil {
mlog.Error("Failed to prepare post for patchPost response", mlog.Any("err", err))
}
w.Write([]byte(patchedPost.ToJson()))
}
func saveIsPinnedPost(c *Context, w http.ResponseWriter, r *http.Request, isPinned bool) {
......
......@@ -390,6 +390,39 @@ func (me *TestHelper) CreateScheme() (*model.Scheme, []*model.Role) {
return scheme, roles
}
func (me *TestHelper) CreateEmoji() *model.Emoji {
utils.DisableDebugLogForTest()
result := <-me.App.Srv.Store.Emoji().Save(&model.Emoji{
CreatorId: me.BasicUser.Id,
Name: model.NewRandomString(10),
})
if result.Err != nil {
panic(result.Err)
}
utils.EnableDebugLogForTest()
return result.Data.(*model.Emoji)
}
func (me *TestHelper) AddReactionToPost(post *model.Post, user *model.User, emojiName string) *model.Reaction {
utils.DisableDebugLogForTest()
reaction, err := me.App.SaveReactionForPost(&model.Reaction{
UserId: user.Id,
PostId: post.Id,
EmojiName: emojiName,
})
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return reaction
}
func (me *TestHelper) TearDown() {
me.App.Shutdown()
os.Remove(me.tempConfigPath)
......
......@@ -185,6 +185,18 @@ func (a *App) GetEmojiByName(emojiName string) (*model.Emoji, *model.AppError) {
return result.Data.(*model.Emoji), nil
}
func (a *App) GetMultipleEmojiByName(names []string) ([]*model.Emoji, *model.AppError) {
if !*a.Config().ServiceSettings.EnableCustomEmoji {
return nil, model.NewAppError("GetMultipleEmojiByName", "api.emoji.disabled.app_error", nil, "", http.StatusNotImplemented)
}
if result := <-a.Srv.Store.Emoji().GetMultipleByName(names); result.Err != nil {
return nil, result.Err
} else {
return result.Data.([]*model.Emoji), nil
}
}
func (a *App) GetEmojiImage(emojiId string) ([]byte, string, *model.AppError) {
result := <-a.Srv.Store.Emoji().Get(emojiId, true)
if result.Err != nil {
......
......@@ -317,8 +317,13 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod
}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
message.Add("channel_type", channel.Type)
message.Add("channel_display_name", notification.GetChannelName(model.SHOW_USERNAME, ""))
message.Add("channel_name", channel.Name)
......
......@@ -301,8 +301,13 @@ func (a *App) SendEphemeralPost(userId string, post *model.Post) *model.Post {
post.Props = model.StringInterface{}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare ephemeral post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
return post
......@@ -423,8 +428,13 @@ func (a *App) PatchPost(postId string, patch *model.PostPatch) (*model.Post, *mo
}
func (a *App) sendUpdatedPostEvent(post *model.Post) {
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare updated post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
}
......@@ -563,8 +573,13 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
return nil, result.Err
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare deleted post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_DELETED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
a.Go(func() {
......@@ -967,13 +982,6 @@ func (a *App) DoPostAction(postId, actionId, userId, selectedOption string) *mod
return nil
}
func (a *App) PostListWithProxyAddedToImageURLs(list *model.PostList) *model.PostList {
if f := a.ImageProxyAdder(); f != nil {
return list.WithRewrittenImageURLs(f)
}
return list
}
func (a *App) PostWithProxyAddedToImageURLs(post *model.Post) *model.Post {
if f := a.ImageProxyAdder(); f != nil {
return post.WithRewrittenImageURLs(f)
......
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
"github.com/mattermost/mattermost-server/model"
)
func (a *App) PreparePostListForClient(originalList *model.PostList) (*model.PostList, *model.AppError) {
list := &model.PostList{
Posts: make(map[string]*model.Post),
Order: originalList.Order,
}
for id, originalPost := range originalList.Posts {
post, err := a.PreparePostForClient(originalPost)
if err != nil {
return originalList, err
}
list.Posts[id] = post
}
return list, nil
}
func (a *App) PreparePostForClient(originalPost *model.Post) (*model.Post, *model.AppError) {
post := originalPost.Clone()
var err *model.AppError
needReactionCounts := post.ReactionCounts == nil
needEmojis := post.Emojis == nil
needImageDimensions := post.ImageDimensions == nil
needOpenGraphData := post.OpenGraphData == nil
var reactions []*model.Reaction
if needReactionCounts || needEmojis {
reactions, err = a.GetReactionsForPost(post.Id)
if err != nil {
return post, err
}
}
if needReactionCounts {
post.ReactionCounts = model.CountReactions(reactions)
}
if post.FileInfos == nil {
fileInfos, err := a.GetFileInfosForPost(post.Id, false)
if err != nil {
return post, err
}
post.FileInfos = fileInfos
}
if needEmojis {
emojis, err := a.getCustomEmojisForPost(post.Message, reactions)
if err != nil {
return post, err
}
post.Emojis = emojis
}
post = a.PostWithProxyAddedToImageURLs(post)
if needImageDimensions || needOpenGraphData {
if needImageDimensions {
post.ImageDimensions = []*model.PostImageDimensions{}
}
if needOpenGraphData {
post.OpenGraphData = []*opengraph.OpenGraph{}
}
// TODO
}
return post, nil
}
func (a *App) getCustomEmojisForPost(message string, reactions []*model.Reaction) ([]*model.Emoji, *model.AppError) {
if !*a.Config().ServiceSettings.EnableCustomEmoji {
// Only custom emoji are returned
return []*model.Emoji{}, nil
}
names := model.EMOJI_PATTERN.FindAllString(message, -1)
for _, reaction := range reactions {
names = append(names, reaction.EmojiName)
}
if len(names) == 0 {
return []*model.Emoji{}, nil
}
names = model.RemoveDuplicateStrings(names)
for i, name := range names {
names[i] = strings.Trim(name, ":")
}
return a.GetMultipleEmojiByName(names)
}
This diff is collapsed.
......@@ -466,7 +466,6 @@ func TestImageProxy(t *testing.T) {
list := model.NewPostList()
list.Posts[post.Id] = post
assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostListWithProxyAddedToImageURLs(list).Posts[post.Id].Message)
assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostWithProxyAddedToImageURLs(post).Message)
assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message)
......
......@@ -6,6 +6,7 @@ package app
import (
"net/http"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
......@@ -42,6 +43,9 @@ func (a *App) SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *m
reaction = result.Data.(*model.Reaction)
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
a.Go(func() {
a.sendReactionEvent(model.WEBSOCKET_EVENT_REACTION_ADDED, reaction, post, true)
})
......@@ -92,6 +96,9 @@ func (a *App) DeleteReactionForPost(reaction *model.Reaction) *model.AppError {
return result.Err
}
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
a.Go(func() {
a.sendReactionEvent(model.WEBSOCKET_EVENT_REACTION_REMOVED, reaction, post, hasReactions)
})
......@@ -105,11 +112,15 @@ func (a *App) sendReactionEvent(event string, reaction *model.Reaction, post *mo
message.Add("reaction", reaction.ToJson())
a.Publish(message)
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
post.HasReactions = hasReactions
post.UpdateAt = model.GetMillis()
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client after reaction", mlog.Any("err", err))
}
clientPost.HasReactions = hasReactions
clientPost.UpdateAt = model.GetMillis()
umessage := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
umessage.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
umessage.Add("post", clientPost.ToJson())
a.Publish(umessage)
}
......@@ -7,6 +7,7 @@ import (
"encoding/json"
"io"
"net/http"
"regexp"
)
const (
......@@ -14,6 +15,8 @@ const (
EMOJI_SORT_BY_NAME = "name"
)
var EMOJI_PATTERN = regexp.MustCompile(`:[a-zA-Z0-9_-]+:`)
type Emoji struct {
Id string `json:"id"`
CreateAt int64 `json:"create_at"`
......
......@@ -11,6 +11,7 @@ import (
"strings"
"unicode/utf8"
"github.com/dyatlov/go-opengraph/opengraph"
"github.com/mattermost/mattermost-server/utils/markdown"
)
......@@ -78,9 +79,22 @@ type Post struct {
Props StringInterface `json:"props"`
Hashtags string `json:"hashtags"`
Filenames StringArray `json:"filenames,omitempty"` // Deprecated, do not use this field any more
FileIds StringArray `json:"file_ids,omitempty"`
FileIds StringArray `json:"file_ids,omitempty"` // Deprecated, do not use this field any more
PendingPostId string `json:"pending_post_id" db:"-"`
HasReactions bool `json:"has_reactions,omitempty"`
HasReactions bool `json:"has_reactions,omitempty"` // Deprecated, do not use this field any more
// Transient fields populated before sending posts to the client
ReactionCounts ReactionCounts `json:"reaction_counts" db:"-"`
FileInfos []*FileInfo `json:"file_infos" db:"-"`
ImageDimensions []*PostImageDimensions `json:"image_dimensions" db:"-"`
OpenGraphData []*opengraph.OpenGraph `json:"opengraph_data" db:"-"`
Emojis []*Emoji `json:"emojis" db:"-"`
}
type PostImageDimensions struct {
URL string `json:"url"`
Width int64 `json:"width"`
Height int64 `json:"height"`
}
type PostEphemeral struct {
......@@ -170,10 +184,16 @@ type PostActionIntegrationResponse struct {
EphemeralText string `json:"ephemeral_text"`
}
func (o *Post) ToJson() string {
// Shallowly clone the a post
func (o *Post) Clone() *Post {
copy := *o
return &copy
}
func (o *Post) ToJson() string {
copy := o.Clone()
copy.StripActionIntegrations()
b, _ := json.Marshal(&copy)
b, _ := json.Marshal(copy)
return string(b)
}
......@@ -502,12 +522,12 @@ var markdownDestinationEscaper = strings.NewReplacer(
// WithRewrittenImageURLs returns a new shallow copy of the post where the message has been
// rewritten via RewriteImageURLs.
func (o *Post) WithRewrittenImageURLs(f func(string) string) *Post {
copy := *o
copy := o.Clone()
copy.Message = RewriteImageURLs(o.Message, f)
if copy.MessageSource == "" && copy.Message != o.Message {
copy.MessageSource = o.Message
}
return &copy
return copy
}
func (o *PostEphemeral) ToUnsanitizedJson() string {
......
......@@ -17,6 +17,8 @@ type Reaction struct {
CreateAt int64 `json:"create_at"`
}
type ReactionCounts map[string]int
func (o *Reaction) ToJson() string {
b, _ := json.Marshal(o)
return string(b)
......@@ -74,3 +76,13 @@ func (o *Reaction) PreSave() {
o.CreateAt = GetMillis()
}
}
func CountReactions(reactions []*Reaction) ReactionCounts {
reactionCounts := ReactionCounts{}
for _, reaction := range reactions {
reactionCounts[reaction.EmojiName] += 1
}
return reactionCounts
}
......@@ -82,3 +82,38 @@ func TestReactionIsValid(t *testing.T) {
t.Fatal("create at should be invalid")
}
}
func TestCountReactions(t *testing.T) {
userId := NewId()
userId2 := NewId()
reactions := []*Reaction{
{
UserId: userId,
EmojiName: "smile",
},
{
UserId: userId,
EmojiName: "frowning",
},
{
UserId: userId2,
EmojiName: "smile",
},
{
UserId: userId2,
EmojiName: "neutral_face",
},
}
reactionCounts := CountReactions(reactions)
if len(reactionCounts) != 3 {
t.Fatal("should've received counts for 3 reactions")
} else if reactionCounts["smile"] != 2 {
t.Fatal("should've received 2 smile reactions")
} else if reactionCounts["frowning"] != 1 {
t.Fatal("should've received 1 frowning reaction")
} else if reactionCounts["neutral_face"] != 1 {
t.Fatal("should've received 2 neutral_face reaction")
}
}
......@@ -18,10 +18,11 @@ func (s *LocalCacheSupplier) handleClusterInvalidateRole(msg *model.ClusterMessa
}
func (s *LocalCacheSupplier) RoleSave(ctx context.Context, role *model.Role, hints ...LayeredStoreHint) *LayeredStoreSupplierResult {
result := s.Next().RoleSave(ctx, role, hints...)
if len(role.Id) != 0 {
defer s.doInvalidateCacheCluster(s.roleCache, role.Name)
s.doInvalidateCacheCluster(s.roleCache, role.Name)
}
return s.Next().RoleSave(ctx, role, hints...)
return result
}
func (s *LocalCacheSupplier) RoleGet(ctx context.Context, roleId string, hints ...LayeredStoreHint) *LayeredStoreSupplierResult {
......@@ -81,8 +82,10 @@ func (s *LocalCacheSupplier) RoleDelete(ctx context.Context, roleId string, hint
}
func (s *LocalCacheSupplier) RolePermanentDeleteAll(ctx context.Context, hints ...LayeredStoreHint) *LayeredStoreSupplierResult {
defer s.roleCache.Purge()
defer s.doClearCacheCluster(s.roleCache)
result := s.Next().RolePermanentDeleteAll(ctx, hints...)
return s.Next().RolePermanentDeleteAll(ctx, hints...)
s.roleCache.Purge()
s.doClearCacheCluster(s.roleCache)
return result
}
......@@ -18,10 +18,11 @@ func (s *LocalCacheSupplier) handleClusterInvalidateScheme(msg *model.ClusterMes
}
func (s *LocalCacheSupplier) SchemeSave(ctx context.Context, scheme *model.Scheme, hints ...LayeredStoreHint) *LayeredStoreSupplierResult {
result := s.Next().<