Unverified Commit 8b17bf9e authored by Jesse Hallam's avatar Jesse Hallam Committed by GitHub

MM-11886: materialize channel search (#9349)

* materialize PublicChannels table

Introduce triggers for each supported database that automatically maintain a subset of the Channels table corresponding to only public channels. This improves corresponding queries that no longer need to filter out 99% DM channels.

This initial commit modifies the channel store directly for easier code reviewing, but the next wraps an experimental version around it to enable a kill switch in case there are unforeseen performance regressions.

This addresses [MM-11886](https://mattermost.atlassian.net/browse/MM-11886) and [MM-11945](https://mattermost.atlassian.net/browse/MM-11945).

* extract the experimental public channels materialization

Wrap the original channel store with an experimental version that
leverages the materialized public channels, but can be disabled to
fallback to the original implementation.

This addresses MM-11947.

* s/ExperimentalPublicChannelsMaterialization/EnablePublicChannelsMaterialization/

* simplify error handling

* move experimental config listener until after store is initialized
parent 0a5f792d
......@@ -212,6 +212,14 @@ func New(options ...Option) (outApp *App, outErr error) {
}
app.Srv.Store = app.newStore()
app.AddConfigListener(func(_, current *model.Config) {
if current.SqlSettings.EnablePublicChannelsMaterialization != nil && !*current.SqlSettings.EnablePublicChannelsMaterialization {
app.Srv.Store.Channel().DisableExperimentalPublicChannelsMaterialization()
} else {
app.Srv.Store.Channel().EnableExperimentalPublicChannelsMaterialization()
}
})
if err := app.ensureAsymmetricSigningKey(); err != nil {
return nil, errors.Wrapf(err, "unable to ensure asymmetric signing key")
}
......
......@@ -130,7 +130,8 @@
"MaxOpenConns": 300,
"Trace": false,
"AtRestEncryptKey": "",
"QueryTimeout": 30
"QueryTimeout": 30,
"EnablePublicChannelsMaterialization": true
},
"LogSettings": {
"EnableConsole": true,
......
......@@ -644,16 +644,17 @@ type SSOSettings struct {
}
type SqlSettings struct {
DriverName *string
DataSource *string
DataSourceReplicas []string
DataSourceSearchReplicas []string
MaxIdleConns *int
ConnMaxLifetimeMilliseconds *int
MaxOpenConns *int
Trace bool
AtRestEncryptKey string
QueryTimeout *int
DriverName *string
DataSource *string
DataSourceReplicas []string
DataSourceSearchReplicas []string
MaxIdleConns *int
ConnMaxLifetimeMilliseconds *int
MaxOpenConns *int
Trace bool
AtRestEncryptKey string
QueryTimeout *int
EnablePublicChannelsMaterialization *bool
}
func (s *SqlSettings) SetDefaults() {
......@@ -684,6 +685,10 @@ func (s *SqlSettings) SetDefaults() {
if s.QueryTimeout == nil {
s.QueryTimeout = NewInt(30)
}
if s.EnablePublicChannelsMaterialization == nil {
s.EnablePublicChannelsMaterialization = NewBool(true)
}
}
type LogSettings struct {
......
......@@ -301,6 +301,21 @@ func (s SqlChannelStore) CreateIndexesIfNotExists() {
s.CreateFullTextIndexIfNotExists("idx_channel_search_txt", "Channels", "Name, DisplayName, Purpose")
}
func (s SqlChannelStore) CreateTriggersIfNotExists() error {
// See SqlChannelStoreExperimental
return nil
}
func (s SqlChannelStore) MigratePublicChannels() error {
// See SqlChannelStoreExperimental
return nil
}
func (s SqlChannelStore) DropPublicChannels() error {
// See SqlChannelStoreExperimental
return nil
}
func (s SqlChannelStore) Save(channel *model.Channel, maxChannelsPerTeam int64) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
if channel.DeleteAt != 0 {
......@@ -804,12 +819,12 @@ func (s SqlChannelStore) GetTeamChannels(teamId string) store.StoreChannel {
_, err := s.GetReplica().Select(data, "SELECT * FROM Channels WHERE TeamId = :TeamId And Type != 'D' ORDER BY DisplayName", map[string]interface{}{"TeamId": teamId})
if err != nil {
result.Err = model.NewAppError("SqlChannelStore.GetChannels", "store.sql_channel.get_channels.get.app_error", nil, "teamId="+teamId+", err="+err.Error(), http.StatusInternalServerError)
result.Err = model.NewAppError("SqlChannelStore.GetTeamChannels", "store.sql_channel.get_channels.get.app_error", nil, "teamId="+teamId+", err="+err.Error(), http.StatusInternalServerError)
return
}
if len(*data) == 0 {
result.Err = model.NewAppError("SqlChannelStore.GetChannels", "store.sql_channel.get_channels.not_found.app_error", nil, "teamId="+teamId, http.StatusNotFound)
result.Err = model.NewAppError("SqlChannelStore.GetTeamChannels", "store.sql_channel.get_channels.not_found.app_error", nil, "teamId="+teamId, http.StatusNotFound)
return
}
......@@ -962,16 +977,16 @@ var CHANNEL_MEMBERS_WITH_SCHEME_SELECT_QUERY = `
TeamScheme.DefaultChannelAdminRole TeamSchemeDefaultAdminRole,
ChannelScheme.DefaultChannelUserRole ChannelSchemeDefaultUserRole,
ChannelScheme.DefaultChannelAdminRole ChannelSchemeDefaultAdminRole
FROM
FROM
ChannelMembers
INNER JOIN
INNER JOIN
Channels ON ChannelMembers.ChannelId = Channels.Id
LEFT JOIN
Schemes ChannelScheme ON Channels.SchemeId = ChannelScheme.Id
LEFT JOIN
Teams ON Channels.TeamId = Teams.Id
LEFT JOIN
Schemes TeamScheme ON Teams.SchemeId = TeamScheme.Id
Schemes TeamScheme ON Teams.SchemeId = TeamScheme.Id
`
func (s SqlChannelStore) SaveMember(member *model.ChannelMember) store.StoreChannel {
......@@ -1988,3 +2003,16 @@ func (s SqlChannelStore) ResetLastPostAt() store.StoreChannel {
}
})
}
func (s SqlChannelStore) EnableExperimentalPublicChannelsMaterialization() {
// See SqlChannelStoreExperimental
}
func (s SqlChannelStore) DisableExperimentalPublicChannelsMaterialization() {
// See SqlChannelStoreExperimental
}
func (s SqlChannelStore) IsExperimentalPublicChannelsMaterializationEnabled() bool {
// See SqlChannelStoreExperimental
return false
}
This diff is collapsed.
......@@ -14,7 +14,7 @@ import (
)
func TestChannelStore(t *testing.T) {
StoreTest(t, storetest.TestChannelStore)
StoreTestWithSqlSupplier(t, storetest.TestChannelStore)
}
func TestChannelStoreInternalDataTypes(t *testing.T) {
......
......@@ -51,6 +51,7 @@ type SqlStore interface {
MarkSystemRanUnitTests()
DoesTableExist(tablename string) bool
DoesColumnExist(tableName string, columName string) bool
DoesTriggerExist(triggerName string) bool
CreateColumnIfNotExists(tableName string, columnName string, mySqlColType string, postgresColType string, defaultValue string) bool
CreateColumnIfNotExistsNoDefault(tableName string, columnName string, mySqlColType string, postgresColType string) bool
RemoveColumnIfExists(tableName string, columnName string) bool
......
......@@ -16,10 +16,11 @@ import (
)
var storeTypes = []*struct {
Name string
Func func() (*storetest.RunningContainer, *model.SqlSettings, error)
Container *storetest.RunningContainer
Store store.Store
Name string
Func func() (*storetest.RunningContainer, *model.SqlSettings, error)
Container *storetest.RunningContainer
SqlSupplier *SqlSupplier
Store store.Store
}{
{
Name: "MySQL",
......@@ -44,6 +45,19 @@ func StoreTest(t *testing.T, f func(*testing.T, store.Store)) {
}
}
func StoreTestWithSqlSupplier(t *testing.T, f func(*testing.T, store.Store, storetest.SqlSupplier)) {
defer func() {
if err := recover(); err != nil {
tearDownStores()
panic(err)
}
}()
for _, st := range storeTypes {
st := st
t.Run(st.Name, func(t *testing.T) { f(t, st.Store, st.SqlSupplier) })
}
}
func initStores() {
defer func() {
if err := recover(); err != nil {
......@@ -64,7 +78,8 @@ func initStores() {
return
}
st.Container = container
st.Store = store.NewLayeredStore(NewSqlSupplier(*settings, nil), nil, nil)
st.SqlSupplier = NewSqlSupplier(*settings, nil)
st.Store = store.NewLayeredStore(st.SqlSupplier, nil, nil)
st.Store.MarkSystemRanUnitTests()
}()
}
......
......@@ -33,6 +33,7 @@ const (
)
const (
EXIT_GENERIC_FAILURE = 1
EXIT_CREATE_TABLE = 100
EXIT_DB_OPEN = 101
EXIT_PING = 102
......@@ -116,8 +117,13 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
supplier.initConnection()
enableExperimentalPublicChannelsMaterialization := true
if settings.EnablePublicChannelsMaterialization != nil && !*settings.EnablePublicChannelsMaterialization {
enableExperimentalPublicChannelsMaterialization = false
}
supplier.oldStores.team = NewSqlTeamStore(supplier)
supplier.oldStores.channel = NewSqlChannelStore(supplier, metrics)
supplier.oldStores.channel = NewSqlChannelStoreExperimental(supplier, metrics, enableExperimentalPublicChannelsMaterialization)
supplier.oldStores.post = NewSqlPostStore(supplier, metrics)
supplier.oldStores.user = NewSqlUserStore(supplier, metrics)
supplier.oldStores.audit = NewSqlAuditStore(supplier)
......@@ -151,10 +157,19 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
os.Exit(EXIT_CREATE_TABLE)
}
// This store's triggers should exist before the migration is run to ensure the
// corresponding tables stay in sync. Whether or not a trigger should be created before
// or after a migration is likely to be decided on a case-by-case basis.
if err := supplier.oldStores.channel.(*SqlChannelStoreExperimental).CreateTriggersIfNotExists(); err != nil {
mlog.Critical("Error creating triggers", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(EXIT_GENERIC_FAILURE)
}
UpgradeDatabase(supplier)
supplier.oldStores.team.(*SqlTeamStore).CreateIndexesIfNotExists()
supplier.oldStores.channel.(*SqlChannelStore).CreateIndexesIfNotExists()
supplier.oldStores.channel.(*SqlChannelStoreExperimental).CreateIndexesIfNotExists()
supplier.oldStores.post.(*SqlPostStore).CreateIndexesIfNotExists()
supplier.oldStores.user.(*SqlUserStore).CreateIndexesIfNotExists()
supplier.oldStores.audit.(*SqlAuditStore).CreateIndexesIfNotExists()
......@@ -461,6 +476,52 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool
}
}
func (ss *SqlSupplier) DoesTriggerExist(triggerName string) bool {
if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
count, err := ss.GetMaster().SelectInt(`
SELECT
COUNT(0)
FROM
pg_trigger
WHERE
tgname = $1
`, triggerName)
if err != nil {
mlog.Critical(fmt.Sprintf("Failed to check if trigger exists %v", err))
time.Sleep(time.Second)
os.Exit(EXIT_GENERIC_FAILURE)
}
return count > 0
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
count, err := ss.GetMaster().SelectInt(`
SELECT
COUNT(0)
FROM
information_schema.triggers
WHERE
trigger_schema = DATABASE()
AND trigger_name = ?
`, triggerName)
if err != nil {
mlog.Critical(fmt.Sprintf("Failed to check if trigger exists %v", err))
time.Sleep(time.Second)
os.Exit(EXIT_GENERIC_FAILURE)
}
return count > 0
} else {
mlog.Critical("Failed to check if column exists because of missing driver")
time.Sleep(time.Second)
os.Exit(EXIT_GENERIC_FAILURE)
return false
}
}
func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName string, mySqlColType string, postgresColType string, defaultValue string) bool {
if ss.DoesColumnExist(tableName, columnName) {
......
......@@ -489,7 +489,6 @@ func UpgradeDatabaseToVersion53(sqlStore SqlStore) {
if shouldPerformUpgrade(sqlStore, VERSION_5_2_0, VERSION_5_3_0) {
saveSchemaVersion(sqlStore, VERSION_5_3_0)
}
}
func UpgradeDatabaseToVersion54(sqlStore SqlStore) {
......@@ -497,6 +496,11 @@ func UpgradeDatabaseToVersion54(sqlStore SqlStore) {
// if shouldPerformUpgrade(sqlStore, VERSION_5_3_0, VERSION_5_4_0) {
sqlStore.AlterColumnTypeIfExists("OutgoingWebhooks", "Description", "varchar(500)", "varchar(500)")
sqlStore.AlterColumnTypeIfExists("IncomingWebhooks", "Description", "varchar(500)", "varchar(500)")
if err := sqlStore.Channel().MigratePublicChannels(); err != nil {
mlog.Critical("Failed to migrate PublicChannels table", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(EXIT_GENERIC_FAILURE)
}
// saveSchemaVersion(sqlStore, VERSION_5_4_0)
// }
}
......@@ -174,6 +174,11 @@ type ChannelStore interface {
ResetAllChannelSchemes() StoreChannel
ClearAllCustomRoleAssignments() StoreChannel
ResetLastPostAt() StoreChannel
MigratePublicChannels() error
DropPublicChannels() error
EnableExperimentalPublicChannelsMaterialization()
DisableExperimentalPublicChannelsMaterialization()
IsExperimentalPublicChannelsMaterializationEnabled() bool
}
type ChannelMemberHistoryStore interface {
......
This diff is collapsed.
......@@ -130,6 +130,30 @@ func (_m *ChannelStore) Delete(channelId string, time int64) store.StoreChannel
return r0
}
// DisableExperimentalPublicChannelsMaterialization provides a mock function with given fields:
func (_m *ChannelStore) DisableExperimentalPublicChannelsMaterialization() {
_m.Called()
}
// DropPublicChannels provides a mock function with given fields:
func (_m *ChannelStore) DropPublicChannels() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// EnableExperimentalPublicChannelsMaterialization provides a mock function with given fields:
func (_m *ChannelStore) EnableExperimentalPublicChannelsMaterialization() {
_m.Called()
}
// Get provides a mock function with given fields: id, allowFromCache
func (_m *ChannelStore) Get(id string, allowFromCache bool) store.StoreChannel {
ret := _m.Called(id, allowFromCache)
......@@ -601,6 +625,20 @@ func (_m *ChannelStore) InvalidateMemberCount(channelId string) {
_m.Called(channelId)
}
// IsExperimentalPublicChannelsMaterializationEnabled provides a mock function with given fields:
func (_m *ChannelStore) IsExperimentalPublicChannelsMaterializationEnabled() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// IsUserInChannelUseCache provides a mock function with given fields: userId, channelId
func (_m *ChannelStore) IsUserInChannelUseCache(userId string, channelId string) bool {
ret := _m.Called(userId, channelId)
......@@ -631,6 +669,20 @@ func (_m *ChannelStore) MigrateChannelMembers(fromChannelId string, fromUserId s
return r0
}
// MigratePublicChannels provides a mock function with given fields:
func (_m *ChannelStore) MigratePublicChannels() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// PermanentDelete provides a mock function with given fields: channelId
func (_m *ChannelStore) PermanentDelete(channelId string) store.StoreChannel {
ret := _m.Called(channelId)
......
......@@ -241,6 +241,20 @@ func (_m *SqlStore) DoesTableExist(tablename string) bool {
return r0
}
// DoesTriggerExist provides a mock function with given fields: triggerName
func (_m *SqlStore) DoesTriggerExist(triggerName string) bool {
ret := _m.Called(triggerName)
var r0 bool
if rf, ok := ret.Get(0).(func(string) bool); ok {
r0 = rf(triggerName)
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// DriverName provides a mock function with given fields:
func (_m *SqlStore) DriverName() string {
ret := _m.Called()
......
// Code generated by mockery v1.0.0. DO NOT EDIT.
// Regenerate this file using `make store-mocks`.
package mocks
import gorp "github.com/mattermost/gorp"
import mock "github.com/stretchr/testify/mock"
// SqlSupplier is an autogenerated mock type for the SqlSupplier type
type SqlSupplier struct {
mock.Mock
}
// GetMaster provides a mock function with given fields:
func (_m *SqlSupplier) GetMaster() *gorp.DbMap {
ret := _m.Called()
var r0 *gorp.DbMap
if rf, ok := ret.Get(0).(func() *gorp.DbMap); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*gorp.DbMap)
}
}
return r0
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment