Commit 0a6b96cb authored by Harrison Healey's avatar Harrison Healey Committed by Jesús Espino

MM-9849 Added tracking of which settings are set through environment variables (#8586)

* MM-9849 Added tracking of which settings are set through environment variables

* Removed old version of viper

* Added forked version of viper

* Fixed unit tests

* Fixed more unit tests

* Removed copy from App.GetEnvironmentConfig
parent 57ee6f50
......@@ -23,6 +23,7 @@ func (api *API) InitSystem() {
api.BaseRoutes.ApiRoot.Handle("/config", api.ApiSessionRequired(updateConfig)).Methods("PUT")
api.BaseRoutes.ApiRoot.Handle("/config/reload", api.ApiSessionRequired(configReload)).Methods("POST")
api.BaseRoutes.ApiRoot.Handle("/config/client", api.ApiHandler(getClientConfig)).Methods("GET")
api.BaseRoutes.ApiRoot.Handle("/config/environment", api.ApiSessionRequired(getEnvironmentConfig)).Methods("GET")
api.BaseRoutes.ApiRoot.Handle("/license", api.ApiSessionRequired(addLicense)).Methods("POST")
api.BaseRoutes.ApiRoot.Handle("/license", api.ApiSessionRequired(removeLicense)).Methods("DELETE")
......@@ -251,6 +252,18 @@ func getClientConfig(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(model.MapToJson(c.App.ClientConfigWithComputed())))
}
func getEnvironmentConfig(c *Context, w http.ResponseWriter, r *http.Request) {
if !c.App.SessionHasPermissionTo(c.Session, model.PERMISSION_MANAGE_SYSTEM) {
c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
return
}
envConfig := c.App.GetEnvironmentConfig()
w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
w.Write([]byte(model.StringInterfaceToJson(envConfig)))
}
func getClientLicense(c *Context, w http.ResponseWriter, r *http.Request) {
format := r.URL.Query().Get("format")
......
......@@ -161,6 +161,70 @@ func TestUpdateConfig(t *testing.T) {
})
}
func TestGetEnvironmentConfig(t *testing.T) {
os.Setenv("MM_SERVICESETTINGS_SITEURL", "http://example.mattermost.com")
os.Setenv("MM_SERVICESETTINGS_ENABLECUSTOMEMOJI", "true")
defer os.Unsetenv("MM_SERVICESETTINGS_SITEURL")
th := Setup().InitBasic().InitSystemAdmin()
defer th.TearDown()
t.Run("as system admin", func(t *testing.T) {
SystemAdminClient := th.SystemAdminClient
envConfig, resp := SystemAdminClient.GetEnvironmentConfig()
CheckNoError(t, resp)
if serviceSettings, ok := envConfig["ServiceSettings"]; !ok {
t.Fatal("should've returned ServiceSettings")
} else if serviceSettingsAsMap, ok := serviceSettings.(map[string]interface{}); !ok {
t.Fatal("should've returned ServiceSettings as a map")
} else {
if siteURL, ok := serviceSettingsAsMap["SiteURL"]; !ok {
t.Fatal("should've returned ServiceSettings.SiteURL")
} else if siteURLAsBool, ok := siteURL.(bool); !ok {
t.Fatal("should've returned ServiceSettings.SiteURL as a boolean")
} else if !siteURLAsBool {
t.Fatal("should've returned ServiceSettings.SiteURL as true")
}
if enableCustomEmoji, ok := serviceSettingsAsMap["EnableCustomEmoji"]; !ok {
t.Fatal("should've returned ServiceSettings.EnableCustomEmoji")
} else if enableCustomEmojiAsBool, ok := enableCustomEmoji.(bool); !ok {
t.Fatal("should've returned ServiceSettings.EnableCustomEmoji as a boolean")
} else if !enableCustomEmojiAsBool {
t.Fatal("should've returned ServiceSettings.EnableCustomEmoji as true")
}
}
if _, ok := envConfig["TeamSettings"]; ok {
t.Fatal("should not have returned TeamSettings")
}
})
t.Run("as team admin", func(t *testing.T) {
TeamAdminClient := th.CreateClient()
th.LoginTeamAdminWithClient(TeamAdminClient)
_, resp := TeamAdminClient.GetEnvironmentConfig()
CheckForbiddenStatus(t, resp)
})
t.Run("as regular user", func(t *testing.T) {
Client := th.Client
_, resp := Client.GetEnvironmentConfig()
CheckForbiddenStatus(t, resp)
})
t.Run("as not-regular user", func(t *testing.T) {
Client := th.CreateClient()
_, resp := Client.GetEnvironmentConfig()
CheckUnauthorizedStatus(t, resp)
})
}
func TestGetOldClientConfig(t *testing.T) {
th := Setup().InitBasic().InitSystemAdmin()
defer th.TearDown()
......
......@@ -156,6 +156,10 @@ func (a *App) GetConfig() *model.Config {
return cfg
}
func (a *App) GetEnvironmentConfig() map[string]interface{} {
return a.EnvironmentConfig()
}
func (a *App) SaveConfig(cfg *model.Config, sendConfigChangeClusterMessage bool) *model.AppError {
oldCfg := a.Config()
cfg.SetDefaults()
......
......@@ -59,6 +59,7 @@ type App struct {
Saml einterfaces.SamlInterface
config atomic.Value
envConfig map[string]interface{}
configFile string
configListeners map[string]func(*model.Config, *model.Config)
......
......@@ -30,6 +30,13 @@ func (a *App) Config() *model.Config {
return &model.Config{}
}
func (a *App) EnvironmentConfig() map[string]interface{} {
if a.envConfig != nil {
return a.envConfig
}
return map[string]interface{}{}
}
func (a *App) UpdateConfig(f func(*model.Config)) {
old := a.Config()
updated := old.Clone()
......@@ -46,7 +53,7 @@ func (a *App) PersistConfig() {
func (a *App) LoadConfig(configFile string) *model.AppError {
old := a.Config()
cfg, configPath, err := utils.LoadConfig(configFile)
cfg, configPath, envConfig, err := utils.LoadConfig(configFile)
if err != nil {
return err
}
......@@ -57,6 +64,7 @@ func (a *App) LoadConfig(configFile string) *model.AppError {
l4g.Info("Using config file at %s", configPath)
a.config.Store(cfg)
a.envConfig = envConfig
a.siteURL = strings.TrimRight(*cfg.ServiceSettings.SiteURL, "/")
......
......@@ -22,7 +22,7 @@ func TestConfigFlag(t *testing.T) {
defer os.RemoveAll(dir)
utils.TranslationsPreInit()
config, _, err := utils.LoadConfig("config.json")
config, _, _, err := utils.LoadConfig("config.json")
require.Nil(t, err)
configPath := filepath.Join(dir, "foo.json")
require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600))
......
......@@ -57,7 +57,7 @@ func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string {
require.NoError(t, err)
utils.TranslationsPreInit()
config, _, appErr := utils.LoadConfig("config.json")
config, _, _, appErr := utils.LoadConfig("config.json")
require.Nil(t, appErr)
config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled)
configPath := filepath.Join(dir, "foo.json")
......
......@@ -205,7 +205,7 @@ imports:
- name: github.com/spf13/cast
version: 8965335b8c7107321228e3e3702cab9832751bac
- name: github.com/spf13/cobra
version: be77323fc05148ef091e83b3866c0d47c8e74a8b
version: 4f5003aa93559718c866d86fbc795439079484f5
- name: github.com/spf13/jwalterweatherman
version: 7c0cea34c8ece3fbeb2b27ab9b59511d360fb394
- name: github.com/spf13/pflag
......
......@@ -49,6 +49,8 @@ import:
version: 2.1.1
- package: github.com/spf13/cobra
- package: github.com/spf13/viper
vcs: git
repo: https://github.com/hmhealey/viper.git
- package: github.com/stretchr/testify
version: v1.2.1
subpackages:
......
......@@ -2167,6 +2167,18 @@ func (c *Client4) GetOldClientConfig(etag string) (map[string]string, *Response)
}
}
// GetEnvironmentConfig will retrieve a map mirroring the server configuration where fields
// are set to true if the corresponding config setting is set through an environment variable.
// Settings that haven't been set through environment variables will be missing from the map.
func (c *Client4) GetEnvironmentConfig() (map[string]interface{}, *Response) {
if r, err := c.DoApiGet(c.GetConfigRoute()+"/environment", ""); err != nil {
return nil, BuildErrorResponse(r, err)
} else {
defer closeBody(r)
return StringInterfaceFromJson(r.Body), BuildResponse(r)
}
}
// GetOldClientLicense will retrieve the parts of the server license needed by the
// client, formatted in the old format.
func (c *Client4) GetOldClientLicense(etag string) (map[string]string, *Response) {
......
......@@ -188,7 +188,7 @@ func NewConfigWatcher(cfgFileName string, f func()) (*ConfigWatcher, error) {
if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
l4g.Info(fmt.Sprintf("Config file watcher detected a change reloading %v", cfgFileName))
if _, configReadErr := ReadConfigFile(cfgFileName, true); configReadErr == nil {
if _, _, configReadErr := ReadConfigFile(cfgFileName, true); configReadErr == nil {
f()
} else {
l4g.Error(fmt.Sprintf("Failed to read while watching config file at %v with err=%v", cfgFileName, configReadErr.Error()))
......@@ -212,11 +212,11 @@ func (w *ConfigWatcher) Close() {
}
// ReadConfig reads and parses the given configuration.
func ReadConfig(r io.Reader, allowEnvironmentOverrides bool) (*model.Config, error) {
func ReadConfig(r io.Reader, allowEnvironmentOverrides bool) (*model.Config, map[string]interface{}, error) {
v := newViper(allowEnvironmentOverrides)
if err := v.ReadConfig(r); err != nil {
return nil, err
return nil, nil, err
}
var config model.Config
......@@ -227,7 +227,15 @@ func ReadConfig(r io.Reader, allowEnvironmentOverrides bool) (*model.Config, err
config.PluginSettings = model.PluginSettings{}
unmarshalErr = v.UnmarshalKey("pluginsettings", &config.PluginSettings)
}
return &config, unmarshalErr
envConfig := v.EnvSettings()
var envErr error
if envConfig, envErr = fixEnvSettingsCase(envConfig); envErr != nil {
return nil, nil, envErr
}
return &config, envConfig, unmarshalErr
}
func newViper(allowEnvironmentOverrides bool) *viper.Viper {
......@@ -254,13 +262,19 @@ func newViper(allowEnvironmentOverrides bool) *viper.Viper {
// Converts a struct type into a nested map with keys matching the struct's fields and values
// matching the zeroed value of the corresponding field.
func structToMap(t reflect.Type) map[string]interface{} {
func structToMap(t reflect.Type) (out map[string]interface{}) {
defer func() {
if r := recover(); r != nil {
l4g.Error("Panicked in structToMap. This should never happen. %v", r)
}
}()
if t.Kind() != reflect.Struct {
// Should never hit this, but this will prevent a panic if that does happen somehow
return nil
}
out := make(map[string]interface{})
out = map[string]interface{}{}
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
......@@ -279,7 +293,7 @@ func structToMap(t reflect.Type) map[string]interface{} {
out[field.Name] = value
}
return out
return
}
// Flattens a nested map so that the result is a single map with keys corresponding to the
......@@ -313,11 +327,51 @@ func flattenStructToMap(in map[string]interface{}) map[string]interface{} {
return out
}
// Fixes the case of the environment variables sent back from Viper since Viper stores
// everything as lower case.
func fixEnvSettingsCase(in map[string]interface{}) (out map[string]interface{}, err error) {
defer func() {
if r := recover(); r != nil {
l4g.Error("Panicked in fixEnvSettingsCase. This should never happen. %v", r)
out = in
}
}()
var fixCase func(map[string]interface{}, reflect.Type) map[string]interface{}
fixCase = func(in map[string]interface{}, t reflect.Type) map[string]interface{} {
if t.Kind() != reflect.Struct {
// Should never hit this, but this will prevent a panic if that does happen somehow
return nil
}
out := make(map[string]interface{}, len(in))
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
key := field.Name
if value, ok := in[strings.ToLower(key)]; ok {
if valueAsMap, ok := value.(map[string]interface{}); ok {
out[key] = fixCase(valueAsMap, field.Type)
} else {
out[key] = value
}
}
}
return out
}
out = fixCase(in, reflect.TypeOf(model.Config{}))
return
}
// ReadConfigFile reads and parses the configuration at the given file path.
func ReadConfigFile(path string, allowEnvironmentOverrides bool) (*model.Config, error) {
func ReadConfigFile(path string, allowEnvironmentOverrides bool) (*model.Config, map[string]interface{}, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
return nil, nil, err
}
defer f.Close()
return ReadConfig(f, allowEnvironmentOverrides)
......@@ -352,22 +406,24 @@ func EnsureConfigFile(fileName string) (string, error) {
// LoadConfig will try to search around for the corresponding config file. It will search
// /tmp/fileName then attempt ./config/fileName, then ../config/fileName and last it will look at
// fileName.
func LoadConfig(fileName string) (config *model.Config, configPath string, appErr *model.AppError) {
func LoadConfig(fileName string) (*model.Config, string, map[string]interface{}, *model.AppError) {
var configPath string
if fileName != filepath.Base(fileName) {
configPath = fileName
} else {
if path, err := EnsureConfigFile(fileName); err != nil {
appErr = model.NewAppError("LoadConfig", "utils.config.load_config.opening.panic", map[string]interface{}{"Filename": fileName, "Error": err.Error()}, "", 0)
return
appErr := model.NewAppError("LoadConfig", "utils.config.load_config.opening.panic", map[string]interface{}{"Filename": fileName, "Error": err.Error()}, "", 0)
return nil, "", nil, appErr
} else {
configPath = path
}
}
config, err := ReadConfigFile(configPath, true)
config, envConfig, err := ReadConfigFile(configPath, true)
if err != nil {
appErr = model.NewAppError("LoadConfig", "utils.config.load_config.decoding.panic", map[string]interface{}{"Filename": fileName, "Error": err.Error()}, "", 0)
return
appErr := model.NewAppError("LoadConfig", "utils.config.load_config.decoding.panic", map[string]interface{}{"Filename": fileName, "Error": err.Error()}, "", 0)
return nil, "", nil, appErr
}
needSave := len(config.SqlSettings.AtRestEncryptKey) == 0 || len(*config.FileSettings.PublicLinkSalt) == 0 ||
......@@ -376,7 +432,7 @@ func LoadConfig(fileName string) (config *model.Config, configPath string, appEr
config.SetDefaults()
if err := config.IsValid(); err != nil {
return nil, "", err
return nil, "", nil, err
}
if needSave {
......@@ -398,7 +454,7 @@ func LoadConfig(fileName string) (config *model.Config, configPath string, appEr
}
}
return config, configPath, nil
return config, configPath, envConfig, nil
}
func GenerateClientConfig(c *model.Config, diagnosticId string, license *model.License) map[string]string {
......
......@@ -18,7 +18,7 @@ import (
func TestConfig(t *testing.T) {
TranslationsPreInit()
cfg, _, err := LoadConfig("config.json")
cfg, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
InitTranslations(cfg.LocalizationSettings)
}
......@@ -67,7 +67,7 @@ func TestConfigFromEnviroVars(t *testing.T) {
os.Setenv("MM_TEAMSETTINGS_SITENAME", "From Environment")
os.Setenv("MM_TEAMSETTINGS_CUSTOMBRANDTEXT", "Custom Brand")
cfg, err := ReadConfig(strings.NewReader(config), true)
cfg, envCfg, err := ReadConfig(strings.NewReader(config), true)
require.Nil(t, err)
if cfg.TeamSettings.SiteName != "From Environment" {
......@@ -78,57 +78,105 @@ func TestConfigFromEnviroVars(t *testing.T) {
t.Fatal("Couldn't read config from environment var")
}
if teamSettings, ok := envCfg["TeamSettings"]; !ok {
t.Fatal("TeamSettings is missing from envConfig")
} else if teamSettingsAsMap, ok := teamSettings.(map[string]interface{}); !ok {
t.Fatal("TeamSettings is not a map in envConfig")
} else {
if siteNameInEnv, ok := teamSettingsAsMap["SiteName"].(bool); !ok || !siteNameInEnv {
t.Fatal("SiteName should be in envConfig")
}
if customBrandTextInEnv, ok := teamSettingsAsMap["CustomBrandText"].(bool); !ok || !customBrandTextInEnv {
t.Fatal("SiteName should be in envConfig")
}
}
os.Unsetenv("MM_TEAMSETTINGS_SITENAME")
os.Unsetenv("MM_TEAMSETTINGS_CUSTOMBRANDTEXT")
cfg, err = ReadConfig(strings.NewReader(config), true)
cfg, envCfg, err = ReadConfig(strings.NewReader(config), true)
require.Nil(t, err)
if cfg.TeamSettings.SiteName != "Mattermost" {
t.Fatal("should have been reset")
}
if _, ok := envCfg["TeamSettings"]; ok {
t.Fatal("TeamSettings should be missing from envConfig")
}
})
t.Run("boolean setting", func(t *testing.T) {
os.Setenv("MM_SERVICESETTINGS_ENABLECOMMANDS", "false")
defer os.Unsetenv("MM_SERVICESETTINGS_ENABLECOMMANDS")
cfg, err := ReadConfig(strings.NewReader(config), true)
cfg, envCfg, err := ReadConfig(strings.NewReader(config), true)
require.Nil(t, err)
if *cfg.ServiceSettings.EnableCommands {
t.Fatal("Couldn't read config from environment var")
}
if serviceSettings, ok := envCfg["ServiceSettings"]; !ok {
t.Fatal("ServiceSettings is missing from envConfig")
} else if serviceSettingsAsMap, ok := serviceSettings.(map[string]interface{}); !ok {
t.Fatal("ServiceSettings is not a map in envConfig")
} else {
if enableCommandsInEnv, ok := serviceSettingsAsMap["EnableCommands"].(bool); !ok || !enableCommandsInEnv {
t.Fatal("EnableCommands should be in envConfig")
}
}
})
t.Run("integer setting", func(t *testing.T) {
os.Setenv("MM_SERVICESETTINGS_READTIMEOUT", "400")
defer os.Unsetenv("MM_SERVICESETTINGS_READTIMEOUT")
cfg, err := ReadConfig(strings.NewReader(config), true)
cfg, envCfg, err := ReadConfig(strings.NewReader(config), true)
require.Nil(t, err)
if *cfg.ServiceSettings.ReadTimeout != 400 {
t.Fatal("Couldn't read config from environment var")
}
if serviceSettings, ok := envCfg["ServiceSettings"]; !ok {
t.Fatal("ServiceSettings is missing from envConfig")
} else if serviceSettingsAsMap, ok := serviceSettings.(map[string]interface{}); !ok {
t.Fatal("ServiceSettings is not a map in envConfig")
} else {
if readTimeoutInEnv, ok := serviceSettingsAsMap["ReadTimeout"].(bool); !ok || !readTimeoutInEnv {
t.Fatal("ReadTimeout should be in envConfig")
}
}
})
t.Run("setting missing from config.json", func(t *testing.T) {
os.Setenv("MM_SERVICESETTINGS_SITEURL", "https://example.com")
defer os.Unsetenv("MM_SERVICESETTINGS_SITEURL")
cfg, err := ReadConfig(strings.NewReader(config), true)
cfg, envCfg, err := ReadConfig(strings.NewReader(config), true)
require.Nil(t, err)
if *cfg.ServiceSettings.SiteURL != "https://example.com" {
t.Fatal("Couldn't read config from environment var")
}
if serviceSettings, ok := envCfg["ServiceSettings"]; !ok {
t.Fatal("ServiceSettings is missing from envConfig")
} else if serviceSettingsAsMap, ok := serviceSettings.(map[string]interface{}); !ok {
t.Fatal("ServiceSettings is not a map in envConfig")
} else {
if siteURLInEnv, ok := serviceSettingsAsMap["SiteURL"].(bool); !ok || !siteURLInEnv {
t.Fatal("SiteURL should be in envConfig")
}
}
})
}
func TestValidateLocales(t *testing.T) {
TranslationsPreInit()
cfg, _, err := LoadConfig("config.json")
cfg, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
*cfg.LocalizationSettings.DefaultServerLocale = "en"
......@@ -326,18 +374,6 @@ func TestGetClientConfig(t *testing.T) {
}
})
}
}
func TestReadConfig(t *testing.T) {
config, err := ReadConfig(strings.NewReader(`{
"ServiceSettings": {
"SiteURL": "http://foo.bar"
}
}`), false)
require.NoError(t, err)
assert.Equal(t, "http://foo.bar", *config.ServiceSettings.SiteURL)
}
func sToP(s string) *string {
......
......@@ -13,7 +13,7 @@ import (
)
func TestMailConnectionFromConfig(t *testing.T) {
cfg, _, err := LoadConfig("config.json")
cfg, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
if conn, err := ConnectToSMTPServer(cfg); err != nil {
......@@ -36,7 +36,7 @@ func TestMailConnectionFromConfig(t *testing.T) {
}
func TestMailConnectionAdvanced(t *testing.T) {
cfg, _, err := LoadConfig("config.json")
cfg, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
if conn, err := ConnectToSMTPServerAdvanced(
......@@ -86,7 +86,7 @@ func TestMailConnectionAdvanced(t *testing.T) {
}
func TestSendMailUsingConfig(t *testing.T) {
cfg, _, err := LoadConfig("config.json")
cfg, _, _, err := LoadConfig("config.json")
require.Nil(t, err)
T = GetUserTranslations("en")
......
QProcess::start: Process is already running
......@@ -1675,6 +1675,26 @@ func (v *Viper) AllSettings() map[string]interface{} {
return m
}
// EnvSettings returns a map[string]interface{} containing all settings set
// through environment variables.
func EnvSettings() map[string]interface{} { return v.EnvSettings() }
func (v *Viper) EnvSettings() map[string]interface{} {
m := map[string]interface{}{}
// start from the list of keys, and construct the map one value at a time
for _, k := range v.AllKeys() {
value := v.getEnv(v.mergeWithEnvPrefix(k))
if value == "" {
continue
}
path := strings.Split(k, v.keyDelim)
lastKey := strings.ToLower(path[len(path)-1])
deepestMap := deepSearch(m, path[0:len(path)-1])
// set innermost value
deepestMap[lastKey] = true
}
return m
}
// SetFs sets the filesystem to use to read configuration.
func SetFs(fs afero.Fs) { v.SetFs(fs) }
func (v *Viper) SetFs(fs afero.Fs) {
......@@ -1720,18 +1740,14 @@ func (v *Viper) getConfigType() string {
}
func (v *Viper) getConfigFile() (string, error) {
// if explicitly set, then use it
if v.configFile != "" {
return v.configFile, nil
}
cf, err := v.findConfigFile()
if err != nil {
return "", err
if v.configFile == "" {
cf, err := v.findConfigFile()
if err != nil {
return "", err
}
v.configFile = cf
}
v.configFile = cf
return v.getConfigFile()
return v.configFile, nil
}
func (v *Viper) searchInPath(in string) (filename string) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment