Unverified Commit 5d45aa81 authored by Jesse Hallam's avatar Jesse Hallam Committed by GitHub
Browse files

MM-16888: fix missing indexes (#12746)

* MM-16888: fix missing indexes

As part of https://mattermost.atlassian.net/browse/MM-16888, we discovered and fixed a number of column and index mismatches between the canonical (i.e. created from scratch) and migrated schemas (i.e migrated from 5.0 through 5.16).

Unfortunately, the migration to fix same was added to `UpgradeDatabaseToVersion514` but never cherry picked to the pending v5.14 release at the time. Customers who upgraded to v5.14 or v5.15 and then get this code as part of v5.16 will never run that migration. Copy it to the UpgradeDatabaseToVersion516 accordingly.

* avoid fixing ChannelMembers.SchemeGuest on MySQL

* synchronize .circleci/config.yml with scripts/mysql-migration-test.sh

* fix circleci invocation

* additional logging on diff

* update build/Jenkinsfile.pr too!
parent eae1acef
...@@ -205,7 +205,7 @@ jobs: ...@@ -205,7 +205,7 @@ jobs:
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
echo "Generating diff" echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (cat diff.txt && exit 1) diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h no_output_timeout: 1h
- run: - run:
name: MySQL schema migration validation name: MySQL schema migration validation
...@@ -234,6 +234,11 @@ jobs: ...@@ -234,6 +234,11 @@ jobs:
-w /go/src/github.com/mattermost/mattermost-server \ -w /go/src/github.com/mattermost/mattermost-server \
mattermost/mattermost-build-server:feb-28-2019 \ mattermost/mattermost-build-server:feb-28-2019 \
bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli' bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli'
echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
echo "Generating dump" echo "Generating dump"
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
...@@ -242,7 +247,7 @@ jobs: ...@@ -242,7 +247,7 @@ jobs:
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
echo "Generating diff" echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (cat diff.txt && exit 1) diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h no_output_timeout: 1h
upload-s3-sha: upload-s3-sha:
docker: docker:
......
...@@ -353,6 +353,10 @@ pipeline { ...@@ -353,6 +353,10 @@ pipeline {
dir('src/github.com/mattermost/mattermost-server') { dir('src/github.com/mattermost/mattermost-server') {
ansiColor('xterm') { ansiColor('xterm') {
sh """ sh """
echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
echo "Generating dump" echo "Generating dump"
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
......
...@@ -22,6 +22,10 @@ make ARGS="config set SqlSettings.DataSource 'mmuser:mostest@tcp(localhost:3306) ...@@ -22,6 +22,10 @@ make ARGS="config set SqlSettings.DataSource 'mmuser:mostest@tcp(localhost:3306)
echo "Setting up fresh db" echo "Setting up fresh db"
make ARGS="version --config $TMPDIR/config.json" run-cli make ARGS="version --config $TMPDIR/config.json" run-cli
echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
docker exec mattermost-mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
docker exec mattermost-mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
echo "Generating dump" echo "Generating dump"
docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > $DUMPDIR/migrated.sql docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > $DUMPDIR/migrated.sql
docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > $DUMPDIR/latest.sql docker exec mattermost-mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > $DUMPDIR/latest.sql
...@@ -33,7 +37,12 @@ echo "Generating diff" ...@@ -33,7 +37,12 @@ echo "Generating diff"
diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt
diffErrorCode=$? diffErrorCode=$?
if [ $diffErrorCode -eq 0 ]; then echo "Both schemas are same";else cat $DUMPDIR/diff.txt; fi if [ $diffErrorCode -eq 0 ]; then
echo "Both schemas are same"
else
echo "Schema mismatch"
cat $DUMPDIR/diff.txt
fi
rm -rf $TMPDIR $DUMPDIR rm -rf $TMPDIR $DUMPDIR
exit $diffErrorCode exit $diffErrorCode
...@@ -33,7 +33,12 @@ echo "Generating diff" ...@@ -33,7 +33,12 @@ echo "Generating diff"
diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt diff $DUMPDIR/migrated.sql $DUMPDIR/latest.sql > $DUMPDIR/diff.txt
diffErrorCode=$? diffErrorCode=$?
if [ $diffErrorCode -eq 0 ]; then echo "Both schemas are same";else cat $DUMPDIR/diff.txt; fi if [ $diffErrorCode -eq 0 ]; then
echo "Both schemas are same"
else
echo "Schema mismatch"
cat $DUMPDIR/diff.txt
fi
rm -rf $TMPDIR $DUMPDIR rm -rf $TMPDIR $DUMPDIR
exit $diffErrorCode exit $diffErrorCode
...@@ -713,17 +713,6 @@ func UpgradeDatabaseToVersion513(sqlStore SqlStore) { ...@@ -713,17 +713,6 @@ func UpgradeDatabaseToVersion513(sqlStore SqlStore) {
func UpgradeDatabaseToVersion514(sqlStore SqlStore) { func UpgradeDatabaseToVersion514(sqlStore SqlStore) {
if shouldPerformUpgrade(sqlStore, VERSION_5_13_0, VERSION_5_14_0) { if shouldPerformUpgrade(sqlStore, VERSION_5_13_0, VERSION_5_14_0) {
sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean")
sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean")
sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)")
sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)")
sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)")
sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean")
sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean")
sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId")
sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId")
saveSchemaVersion(sqlStore, VERSION_5_14_0) saveSchemaVersion(sqlStore, VERSION_5_14_0)
} }
} }
...@@ -742,5 +731,21 @@ func UpgradeDatabaseToVersion516(sqlStore SqlStore) { ...@@ -742,5 +731,21 @@ func UpgradeDatabaseToVersion516(sqlStore SqlStore) {
sqlStore.GetMaster().Exec("ALTER TABLE Tokens MODIFY Extra text") sqlStore.GetMaster().Exec("ALTER TABLE Tokens MODIFY Extra text")
} }
saveSchemaVersion(sqlStore, VERSION_5_16_0) saveSchemaVersion(sqlStore, VERSION_5_16_0)
// Fix mismatches between the canonical and migrated schemas.
sqlStore.AlterColumnTypeIfExists("TeamMembers", "SchemeGuest", "tinyint(4)", "boolean")
sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultTeamGuestRole", "varchar(64)", "VARCHAR(64)")
sqlStore.AlterColumnTypeIfExists("Schemes", "DefaultChannelGuestRole", "varchar(64)", "VARCHAR(64)")
sqlStore.AlterColumnTypeIfExists("Teams", "AllowedDomains", "text", "VARCHAR(1000)")
sqlStore.AlterColumnTypeIfExists("Channels", "GroupConstrained", "tinyint(1)", "boolean")
sqlStore.AlterColumnTypeIfExists("Teams", "GroupConstrained", "tinyint(1)", "boolean")
// One known mismatch remains: ChannelMembers.SchemeGuest. The requisite migration
// is left here for posterity, but we're avoiding fix this given the corresponding
// table rewrite in most MySQL and Postgres instances.
// sqlStore.AlterColumnTypeIfExists("ChannelMembers", "SchemeGuest", "tinyint(4)", "boolean")
sqlStore.CreateIndexIfNotExists("idx_groupteams_teamid", "GroupTeams", "TeamId")
sqlStore.CreateIndexIfNotExists("idx_groupchannels_channelid", "GroupChannels", "ChannelId")
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment