From 909f2be99d0c699f506f463bb6238378f12c0dc1 Mon Sep 17 00:00:00 2001
From: zeripath <art27@cantab.net>
Date: Fri, 19 Mar 2021 03:23:58 +0000
Subject: [PATCH] Fix postgres ID sequences broken by recreate-table (#15015)
(#15029)
Backport #15015
Unfortunately there is a subtle problem with recreatetable on postgres which
leads to the sequences not being renamed and not being left at 0.
Fix #14725
Signed-off-by: Andrew Thornton <art27@cantab.net>
---
cmd/doctor.go | 17 +++++++++
models/consistency.go | 61 +++++++++++++++++++++++++++++++
models/migrations/migrations.go | 63 ++++++++++++++++++++++++++++++++-
3 files changed, 140 insertions(+), 1 deletion(-)
diff --git a/cmd/doctor.go b/cmd/doctor.go
index 2ca2bb5e70..5ba0451110 100644
--- a/cmd/doctor.go
+++ b/cmd/doctor.go
@@ -670,6 +670,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
}
}
+ if setting.Database.UsePostgreSQL {
+ count, err = models.CountBadSequences()
+ if err != nil {
+ return nil, err
+ }
+ if count > 0 {
+ if ctx.Bool("fix") {
+ err := models.FixBadSequences()
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, fmt.Sprintf("%d sequences updated", count))
+ } else {
+ results = append(results, fmt.Sprintf("%d sequences with incorrect values", count))
+ }
+ }
+ }
//ToDo: function to recalc all counters
return results, nil
diff --git a/models/consistency.go b/models/consistency.go
index fbb99ca80c..0c62d4dc24 100644
--- a/models/consistency.go
+++ b/models/consistency.go
@@ -5,10 +5,13 @@
package models
import (
+ "fmt"
"reflect"
+ "regexp"
"strings"
"testing"
+ "code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
"xorm.io/builder"
)
@@ -295,3 +298,61 @@ func FixNullArchivedRepository() (int64, error) {
IsArchived: false,
})
}
+
+// CountBadSequences looks for broken sequences from recreate-table mistakes
+func CountBadSequences() (int64, error) {
+ if !setting.Database.UsePostgreSQL {
+ return 0, nil
+ }
+
+ sess := x.NewSession()
+ defer sess.Close()
+
+ var sequences []string
+ schema := sess.Engine().Dialect().URI().Schema
+
+ sess.Engine().SetSchema("")
+ if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+ return 0, err
+ }
+ sess.Engine().SetSchema(schema)
+
+ return int64(len(sequences)), nil
+}
+
+// FixBadSequences fixes for broken sequences from recreate-table mistakes
+func FixBadSequences() error {
+ if !setting.Database.UsePostgreSQL {
+ return nil
+ }
+
+ sess := x.NewSession()
+ defer sess.Close()
+ if err := sess.Begin(); err != nil {
+ return err
+ }
+
+ var sequences []string
+ schema := sess.Engine().Dialect().URI().Schema
+
+ sess.Engine().SetSchema("")
+ if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+ return err
+ }
+ sess.Engine().SetSchema(schema)
+
+ sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
+
+ for _, sequence := range sequences {
+ tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
+ newSequenceName := tableName + "_id_seq"
+ if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+ return err
+ }
+ if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+ return err
+ }
+ }
+
+ return sess.Commit()
+}
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index 5cb85cc18c..d16f8c7ec9 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -516,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
return err
}
case setting.Database.UsePostgreSQL:
+ var originalSequences []string
+ type sequenceData struct {
+ LastValue int `xorm:"'last_value'"`
+ IsCalled bool `xorm:"'is_called'"`
+ }
+ sequenceMap := map[string]sequenceData{}
+
+ schema := sess.Engine().Dialect().URI().Schema
+ sess.Engine().SetSchema("")
+ if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
+ log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+ return err
+ }
+ sess.Engine().SetSchema(schema)
+
+ for _, sequence := range originalSequences {
+ sequenceData := sequenceData{}
+ if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
+ log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
+ return err
+ }
+ sequenceMap[sequence] = sequenceData
+
+ }
+
// CASCADE causes postgres to drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@@ -529,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
var indices []string
- schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@@ -545,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
}
+ var sequences []string
+ sess.Engine().SetSchema("")
+ if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
+ log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+ return err
+ }
+ sess.Engine().SetSchema(schema)
+
+ for _, sequence := range sequences {
+ newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
+ if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+ log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
+ return err
+ }
+ val, ok := sequenceMap[newSequenceName]
+ if newSequenceName == tableName+"_id_seq" {
+ if ok && val.LastValue != 0 {
+ if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+ log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+ return err
+ }
+ } else {
+ // We're going to try to guess this
+ if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+ log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
+ return err
+ }
+ }
+ } else if ok {
+ if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+ log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+ return err
+ }
+ }
+
+ }
+
case setting.Database.UseMSSQL:
// MSSQL will drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {