2015-02-11 21:58:37 -05:00
// Copyright 2015 The Gogs Authors. All rights reserved.
2019-02-18 11:00:27 -05:00
// Copyright 2017 The Gitea Authors. All rights reserved.
2015-02-11 21:58:37 -05:00
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
2015-01-22 07:49:52 -05:00
package migrations
import (
2021-01-06 10:11:23 -05:00
"context"
2021-11-14 03:11:49 -05:00
"errors"
2015-02-11 21:58:37 -05:00
"fmt"
2020-10-12 10:35:56 -04:00
"os"
2020-09-06 17:52:01 -04:00
"reflect"
2019-07-01 15:26:59 -04:00
"regexp"
2015-01-23 02:54:16 -05:00
"strings"
2015-01-22 07:56:50 -05:00
2016-11-10 11:24:48 -05:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
2019-08-23 12:40:30 -04:00
2019-10-17 05:26:49 -04:00
"xorm.io/xorm"
2021-01-30 10:24:25 -05:00
"xorm.io/xorm/names"
2021-01-06 10:11:23 -05:00
"xorm.io/xorm/schemas"
2015-01-22 07:49:52 -05:00
)
2020-01-31 08:42:45 -05:00
const minDBVersion = 70 // Gitea 1.5.3
2015-02-11 21:58:37 -05:00
2016-11-28 10:44:17 -05:00
// Migration describes on migration from lower version to high version
2015-02-11 23:10:30 -05:00
type Migration interface {
Description ( ) string
Migrate ( * xorm . Engine ) error
}
type migration struct {
description string
migrate func ( * xorm . Engine ) error
}
2016-11-28 10:44:17 -05:00
// NewMigration creates a new migration
2015-02-11 23:10:30 -05:00
func NewMigration ( desc string , fn func ( * xorm . Engine ) error ) Migration {
return & migration { desc , fn }
}
2016-11-28 10:44:17 -05:00
// Description returns the migration's description
2015-02-11 23:10:30 -05:00
func ( m * migration ) Description ( ) string {
return m . description
}
2016-11-28 10:44:17 -05:00
// Migrate executes the migration
2015-02-11 23:10:30 -05:00
func ( m * migration ) Migrate ( x * xorm . Engine ) error {
return m . migrate ( x )
}
2015-01-22 07:49:52 -05:00
2016-11-28 10:44:17 -05:00
// Version describes the version table. Should have only one row with id==1
2015-01-22 07:49:52 -05:00
type Version struct {
2016-07-15 22:08:04 -04:00
ID int64 ` xorm:"pk autoincr" `
2015-01-22 07:49:52 -05:00
Version int64
}
2022-07-01 12:04:01 -04:00
// Use noopMigration when there is a migration that has been no-oped
var noopMigration = func ( _ * xorm . Engine ) error { return nil }
2015-01-22 07:49:52 -05:00
// This is a sequence of migrations. Add new migrations to the bottom of the list.
2015-02-12 12:46:21 -05:00
// If you want to "retire" a migration, remove it from the top of the list and
2016-11-28 10:44:17 -05:00
// update minDBVersion accordingly
2015-02-11 23:10:30 -05:00
var migrations = [ ] Migration {
2020-12-02 11:04:19 -05:00
// Gitea 1.5.0 ends at v69
2022-04-28 11:45:33 -04:00
2018-07-17 17:23:58 -04:00
// v70 -> v71
NewMigration ( "add issue_dependencies" , addIssueDependencies ) ,
2018-08-06 00:43:22 -04:00
// v71 -> v72
2018-07-27 08:54:50 -04:00
NewMigration ( "protect each scratch token" , addScratchHash ) ,
2018-08-06 00:43:22 -04:00
// v72 -> v73
NewMigration ( "add review" , addReview ) ,
2020-01-31 08:42:45 -05:00
2020-12-02 11:04:19 -05:00
// Gitea 1.6.0 ends at v73
2020-01-31 08:42:45 -05:00
2018-09-13 08:04:25 -04:00
// v73 -> v74
NewMigration ( "add must_change_password column for users table" , addMustChangePassword ) ,
2018-12-11 06:28:37 -05:00
// v74 -> v75
NewMigration ( "add approval whitelists to protected branches" , addApprovalWhitelistsToProtectedBranches ) ,
2018-12-18 11:26:26 -05:00
// v75 -> v76
NewMigration ( "clear nonused data which not deleted when user was deleted" , clearNonusedData ) ,
2020-01-31 08:42:45 -05:00
2020-12-02 11:04:19 -05:00
// Gitea 1.7.0 ends at v76
2020-01-31 08:42:45 -05:00
2018-12-27 05:27:08 -05:00
// v76 -> v77
NewMigration ( "add pull request rebase with merge commit" , addPullRequestRebaseWithMerge ) ,
2019-01-09 12:22:57 -05:00
// v77 -> v78
NewMigration ( "add theme to users" , addUserDefaultTheme ) ,
2019-01-17 19:01:04 -05:00
// v78 -> v79
NewMigration ( "rename repo is_bare to repo is_empty" , renameRepoIsBareToIsEmpty ) ,
2019-02-10 14:27:19 -05:00
// v79 -> v80
NewMigration ( "add can close issues via commit in any branch" , addCanCloseIssuesViaCommitInAnyBranch ) ,
2019-02-18 15:55:04 -05:00
// v80 -> v81
NewMigration ( "add is locked to issues" , addIsLockedToIssues ) ,
2019-03-04 21:34:52 -05:00
// v81 -> v82
NewMigration ( "update U2F counter type" , changeU2FCounterType ) ,
2020-01-31 08:42:45 -05:00
2020-12-02 11:04:19 -05:00
// Gitea 1.8.0 ends at v82
2020-01-31 08:42:45 -05:00
2019-03-10 23:44:58 -04:00
// v82 -> v83
NewMigration ( "hot fix for wrong release sha1 on release table" , fixReleaseSha1OnReleaseTable ) ,
2019-04-02 15:25:05 -04:00
// v83 -> v84
NewMigration ( "add uploader id for table attachment" , addUploaderIDForAttachment ) ,
2019-04-14 12:43:56 -04:00
// v84 -> v85
NewMigration ( "add table to store original imported gpg keys" , addGPGKeyImport ) ,
2019-05-04 11:45:34 -04:00
// v85 -> v86
NewMigration ( "hash application token" , hashAppToken ) ,
2019-05-05 14:09:02 -04:00
// v86 -> v87
NewMigration ( "add http method to webhook" , addHTTPMethodToWebhook ) ,
2019-05-29 22:22:26 -04:00
// v87 -> v88
NewMigration ( "add avatar field to repository" , addAvatarFieldToRepository ) ,
2020-01-31 08:42:45 -05:00
2020-12-02 11:04:19 -05:00
// Gitea 1.9.0 ends at v88
2020-01-31 08:42:45 -05:00
2019-06-30 03:57:59 -04:00
// v88 -> v89
NewMigration ( "add commit status context field to commit_status" , addCommitStatusContext ) ,
2019-07-07 22:14:12 -04:00
// v89 -> v90
NewMigration ( "add original author/url migration info to issues, comments, and repo " , addOriginalMigrationInfo ) ,
2019-07-28 23:29:42 -04:00
// v90 -> v91
NewMigration ( "change length of some repository columns" , changeSomeColumnsLengthOfRepo ) ,
2019-08-05 10:29:40 -04:00
// v91 -> v92
NewMigration ( "add index on owner_id of repository and type, review_id of comment" , addIndexOnRepositoryAndComment ) ,
2019-08-06 04:57:55 -04:00
// v92 -> v93
NewMigration ( "remove orphaned repository index statuses" , removeLingeringIndexStatus ) ,
2019-08-29 10:05:42 -04:00
// v93 -> v94
NewMigration ( "add email notification enabled preference to user" , addEmailNotificationEnabledToUser ) ,
2019-09-18 01:39:45 -04:00
// v94 -> v95
NewMigration ( "add enable_status_check, status_check_contexts to protected_branch" , addStatusCheckColumnsForProtectedBranches ) ,
2019-09-20 01:45:38 -04:00
// v95 -> v96
NewMigration ( "add table columns for cross referencing issues" , addCrossReferenceColumns ) ,
2019-09-22 05:05:48 -04:00
// v96 -> v97
NewMigration ( "delete orphaned attachments" , deleteOrphanedAttachments ) ,
2019-09-23 16:08:03 -04:00
// v97 -> v98
NewMigration ( "add repo_admin_change_team_access to user" , addRepoAdminChangeTeamAccessColumnForUser ) ,
2019-10-05 07:09:27 -04:00
// v98 -> v99
NewMigration ( "add original author name and id on migrated release" , addOriginalAuthorOnMigratedReleases ) ,
2019-10-13 09:23:14 -04:00
// v99 -> v100
NewMigration ( "add task table and status column for repository table" , addTaskTable ) ,
2019-10-14 02:10:42 -04:00
// v100 -> v101
NewMigration ( "update migration repositories' service type" , updateMigrationServiceTypes ) ,
2019-10-18 02:58:36 -04:00
// v101 -> v102
2020-12-08 18:24:23 -05:00
NewMigration ( "change length of some external login users columns" , changeSomeColumnsLengthOfExternalLoginUser ) ,
2020-12-02 11:04:19 -05:00
// Gitea 1.10.0 ends at v102
2019-10-18 07:13:31 -04:00
// v102 -> v103
NewMigration ( "update migration repositories' service type" , dropColumnHeadUserNameOnPullRequest ) ,
2019-10-21 04:21:45 -04:00
// v103 -> v104
NewMigration ( "Add WhitelistDeployKeys to protected branch" , addWhitelistDeployKeysToBranches ) ,
2019-10-23 07:48:32 -04:00
// v104 -> v105
NewMigration ( "remove unnecessary columns from label" , removeLabelUneededCols ) ,
2019-11-06 04:37:14 -05:00
// v105 -> v106
NewMigration ( "add includes_all_repositories to teams" , addTeamIncludesAllRepositories ) ,
2019-11-10 04:22:19 -05:00
// v106 -> v107
NewMigration ( "add column `mode` to table watch" , addModeColumnToWatch ) ,
2019-11-11 10:15:29 -05:00
// v107 -> v108
NewMigration ( "Add template options to repository" , addTemplateToRepo ) ,
2019-11-12 03:33:34 -05:00
// v108 -> v109
NewMigration ( "Add comment_id on table notification" , addCommentIDOnNotification ) ,
2019-11-20 06:27:49 -05:00
// v109 -> v110
NewMigration ( "add can_create_org_repo to team" , addCanCreateOrgRepoColumnForTeam ) ,
2019-12-02 13:32:40 -05:00
// v110 -> v111
NewMigration ( "change review content type to text" , changeReviewContentToText ) ,
2019-12-03 20:08:56 -05:00
// v111 -> v112
NewMigration ( "update branch protection for can push and whitelist enable" , addBranchProtectionCanPushAndEnableWhitelist ) ,
2019-12-13 22:30:39 -05:00
// v112 -> v113
NewMigration ( "remove release attachments which repository deleted" , removeAttachmentMissedRepo ) ,
2019-12-16 01:20:25 -05:00
// v113 -> v114
NewMigration ( "new feature: change target branch of pull requests" , featureChangeTargetBranch ) ,
2019-12-19 04:49:48 -05:00
// v114 -> v115
NewMigration ( "Remove authentication credentials from stored URL" , sanitizeOriginalURL ) ,
2019-12-27 13:27:59 -05:00
// v115 -> v116
NewMigration ( "add user_id prefix to existing user avatar name" , renameExistingUserAvatarName ) ,
2019-12-27 15:30:58 -05:00
// v116 -> v117
2020-12-08 18:24:23 -05:00
NewMigration ( "Extend TrackedTimes" , extendTrackedTimes ) ,
2020-12-02 11:04:19 -05:00
// Gitea 1.11.0 ends at v117
2020-01-03 12:47:10 -05:00
// v117 -> v118
NewMigration ( "Add block on rejected reviews branch protection" , addBlockOnRejectedReviews ) ,
2020-01-08 20:47:45 -05:00
// v118 -> v119
NewMigration ( "Add commit id and stale to reviews" , addReviewCommitAndStale ) ,
2020-01-10 10:35:17 -05:00
// v119 -> v120
NewMigration ( "Fix migrated repositories' git service type" , fixMigratedRepositoryServiceType ) ,
2020-01-12 04:36:21 -05:00
// v120 -> v121
NewMigration ( "Add owner_name on table repository" , addOwnerNameOnRepository ) ,
2020-01-13 12:33:46 -05:00
// v121 -> v122
NewMigration ( "add is_restricted column for users table" , addIsRestricted ) ,
2020-01-15 03:32:57 -05:00
// v122 -> v123
NewMigration ( "Add Require Signed Commits to ProtectedBranch" , addRequireSignedCommits ) ,
2020-01-15 06:14:07 -05:00
// v123 -> v124
2021-07-08 07:38:13 -04:00
NewMigration ( "Add original information for reactions" , addReactionOriginals ) ,
2020-01-19 17:27:44 -05:00
// v124 -> v125
NewMigration ( "Add columns to user and repository" , addUserRepoMissingColumns ) ,
2020-01-23 12:28:15 -05:00
// v125 -> v126
NewMigration ( "Add some columns on review for migration" , addReviewMigrateInfo ) ,
2020-01-31 01:57:19 -05:00
// v126 -> v127
NewMigration ( "Fix topic repository count" , fixTopicRepositoryCount ) ,
2020-02-11 04:34:17 -05:00
// v127 -> v128
NewMigration ( "add repository code language statistics" , addLanguageStats ) ,
2020-03-05 02:18:07 -05:00
// v128 -> v129
NewMigration ( "fix merge base for pull requests" , fixMergeBase ) ,
2020-03-05 10:54:50 -05:00
// v129 -> v130
NewMigration ( "remove dependencies from deleted repositories" , purgeUnusedDependencies ) ,
2020-03-06 00:10:48 -05:00
// v130 -> v131
NewMigration ( "Expand webhooks for more granularity" , expandWebhooks ) ,
2020-03-08 18:08:05 -04:00
// v131 -> v132
NewMigration ( "Add IsSystemWebhook column to webhooks table" , addSystemWebhookColumn ) ,
2020-03-26 18:26:34 -04:00
// v132 -> v133
NewMigration ( "Add Branch Protection Protected Files Column" , addBranchProtectionProtectedFilesColumn ) ,
2020-03-27 08:34:39 -04:00
// v133 -> v134
NewMigration ( "Add EmailHash Table" , addEmailHashTable ) ,
2020-03-31 09:42:44 -04:00
// v134 -> v135
NewMigration ( "Refix merge base for merged pull requests" , refixMergeBase ) ,
2020-04-16 21:00:36 -04:00
// v135 -> v136
Add Organization Wide Labels (#10814)
* Add organization wide labels
Implement organization wide labels similar to organization wide
webhooks. This lets you create individual labels for organizations that can be used
for all repos under that organization (so being able to reuse the same
label across multiple repos).
This makes it possible for small organizations with many repos to use
labels effectively.
Fixes #7406
* Add migration
* remove comments
* fix tests
* Update options/locale/locale_en-US.ini
Removed unused translation string
* show org labels in issue search label filter
* Use more clear var name
* rename migration after merge from master
* comment typo
* update migration again after rebase with master
* check for orgID <=0 per guillep2k review
* fmt
* Apply suggestions from code review
Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* remove unused code
* Make sure RepoID is 0 when searching orgID per code review
* more changes/code review requests
* More descriptive translation var per code review
* func description/delete comment when issue label deleted instead of hiding it
* remove comment
* only use issues in that repo when calculating number of open issues for org label on repo label page
* Add integration test for IssuesSearch API with labels
* remove unused function
* Update models/issue_label.go
Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* Use subquery in GetLabelIDsInReposByNames
* Fix tests to use correct orgID
* fix more tests
* IssuesSearch api now uses new BuildLabelNamesIssueIDsCondition. Add a few more tests as well
* update comment for clarity
* Revert previous code change now that we can use the new BuildLabelNamesIssueIDsCondition
* Don't sort repos by date in IssuesSearch API
After much debugging I've found a strange issue where in some cases MySQL will return a different result than other enigines if a query is sorted by a null collumn. For example with our integration test data where we don't set updated_unix in repository fixtures:
SELECT `id`, `owner_id`, `owner_name`, `lower_name`, `name`, `description`, `website`, `original_service_type`, `original_url`, `default_branch`, `num_watches`, `num_stars`, `num_forks`, `num_issues`, `num_closed_issues`, `num_pulls`, `num_closed_pulls`, `num_milestones`, `num_closed_milestones`, `is_private`, `is_empty`, `is_archived`, `is_mirror`, `status`, `is_fork`, `fork_id`, `is_template`, `template_id`, `size`, `is_fsck_enabled`, `close_issues_via_commit_in_any_branch`, `topics`, `avatar`, `created_unix`, `updated_unix` FROM `repository` ORDER BY updated_unix DESC LIMIT 15 OFFSET 45
Returns different results for MySQL than other engines. However, the similar query:
SELECT `id`, `owner_id`, `owner_name`, `lower_name`, `name`, `description`, `website`, `original_service_type`, `original_url`, `default_branch`, `num_watches`, `num_stars`, `num_forks`, `num_issues`, `num_closed_issues`, `num_pulls`, `num_closed_pulls`, `num_milestones`, `num_closed_milestones`, `is_private`, `is_empty`, `is_archived`, `is_mirror`, `status`, `is_fork`, `fork_id`, `is_template`, `template_id`, `size`, `is_fsck_enabled`, `close_issues_via_commit_in_any_branch`, `topics`, `avatar`, `created_unix`, `updated_unix` FROM `repository` ORDER BY updated_unix DESC LIMIT 15 OFFSET 30
Returns the same results.
This causes integration tests to fail on MySQL in certain cases but would never show up in a real installation. Since this API call always returns issues based on the optionally provided repo_priority_id or the issueID itself, there is no change to results by changing the repo sorting method used to get ids earlier in the function.
* linter is back!
* code review
* remove now unused option
* Fix newline at end of files
* more unused code
* update to master
* check for matching ids before query
* Update models/issue_label.go
Co-Authored-By: 6543 <6543@obermui.de>
* Update models/issue_label.go
* update comments
* Update routers/org/setting.go
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: 6543 <6543@obermui.de>
2020-04-01 00:14:46 -04:00
NewMigration ( "Add OrgID column to Labels table" , addOrgIDLabelColumn ) ,
2020-04-16 21:00:36 -04:00
// v136 -> v137
2020-04-14 09:53:34 -04:00
NewMigration ( "Add CommitsAhead and CommitsBehind Column to PullRequest Table" , addCommitDivergenceToPulls ) ,
2020-04-16 21:00:36 -04:00
// v137 -> v138
NewMigration ( "Add Branch Protection Block Outdated Branch" , addBlockOnOutdatedBranch ) ,
2020-04-18 09:50:25 -04:00
// v138 -> v139
NewMigration ( "Add ResolveDoerID to Comment table" , addResolveDoerIDCommentColumn ) ,
2020-05-14 18:55:43 -04:00
// v139 -> v140
2020-12-08 18:24:23 -05:00
NewMigration ( "prepend refs/heads/ to issue refs" , prependRefsHeadsToIssueRefs ) ,
2020-12-02 11:04:19 -05:00
// Gitea 1.12.0 ends at v140
2020-05-30 03:46:15 -04:00
// v140 -> v141
NewMigration ( "Save detected language file size to database instead of percent" , fixLanguageStatsToSaveSize ) ,
2020-06-11 16:18:11 -04:00
// v141 -> v142
2020-06-05 16:01:53 -04:00
NewMigration ( "Add KeepActivityPrivate to User table" , addKeepActivityPrivateUserColumn ) ,
2020-06-11 16:18:11 -04:00
// v142 -> v143
NewMigration ( "Ensure Repository.IsArchived is not null" , setIsArchivedToFalse ) ,
2020-07-07 15:16:34 -04:00
// v143 -> v144
NewMigration ( "recalculate Stars number for all user" , recalculateStars ) ,
2020-07-30 18:04:19 -04:00
// v144 -> v145
NewMigration ( "update Matrix Webhook http method to 'PUT'" , updateMatrixWebhookHTTPMethod ) ,
2020-08-04 09:54:29 -04:00
// v145 -> v146
NewMigration ( "Increase Language field to 50 in LanguageStats" , increaseLanguageField ) ,
2020-08-16 23:07:38 -04:00
// v146 -> v147
NewMigration ( "Add projects info to repository table" , addProjectsInfo ) ,
2020-08-21 03:53:14 -04:00
// v147 -> v148
NewMigration ( "create review for 0 review id code comments" , createReviewsForCodeComments ) ,
2020-09-03 21:36:56 -04:00
// v148 -> v149
NewMigration ( "remove issue dependency comments who refer to non existing issues" , purgeInvalidDependenciesComments ) ,
2020-09-05 13:38:54 -04:00
// v149 -> v150
NewMigration ( "Add Created and Updated to Milestone table" , addCreatedAndUpdatedToMilestones ) ,
2020-09-10 15:45:01 -04:00
// v150 -> v151
NewMigration ( "add primary key to repo_topic" , addPrimaryKeyToRepoTopic ) ,
2020-09-15 18:02:41 -04:00
// v151 -> v152
NewMigration ( "set default password algorithm to Argon2" , setDefaultPasswordToArgon2 ) ,
2020-09-19 12:44:55 -04:00
// v152 -> v153
NewMigration ( "add TrustModel field to Repository" , addTrustModelToRepository ) ,
2020-10-12 15:55:13 -04:00
// v153 > v154
NewMigration ( "add Team review request support" , addTeamReviewRequestSupport ) ,
2020-10-12 20:01:57 -04:00
// v154 > v155
2020-12-08 18:24:23 -05:00
NewMigration ( "add timestamps to Star, Label, Follow, Watch and Collaboration" , addTimeStamps ) ,
2020-12-02 11:04:19 -05:00
// Gitea 1.13.0 ends at v155
2020-10-13 14:50:57 -04:00
// v155 -> v156
NewMigration ( "add changed_protected_files column for pull_request table" , addChangedProtectedFilesPullRequestColumn ) ,
2020-10-21 20:55:25 -04:00
// v156 -> v157
NewMigration ( "fix publisher ID for tag releases" , fixPublisherIDforTagReleases ) ,
2020-10-24 10:11:30 -04:00
// v157 -> v158
NewMigration ( "ensure repo topics are up-to-date" , fixRepoTopics ) ,
2020-11-09 01:15:09 -05:00
// v158 -> v159
NewMigration ( "code comment replies should have the commitID of the review they are replying to" , updateCodeCommentReplies ) ,
2020-11-10 17:37:11 -05:00
// v159 -> v160
NewMigration ( "update reactions constraint" , updateReactionConstraint ) ,
2020-11-28 14:30:46 -05:00
// v160 -> v161
NewMigration ( "Add block on official review requests branch protection" , addBlockOnOfficialReviewRequests ) ,
2020-12-08 05:41:14 -05:00
// v161 -> v162
NewMigration ( "Convert task type from int to string" , convertTaskTypeToString ) ,
2020-12-09 12:20:13 -05:00
// v162 -> v163
NewMigration ( "Convert webhook task type from int to string" , convertWebhookTaskTypeToString ) ,
2020-12-26 18:28:47 -05:00
// v163 -> v164
NewMigration ( "Convert topic name from 25 to 50" , convertTopicNameFrom25To50 ) ,
2021-01-01 11:33:27 -05:00
// v164 -> v165
NewMigration ( "Add scope and nonce columns to oauth2_grant table" , addScopeAndNonceColumnsToOAuth2Grant ) ,
2021-01-06 10:11:23 -05:00
// v165 -> v166
NewMigration ( "Convert hook task type from char(16) to varchar(16) and trim the column" , convertHookTaskTypeToVarcharAndTrim ) ,
2021-01-10 13:05:18 -05:00
// v166 -> v167
NewMigration ( "Where Password is Valid with Empty String delete it" , recalculateUserEmptyPWD ) ,
2021-01-24 10:23:05 -05:00
// v167 -> v168
NewMigration ( "Add user redirect" , addUserRedirect ) ,
2021-01-28 17:58:33 -05:00
// v168 -> v169
NewMigration ( "Recreate user table to fix default values" , recreateUserTableToFixDefaultValues ) ,
2021-02-07 22:09:14 -05:00
// v169 -> v170
NewMigration ( "Update DeleteBranch comments to set the old_ref to the commit_sha" , commentTypeDeleteBranchUseOldRef ) ,
2021-02-11 12:32:25 -05:00
// v170 -> v171
NewMigration ( "Add Dismissed to Review table" , addDismissedReviewColumn ) ,
2021-02-12 06:01:26 -05:00
// v171 -> v172
NewMigration ( "Add Sorting to ProjectBoard table" , addSortingColToProjectBoard ) ,
2021-02-15 00:33:31 -05:00
// v172 -> v173
NewMigration ( "Add sessions table for go-chi/session" , addSessionTable ) ,
2021-02-19 05:52:11 -05:00
// v173 -> v174
NewMigration ( "Add time_id column to Comment" , addTimeIDCommentColumn ) ,
2021-02-28 19:47:30 -05:00
// v174 -> v175
2021-03-19 15:01:24 -04:00
NewMigration ( "Create repo transfer table" , addRepoTransfer ) ,
2021-03-18 02:06:40 -04:00
// v175 -> v176
NewMigration ( "Fix Postgres ID Sequences broken by recreate-table" , fixPostgresIDSequences ) ,
2021-03-19 09:25:14 -04:00
// v176 -> v177
NewMigration ( "Remove invalid labels from comments" , removeInvalidLabels ) ,
2021-03-19 15:01:24 -04:00
// v177 -> v178
NewMigration ( "Delete orphaned IssueLabels" , deleteOrphanedIssueLabels ) ,
2021-04-14 02:46:17 -04:00
// Gitea 1.14.0 ends at v178
2021-04-08 18:25:57 -04:00
// v178 -> v179
NewMigration ( "Add LFS columns to Mirror" , addLFSMirrorColumns ) ,
2021-04-14 08:02:12 -04:00
// v179 -> v180
NewMigration ( "Convert avatar url to text" , convertAvatarURLToText ) ,
2021-05-31 04:25:47 -04:00
// v180 -> v181
NewMigration ( "Delete credentials from past migrations" , deleteMigrationCredentials ) ,
2021-06-07 23:52:51 -04:00
// v181 -> v182
NewMigration ( "Always save primary email on email address table" , addPrimaryEmail2EmailAddress ) ,
2021-06-13 22:22:55 -04:00
// v182 -> v183
NewMigration ( "Add issue resource index table" , addIssueResourceIndexTable ) ,
2021-06-14 13:20:43 -04:00
// v183 -> v184
NewMigration ( "Create PushMirror table" , createPushMirrorTable ) ,
2021-06-16 18:02:24 -04:00
// v184 -> v185
NewMigration ( "Rename Task errors to message" , renameTaskErrorsToMessage ) ,
2021-06-23 17:12:38 -04:00
// v185 -> v186
NewMigration ( "Add new table repo_archiver" , addRepoArchiver ) ,
2021-06-25 10:28:55 -04:00
// v186 -> v187
NewMigration ( "Create protected tag table" , createProtectedTagTable ) ,
2021-06-27 15:21:09 -04:00
// v187 -> v188
NewMigration ( "Drop unneeded webhook related columns" , dropWebhookColumns ) ,
2021-07-13 09:28:07 -04:00
// v188 -> v189
NewMigration ( "Add key is verified to gpg key" , addKeyIsVerified ) ,
2021-08-08 13:34:42 -04:00
// Gitea 1.15.0 ends at v189
2021-07-24 06:16:34 -04:00
// v189 -> v190
NewMigration ( "Unwrap ldap.Sources" , unwrapLDAPSourceCfg ) ,
2021-07-28 05:42:56 -04:00
// v190 -> v191
NewMigration ( "Add agit flow pull request support" , addAgitFlowPullRequest ) ,
2021-08-22 11:33:05 -04:00
// v191 -> v192
NewMigration ( "Alter issue/comment table TEXT fields to LONGTEXT" , alterIssueAndCommentTextFieldsToLongText ) ,
2021-08-25 04:42:51 -04:00
// v192 -> v193
NewMigration ( "RecreateIssueResourceIndexTable to have a primary key instead of an unique index" , recreateIssueResourceIndexTable ) ,
2021-09-08 11:19:30 -04:00
// v193 -> v194
NewMigration ( "Add repo id column for attachment table" , addRepoIDForAttachment ) ,
2021-09-11 10:21:17 -04:00
// v194 -> v195
NewMigration ( "Add Branch Protection Unprotected Files Column" , addBranchProtectionUnprotectedFilesColumn ) ,
2021-09-23 08:42:42 -04:00
// v195 -> v196
2021-09-23 06:50:06 -04:00
NewMigration ( "Add table commit_status_index" , addTableCommitStatusIndex ) ,
2021-09-29 16:53:12 -04:00
// v196 -> v197
NewMigration ( "Add Color to ProjectBoard table" , addColorColToProjectBoard ) ,
2021-10-08 13:03:04 -04:00
// v197 -> v198
NewMigration ( "Add renamed_branch table" , addRenamedBranchTable ) ,
2021-10-10 18:40:03 -04:00
// v198 -> v199
NewMigration ( "Add issue content history table" , addTableIssueContentHistory ) ,
2021-10-16 02:14:34 -04:00
// v199 -> v200
2022-07-01 12:04:01 -04:00
NewMigration ( "No-op (remote version is using AppState now)" , noopMigration ) ,
2021-10-21 05:22:43 -04:00
// v200 -> v201
NewMigration ( "Add table app_state" , addTableAppState ) ,
2021-10-21 12:10:49 -04:00
// v201 -> v202
NewMigration ( "Drop table remote_version (if exists)" , dropTableRemoteVersion ) ,
2021-11-22 04:47:23 -05:00
// v202 -> v203
NewMigration ( "Create key/value table for user settings" , createUserSettingsTable ) ,
2021-12-08 01:57:18 -05:00
// v203 -> v204
NewMigration ( "Add Sorting to ProjectIssue table" , addProjectIssueSorting ) ,
2021-12-19 00:37:18 -05:00
// v204 -> v205
NewMigration ( "Add key is verified to ssh key" , addSSHKeyIsVerified ) ,
2022-01-04 10:13:52 -05:00
// v205 -> v206
NewMigration ( "Migrate to higher varchar on user struct" , migrateUserPasswordSalt ) ,
2022-01-04 22:37:00 -05:00
// v206 -> v207
NewMigration ( "Add authorize column to team_unit table" , addAuthorizeColForTeamUnit ) ,
2022-01-14 10:03:31 -05:00
// v207 -> v208
2022-02-16 16:03:58 -05:00
NewMigration ( "Add webauthn table and migrate u2f data to webauthn - NO-OPED" , addWebAuthnCred ) ,
2022-01-15 11:52:56 -05:00
// v208 -> v209
2022-02-16 16:03:58 -05:00
NewMigration ( "Use base32.HexEncoding instead of base64 encoding for cred ID as it is case insensitive - NO-OPED" , useBase32HexForCredIDInWebAuthnCredential ) ,
2022-02-13 16:19:12 -05:00
// v209 -> v210
2022-02-16 16:03:58 -05:00
NewMigration ( "Increase WebAuthentication CredentialID size to 410 - NO-OPED" , increaseCredentialIDTo410 ) ,
// v210 -> v211
NewMigration ( "v208 was completely broken - remigrate" , remigrateU2FCredentials ) ,
2022-03-17 14:04:36 -04:00
// Gitea 1.16.2 ends at v211
Store the foreign ID of issues during migration (#18446)
Storing the foreign identifier of an imported issue in the database is a prerequisite to implement idempotent migrations or mirror for issues. It is a baby step towards mirroring that introduces a new table.
At the moment when an issue is created by the Gitea uploader, it fails if the issue already exists. The Gitea uploader could be modified so that, instead of failing, it looks up the database to find an existing issue. And if it does it would update the issue instead of creating a new one. However this is not currently possible because an information is missing from the database: the foreign identifier that uniquely represents the issue being migrated is not persisted. With this change, the foreign identifier is stored in the database and the Gitea uploader will then be able to run a query to figure out if a given issue being imported already exists.
The implementation of mirroring for issues, pull requests, releases, etc. can be done in three steps:
1. Store an identifier for the element being mirrored (issue, pull request...) in the database (this is the purpose of these changes)
2. Modify the Gitea uploader to be able to update an existing repository with all it contains (issues, pull request...) instead of failing if it exists
3. Optimize the Gitea uploader to speed up the updates, when possible.
The second step creates code that does not yet exist to enable idempotent migrations with the Gitea uploader. When a migration is done for the first time, the behavior is not changed. But when a migration is done for a repository that already exists, this new code is used to update it.
The third step can use the code created in the second step to optimize and speed up migrations. For instance, when a migration is resumed, an issue that has an update time that is not more recent can be skipped and only newly created issues or updated ones will be updated. Another example of optimization could be that a webhook notifies Gitea when an issue is updated. The code triggered by the webhook would download only this issue and call the code created in the second step to update the issue, as if it was in the process of an idempotent migration.
The ForeignReferences table is added to contain local and foreign ID pairs relative to a given repository. It can later be used for pull requests and other artifacts that can be mirrored. Although the foreign id could be added as a single field in issues or pull requests, it would need to be added to all tables that represent something that can be mirrored. Creating a new table makes for a simpler and more generic design. The drawback is that it requires an extra lookup to obtain the information. However, this extra information is only required during migration or mirroring and does not impact the way Gitea currently works.
The foreign identifier of an issue or pull request is similar to the identifier of an external user, which is stored in reactions, issues, etc. as OriginalPosterID and so on. The representation of a user is however different and the ability of users to link their account to an external user at a later time is also a logic that is different from what is involved in mirroring or migrations. For these reasons, despite some commonalities, it is unclear at this time how the two tables (foreign reference and external user) could be merged together.
The ForeignID field is extracted from the issue migration context so that it can be dumped in files with dump-repo and later restored via restore-repo.
The GetAllComments downloader method is introduced to simplify the implementation and not overload the Context for the purpose of pagination. It also clarifies in which context the comments are paginated and in which context they are not.
The Context interface is no longer useful for the purpose of retrieving the LocalID and ForeignID since they are now both available from the PullRequest and Issue struct. The Reviewable and Commentable interfaces replace and serve the same purpose.
The Context data member of PullRequest and Issue becomes a DownloaderContext to clarify that its purpose is not to support in memory operations while the current downloader is acting but is not otherwise persisted. It is, for instance, used by the GitLab downloader to store the IsMergeRequest boolean and sort out issues.
---
[source](https://lab.forgefriends.org/forgefriends/forgefriends/-/merge_requests/36)
Signed-off-by: Loïc Dachary <loic@dachary.org>
Co-authored-by: Loïc Dachary <loic@dachary.org>
2022-03-17 13:08:35 -04:00
// v211 -> v212
NewMigration ( "Create ForeignReference table" , createForeignReferenceTable ) ,
2022-03-30 04:42:47 -04:00
// v212 -> v213
NewMigration ( "Add package tables" , addPackageTables ) ,
2022-04-28 11:45:33 -04:00
// v213 -> v214
NewMigration ( "Add allow edits from maintainers to PullRequest table" , addAllowMaintainerEdit ) ,
2022-05-07 13:05:52 -04:00
// v214 -> v215
NewMigration ( "Add auto merge table" , addAutoMergeTable ) ,
2022-05-07 14:28:10 -04:00
// v215 -> v216
NewMigration ( "allow to view files in PRs" , addReviewViewedFiles ) ,
2022-06-18 04:46:50 -04:00
// v216 -> v217
2022-07-01 12:04:01 -04:00
NewMigration ( "No-op (Improve Action table indices v1)" , noopMigration ) ,
2022-06-19 14:47:04 -04:00
// v217 -> v218
NewMigration ( "Alter hook_task table TEXT fields to LONGTEXT" , alterHookTaskTextFieldsToLongText ) ,
2022-07-01 12:04:01 -04:00
// v218 -> v219
NewMigration ( "Improve Action table indices v2" , improveActionTableIndices ) ,
2022-07-08 15:45:12 -04:00
// v219 -> v220
NewMigration ( "Add sync_on_commit column to push_mirror table" , addSyncOnCommitColForPushMirror ) ,
2022-07-27 23:59:39 -04:00
// v220 -> v221
NewMigration ( "Add container repository property" , addContainerRepositoryProperty ) ,
2022-07-30 09:25:26 -04:00
// v221 -> v222
NewMigration ( "Store WebAuthentication CredentialID as bytes and increase size to at least 1024" , storeWebauthnCredentialIDAsBytes ) ,
// v222 -> v223
NewMigration ( "Drop old CredentialID column" , dropOldCredentialIDColumn ) ,
// v223 -> v224
NewMigration ( "Rename CredentialIDBytes column to CredentialID" , renameCredentialIDBytes ) ,
2022-08-22 09:32:28 -04:00
// Gitea 1.17.0 ends at v224
2022-08-18 01:38:59 -04:00
// v224 -> v225
2022-08-22 09:32:28 -04:00
NewMigration ( "Add badges to users" , createUserBadgesTable ) ,
// v225 -> v226
NewMigration ( "Alter gpg_key/public_key content TEXT fields to MEDIUMTEXT" , alterPublicGPGKeyContentFieldsToMediumText ) ,
2022-10-07 00:22:05 -04:00
// v226 -> v227
NewMigration ( "Conan and generic packages do not need to be semantically versioned" , fixPackageSemverField ) ,
2022-10-16 19:29:26 -04:00
// v227 -> v228
NewMigration ( "Create key/value table for system settings" , createSystemSettingsTable ) ,
2022-10-19 08:40:28 -04:00
// v228 -> v229
NewMigration ( "Add TeamInvite table" , addTeamInviteTable ) ,
2022-10-22 11:08:10 -04:00
// v229 -> v230
NewMigration ( "Update counts of all open milestones" , updateOpenMilestoneCounts ) ,
2022-10-24 03:59:24 -04:00
// v230 -> v231
NewMigration ( "Add ConfidentialClient column (default true) to OAuth2Application table" , addConfidentialClientColumnToOAuth2ApplicationTable ) ,
2015-01-23 02:54:16 -05:00
}
2015-01-22 07:49:52 -05:00
2020-04-06 06:44:47 -04:00
// GetCurrentDBVersion returns the current db version
func GetCurrentDBVersion ( x * xorm . Engine ) ( int64 , error ) {
if err := x . Sync ( new ( Version ) ) ; err != nil {
return - 1 , fmt . Errorf ( "sync: %v" , err )
}
currentVersion := & Version { ID : 1 }
has , err := x . Get ( currentVersion )
if err != nil {
return - 1 , fmt . Errorf ( "get: %v" , err )
}
if ! has {
return - 1 , nil
}
return currentVersion . Version , nil
}
// ExpectedVersion returns the expected db version
func ExpectedVersion ( ) int64 {
return int64 ( minDBVersion + len ( migrations ) )
}
// EnsureUpToDate will check if the db is at the correct version
func EnsureUpToDate ( x * xorm . Engine ) error {
currentDB , err := GetCurrentDBVersion ( x )
if err != nil {
return err
}
if currentDB < 0 {
2022-06-13 03:34:46 -04:00
return fmt . Errorf ( "Database has not been initialized" )
2020-04-06 06:44:47 -04:00
}
if minDBVersion > currentDB {
return fmt . Errorf ( "DB version %d (<= %d) is too old for auto-migration. Upgrade to Gitea 1.6.4 first then upgrade to this version" , currentDB , minDBVersion )
}
expected := ExpectedVersion ( )
if currentDB != expected {
return fmt . Errorf ( ` Current database version %d is not equal to the expected version %d. Please run "gitea [--config /path/to/app.ini] migrate" to update the database version ` , currentDB , expected )
}
return nil
}
2015-01-22 07:49:52 -05:00
// Migrate database to current version
func Migrate ( x * xorm . Engine ) error {
2021-01-30 10:24:25 -05:00
// Set a new clean the default mapper to GonicMapper as that is the default for Gitea.
x . SetMapper ( names . GonicMapper { } )
2015-01-22 08:01:45 -05:00
if err := x . Sync ( new ( Version ) ) ; err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "sync: %v" , err )
2015-01-22 08:01:45 -05:00
}
2015-01-22 07:49:52 -05:00
2016-07-15 22:08:04 -04:00
currentVersion := & Version { ID : 1 }
2015-01-22 07:49:52 -05:00
has , err := x . Get ( currentVersion )
if err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "get: %v" , err )
2015-01-22 07:56:50 -05:00
} else if ! has {
2015-12-10 19:52:06 -05:00
// If the version record does not exist we think
// it is a fresh installation and we can skip all migrations.
2016-12-23 20:37:35 -05:00
currentVersion . ID = 0
2016-11-28 10:44:17 -05:00
currentVersion . Version = int64 ( minDBVersion + len ( migrations ) )
2015-01-23 02:54:16 -05:00
2015-01-22 07:56:50 -05:00
if _ , err = x . InsertOne ( currentVersion ) ; err != nil {
2015-02-11 21:58:37 -05:00
return fmt . Errorf ( "insert: %v" , err )
2015-01-22 07:56:50 -05:00
}
2015-01-22 07:49:52 -05:00
}
v := currentVersion . Version
2016-11-28 10:44:17 -05:00
if minDBVersion > v {
2019-04-02 03:48:31 -04:00
log . Fatal ( ` Gitea no longer supports auto - migration from your previously installed version .
2020-01-31 08:42:45 -05:00
Please try upgrading to a lower version first ( suggested v1 .6 .4 ) , then upgrade to this version . ` )
2015-11-25 09:27:27 -05:00
return nil
}
2020-10-12 20:41:49 -04:00
// Downgrading Gitea's database version not supported
2016-11-28 10:44:17 -05:00
if int ( v - minDBVersion ) > len ( migrations ) {
2022-04-26 12:01:42 -04:00
msg := fmt . Sprintf ( "Your database (migration version: %d) is for a newer Gitea, you can not use the newer database for this old Gitea release (%d)." , v , minDBVersion + len ( migrations ) )
2022-02-07 05:04:12 -05:00
msg += "\nGitea will exit to keep your database safe and unchanged. Please use the correct Gitea release, do not change the migration version manually (incorrect manual operation may lose data)."
if ! setting . IsProd {
msg += fmt . Sprintf ( "\nIf you are in development and really know what you're doing, you can force changing the migration version by executing: UPDATE version SET version=%d WHERE id=1;" , minDBVersion + len ( migrations ) )
}
_ , _ = fmt . Fprintln ( os . Stderr , msg )
2020-10-12 10:35:56 -04:00
log . Fatal ( msg )
return nil
2015-08-10 10:59:12 -04:00
}
2020-10-12 10:35:56 -04:00
// Migrate
2016-11-28 10:44:17 -05:00
for i , m := range migrations [ v - minDBVersion : ] {
2019-05-05 19:42:29 -04:00
log . Info ( "Migration[%d]: %s" , v + int64 ( i ) , m . Description ( ) )
2021-01-30 10:24:25 -05:00
// Reset the mapper between each migration - migrations are not supposed to depend on each other
x . SetMapper ( names . GonicMapper { } )
2015-02-11 23:10:30 -05:00
if err = m . Migrate ( x ) ; err != nil {
2021-08-24 13:46:55 -04:00
return fmt . Errorf ( "migration[%d]: %s failed: %v" , v + int64 ( i ) , m . Description ( ) , err )
2015-01-22 07:49:52 -05:00
}
currentVersion . Version = v + int64 ( i ) + 1
2017-10-05 00:43:04 -04:00
if _ , err = x . ID ( 1 ) . Update ( currentVersion ) ; err != nil {
2015-01-22 08:01:45 -05:00
return err
}
2015-01-22 07:49:52 -05:00
}
return nil
}
2020-09-06 17:52:01 -04:00
// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table
// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
func RecreateTables ( beans ... interface { } ) func ( * xorm . Engine ) error {
return func ( x * xorm . Engine ) error {
sess := x . NewSession ( )
defer sess . Close ( )
if err := sess . Begin ( ) ; err != nil {
return err
}
sess = sess . StoreEngine ( "InnoDB" )
for _ , bean := range beans {
log . Info ( "Recreating Table: %s for Bean: %s" , x . TableName ( bean ) , reflect . Indirect ( reflect . ValueOf ( bean ) ) . Type ( ) . Name ( ) )
if err := recreateTable ( sess , bean ) ; err != nil {
return err
}
}
return sess . Commit ( )
}
}
// recreateTable will recreate the table using the newly provided bean definition and move all data to that new table
// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
// WARNING: YOU MUST COMMIT THE SESSION AT THE END
func recreateTable ( sess * xorm . Session , bean interface { } ) error {
// TODO: This will not work if there are foreign keys
tableName := sess . Engine ( ) . TableName ( bean )
tempTableName := fmt . Sprintf ( "tmp_recreate__%s" , tableName )
// We need to move the old table away and create a new one with the correct columns
// We will need to do this in stages to prevent data loss
//
// First create the temporary table
if err := sess . Table ( tempTableName ) . CreateTable ( bean ) ; err != nil {
log . Error ( "Unable to create table %s. Error: %v" , tempTableName , err )
return err
}
if err := sess . Table ( tempTableName ) . CreateUniques ( bean ) ; err != nil {
log . Error ( "Unable to create uniques for table %s. Error: %v" , tempTableName , err )
return err
}
if err := sess . Table ( tempTableName ) . CreateIndexes ( bean ) ; err != nil {
log . Error ( "Unable to create indexes for table %s. Error: %v" , tempTableName , err )
return err
}
// Work out the column names from the bean - these are the columns to select from the old table and install into the new table
table , err := sess . Engine ( ) . TableInfo ( bean )
if err != nil {
log . Error ( "Unable to get table info. Error: %v" , err )
return err
}
newTableColumns := table . Columns ( )
if len ( newTableColumns ) == 0 {
return fmt . Errorf ( "no columns in new table" )
}
hasID := false
for _ , column := range newTableColumns {
hasID = hasID || ( column . IsPrimaryKey && column . IsAutoIncrement )
}
if hasID && setting . Database . UseMSSQL {
if _ , err := sess . Exec ( fmt . Sprintf ( "SET IDENTITY_INSERT `%s` ON" , tempTableName ) ) ; err != nil {
log . Error ( "Unable to set identity insert for table %s. Error: %v" , tempTableName , err )
return err
}
}
sqlStringBuilder := & strings . Builder { }
_ , _ = sqlStringBuilder . WriteString ( "INSERT INTO `" )
_ , _ = sqlStringBuilder . WriteString ( tempTableName )
_ , _ = sqlStringBuilder . WriteString ( "` (`" )
_ , _ = sqlStringBuilder . WriteString ( newTableColumns [ 0 ] . Name )
_ , _ = sqlStringBuilder . WriteString ( "`" )
for _ , column := range newTableColumns [ 1 : ] {
_ , _ = sqlStringBuilder . WriteString ( ", `" )
_ , _ = sqlStringBuilder . WriteString ( column . Name )
_ , _ = sqlStringBuilder . WriteString ( "`" )
}
_ , _ = sqlStringBuilder . WriteString ( ")" )
_ , _ = sqlStringBuilder . WriteString ( " SELECT " )
if newTableColumns [ 0 ] . Default != "" {
_ , _ = sqlStringBuilder . WriteString ( "COALESCE(`" )
_ , _ = sqlStringBuilder . WriteString ( newTableColumns [ 0 ] . Name )
_ , _ = sqlStringBuilder . WriteString ( "`, " )
_ , _ = sqlStringBuilder . WriteString ( newTableColumns [ 0 ] . Default )
_ , _ = sqlStringBuilder . WriteString ( ")" )
} else {
_ , _ = sqlStringBuilder . WriteString ( "`" )
_ , _ = sqlStringBuilder . WriteString ( newTableColumns [ 0 ] . Name )
_ , _ = sqlStringBuilder . WriteString ( "`" )
}
for _ , column := range newTableColumns [ 1 : ] {
if column . Default != "" {
_ , _ = sqlStringBuilder . WriteString ( ", COALESCE(`" )
_ , _ = sqlStringBuilder . WriteString ( column . Name )
_ , _ = sqlStringBuilder . WriteString ( "`, " )
_ , _ = sqlStringBuilder . WriteString ( column . Default )
_ , _ = sqlStringBuilder . WriteString ( ")" )
} else {
_ , _ = sqlStringBuilder . WriteString ( ", `" )
_ , _ = sqlStringBuilder . WriteString ( column . Name )
_ , _ = sqlStringBuilder . WriteString ( "`" )
}
}
_ , _ = sqlStringBuilder . WriteString ( " FROM `" )
_ , _ = sqlStringBuilder . WriteString ( tableName )
_ , _ = sqlStringBuilder . WriteString ( "`" )
if _ , err := sess . Exec ( sqlStringBuilder . String ( ) ) ; err != nil {
log . Error ( "Unable to set copy data in to temp table %s. Error: %v" , tempTableName , err )
return err
}
if hasID && setting . Database . UseMSSQL {
if _ , err := sess . Exec ( fmt . Sprintf ( "SET IDENTITY_INSERT `%s` OFF" , tempTableName ) ) ; err != nil {
log . Error ( "Unable to switch off identity insert for table %s. Error: %v" , tempTableName , err )
return err
}
}
switch {
case setting . Database . UseSQLite3 :
// SQLite will drop all the constraints on the old table
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP TABLE `%s`" , tableName ) ) ; err != nil {
log . Error ( "Unable to drop old table %s. Error: %v" , tableName , err )
return err
}
if err := sess . Table ( tempTableName ) . DropIndexes ( bean ) ; err != nil {
log . Error ( "Unable to drop indexes on temporary table %s. Error: %v" , tempTableName , err )
return err
}
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` RENAME TO `%s`" , tempTableName , tableName ) ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
if err := sess . Table ( tableName ) . CreateIndexes ( bean ) ; err != nil {
log . Error ( "Unable to recreate indexes on table %s. Error: %v" , tableName , err )
return err
}
if err := sess . Table ( tableName ) . CreateUniques ( bean ) ; err != nil {
log . Error ( "Unable to recreate uniques on table %s. Error: %v" , tableName , err )
return err
}
case setting . Database . UseMySQL :
// MySQL will drop all the constraints on the old table
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP TABLE `%s`" , tableName ) ) ; err != nil {
log . Error ( "Unable to drop old table %s. Error: %v" , tableName , err )
return err
}
2021-08-19 00:11:36 -04:00
if err := sess . Table ( tempTableName ) . DropIndexes ( bean ) ; err != nil {
log . Error ( "Unable to drop indexes on temporary table %s. Error: %v" , tempTableName , err )
return err
}
2020-09-06 17:52:01 -04:00
// SQLite and MySQL will move all the constraints from the temporary table to the new table
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` RENAME TO `%s`" , tempTableName , tableName ) ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
2021-08-19 00:11:36 -04:00
if err := sess . Table ( tableName ) . CreateIndexes ( bean ) ; err != nil {
log . Error ( "Unable to recreate indexes on table %s. Error: %v" , tableName , err )
return err
}
if err := sess . Table ( tableName ) . CreateUniques ( bean ) ; err != nil {
log . Error ( "Unable to recreate uniques on table %s. Error: %v" , tableName , err )
return err
}
2020-09-06 17:52:01 -04:00
case setting . Database . UsePostgreSQL :
2021-03-18 02:06:40 -04:00
var originalSequences [ ] string
type sequenceData struct {
LastValue int ` xorm:"'last_value'" `
IsCalled bool ` xorm:"'is_called'" `
}
sequenceMap := map [ string ] sequenceData { }
schema := sess . Engine ( ) . Dialect ( ) . URI ( ) . Schema
sess . Engine ( ) . SetSchema ( "" )
if err := sess . Table ( "information_schema.sequences" ) . Cols ( "sequence_name" ) . Where ( "sequence_name LIKE ? || '_%' AND sequence_catalog = ?" , tableName , setting . Database . Name ) . Find ( & originalSequences ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
sess . Engine ( ) . SetSchema ( schema )
for _ , sequence := range originalSequences {
sequenceData := sequenceData { }
if _ , err := sess . Table ( sequence ) . Cols ( "last_value" , "is_called" ) . Get ( & sequenceData ) ; err != nil {
log . Error ( "Unable to get last_value and is_called from %s. Error: %v" , sequence , err )
return err
}
sequenceMap [ sequence ] = sequenceData
}
2020-09-06 17:52:01 -04:00
// CASCADE causes postgres to drop all the constraints on the old table
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP TABLE `%s` CASCADE" , tableName ) ) ; err != nil {
log . Error ( "Unable to drop old table %s. Error: %v" , tableName , err )
return err
}
// CASCADE causes postgres to move all the constraints from the temporary table to the new table
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` RENAME TO `%s`" , tempTableName , tableName ) ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
var indices [ ] string
sess . Engine ( ) . SetSchema ( "" )
if err := sess . Table ( "pg_indexes" ) . Cols ( "indexname" ) . Where ( "tablename = ? " , tableName ) . Find ( & indices ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
sess . Engine ( ) . SetSchema ( schema )
for _ , index := range indices {
newIndexName := strings . Replace ( index , "tmp_recreate__" , "" , 1 )
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER INDEX `%s` RENAME TO `%s`" , index , newIndexName ) ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , index , newIndexName , err )
return err
}
}
2021-03-18 02:06:40 -04:00
var sequences [ ] string
sess . Engine ( ) . SetSchema ( "" )
if err := sess . Table ( "information_schema.sequences" ) . Cols ( "sequence_name" ) . Where ( "sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?" , tableName , setting . Database . Name ) . Find ( & sequences ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
sess . Engine ( ) . SetSchema ( schema )
for _ , sequence := range sequences {
newSequenceName := strings . Replace ( sequence , "tmp_recreate__" , "" , 1 )
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER SEQUENCE `%s` RENAME TO `%s`" , sequence , newSequenceName ) ) ; err != nil {
log . Error ( "Unable to rename %s sequence to %s. Error: %v" , sequence , newSequenceName , err )
return err
}
val , ok := sequenceMap [ newSequenceName ]
if newSequenceName == tableName + "_id_seq" {
if ok && val . LastValue != 0 {
if _ , err := sess . Exec ( fmt . Sprintf ( "SELECT setval('%s', %d, %t)" , newSequenceName , val . LastValue , val . IsCalled ) ) ; err != nil {
log . Error ( "Unable to reset %s to %d. Error: %v" , newSequenceName , val , err )
return err
}
} else {
// We're going to try to guess this
if _ , err := sess . Exec ( fmt . Sprintf ( "SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)" , newSequenceName , tableName ) ) ; err != nil {
log . Error ( "Unable to reset %s. Error: %v" , newSequenceName , err )
return err
}
}
} else if ok {
if _ , err := sess . Exec ( fmt . Sprintf ( "SELECT setval('%s', %d, %t)" , newSequenceName , val . LastValue , val . IsCalled ) ) ; err != nil {
log . Error ( "Unable to reset %s to %d. Error: %v" , newSequenceName , val , err )
return err
}
}
}
2020-09-06 17:52:01 -04:00
case setting . Database . UseMSSQL :
// MSSQL will drop all the constraints on the old table
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP TABLE `%s`" , tableName ) ) ; err != nil {
log . Error ( "Unable to drop old table %s. Error: %v" , tableName , err )
return err
}
// MSSQL sp_rename will move all the constraints from the temporary table to the new table
if _ , err := sess . Exec ( fmt . Sprintf ( "sp_rename `%s`,`%s`" , tempTableName , tableName ) ) ; err != nil {
log . Error ( "Unable to rename %s to %s. Error: %v" , tempTableName , tableName , err )
return err
}
default :
log . Fatal ( "Unrecognized DB" )
}
return nil
}
// WARNING: YOU MUST COMMIT THE SESSION AT THE END
2018-05-09 12:29:04 -04:00
func dropTableColumns ( sess * xorm . Session , tableName string , columnNames ... string ) ( err error ) {
2018-03-07 01:44:12 -05:00
if tableName == "" || len ( columnNames ) == 0 {
return nil
}
2019-07-01 15:26:59 -04:00
// TODO: This will not work if there are foreign keys
2018-03-07 01:44:12 -05:00
switch {
2019-08-24 05:24:45 -04:00
case setting . Database . UseSQLite3 :
2019-07-01 15:26:59 -04:00
// First drop the indexes on the columns
res , errIndex := sess . Query ( fmt . Sprintf ( "PRAGMA index_list(`%s`)" , tableName ) )
if errIndex != nil {
return errIndex
}
for _ , row := range res {
indexName := row [ "name" ]
indexRes , err := sess . Query ( fmt . Sprintf ( "PRAGMA index_info(`%s`)" , indexName ) )
if err != nil {
return err
}
if len ( indexRes ) != 1 {
continue
}
indexColumn := string ( indexRes [ 0 ] [ "name" ] )
for _ , name := range columnNames {
if name == indexColumn {
_ , err := sess . Exec ( fmt . Sprintf ( "DROP INDEX `%s`" , indexName ) )
if err != nil {
return err
}
}
}
}
// Here we need to get the columns from the original table
sql := fmt . Sprintf ( "SELECT sql FROM sqlite_master WHERE tbl_name='%s' and type='table'" , tableName )
res , err := sess . Query ( sql )
if err != nil {
return err
}
tableSQL := string ( res [ 0 ] [ "sql" ] )
2019-08-05 17:49:49 -04:00
2021-11-14 03:11:49 -05:00
// Get the string offset for column definitions: `CREATE TABLE ( column-definitions... )`
columnDefinitionsIndex := strings . Index ( tableSQL , "(" )
if columnDefinitionsIndex < 0 {
return errors . New ( "couldn't find column definitions" )
}
2019-08-05 17:49:49 -04:00
// Separate out the column definitions
2021-11-14 03:11:49 -05:00
tableSQL = tableSQL [ columnDefinitionsIndex : ]
2019-08-05 17:49:49 -04:00
// Remove the required columnNames
2019-07-01 15:26:59 -04:00
for _ , name := range columnNames {
2019-08-05 17:49:49 -04:00
tableSQL = regexp . MustCompile ( regexp . QuoteMeta ( "`" + name + "`" ) + "[^`,)]*?[,)]" ) . ReplaceAllString ( tableSQL , "" )
}
// Ensure the query is ended properly
tableSQL = strings . TrimSpace ( tableSQL )
if tableSQL [ len ( tableSQL ) - 1 ] != ')' {
if tableSQL [ len ( tableSQL ) - 1 ] == ',' {
tableSQL = tableSQL [ : len ( tableSQL ) - 1 ]
}
tableSQL += ")"
2019-07-01 15:26:59 -04:00
}
2019-08-05 17:49:49 -04:00
// Find all the columns in the table
2019-07-01 15:26:59 -04:00
columns := regexp . MustCompile ( "`([^`]*)`" ) . FindAllString ( tableSQL , - 1 )
tableSQL = fmt . Sprintf ( "CREATE TABLE `new_%s_new` " , tableName ) + tableSQL
if _ , err := sess . Exec ( tableSQL ) ; err != nil {
return err
}
// Now restore the data
columnsSeparated := strings . Join ( columns , "," )
insertSQL := fmt . Sprintf ( "INSERT INTO `new_%s_new` (%s) SELECT %s FROM %s" , tableName , columnsSeparated , columnsSeparated , tableName )
if _ , err := sess . Exec ( insertSQL ) ; err != nil {
return err
}
// Now drop the old table
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP TABLE `%s`" , tableName ) ) ; err != nil {
return err
}
// Rename the table
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `new_%s_new` RENAME TO `%s`" , tableName , tableName ) ) ; err != nil {
return err
}
2019-08-24 05:24:45 -04:00
case setting . Database . UsePostgreSQL :
2019-07-01 15:26:59 -04:00
cols := ""
for _ , col := range columnNames {
if cols != "" {
cols += ", "
}
cols += "DROP COLUMN `" + col + "` CASCADE"
}
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` %s" , tableName , cols ) ) ; err != nil {
return fmt . Errorf ( "Drop table `%s` columns %v: %v" , tableName , columnNames , err )
}
2019-08-24 05:24:45 -04:00
case setting . Database . UseMySQL :
2019-07-01 15:26:59 -04:00
// Drop indexes on columns first
sql := fmt . Sprintf ( "SHOW INDEX FROM %s WHERE column_name IN ('%s')" , tableName , strings . Join ( columnNames , "','" ) )
res , err := sess . Query ( sql )
if err != nil {
return err
}
for _ , index := range res {
indexName := index [ "column_name" ]
2019-10-12 00:55:07 -04:00
if len ( indexName ) > 0 {
_ , err := sess . Exec ( fmt . Sprintf ( "DROP INDEX `%s` ON `%s`" , indexName , tableName ) )
if err != nil {
return err
}
2019-07-01 15:26:59 -04:00
}
}
// Now drop the columns
2018-03-07 01:44:12 -05:00
cols := ""
for _ , col := range columnNames {
if cols != "" {
cols += ", "
}
cols += "DROP COLUMN `" + col + "`"
}
2018-05-09 12:29:04 -04:00
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` %s" , tableName , cols ) ) ; err != nil {
2018-03-07 01:44:12 -05:00
return fmt . Errorf ( "Drop table `%s` columns %v: %v" , tableName , columnNames , err )
}
2019-08-24 05:24:45 -04:00
case setting . Database . UseMSSQL :
2018-03-07 01:44:12 -05:00
cols := ""
for _ , col := range columnNames {
if cols != "" {
cols += ", "
}
cols += "`" + strings . ToLower ( col ) + "`"
}
2021-08-08 06:34:24 -04:00
sql := fmt . Sprintf ( "SELECT Name FROM sys.default_constraints WHERE parent_object_id = OBJECT_ID('%[1]s') AND parent_column_id IN (SELECT column_id FROM sys.columns WHERE LOWER(name) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))" ,
2020-10-11 16:27:20 -04:00
tableName , strings . ReplaceAll ( cols , "`" , "'" ) )
2018-03-07 01:44:12 -05:00
constraints := make ( [ ] string , 0 )
if err := sess . SQL ( sql ) . Find ( & constraints ) ; err != nil {
return fmt . Errorf ( "Find constraints: %v" , err )
}
for _ , constraint := range constraints {
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` DROP CONSTRAINT `%s`" , tableName , constraint ) ) ; err != nil {
2021-03-24 14:27:22 -04:00
return fmt . Errorf ( "Drop table `%s` default constraint `%s`: %v" , tableName , constraint , err )
2018-03-07 01:44:12 -05:00
}
}
2021-08-08 06:34:24 -04:00
sql = fmt . Sprintf ( "SELECT DISTINCT Name FROM sys.indexes INNER JOIN sys.index_columns ON indexes.index_id = index_columns.index_id AND indexes.object_id = index_columns.object_id WHERE indexes.object_id = OBJECT_ID('%[1]s') AND index_columns.column_id IN (SELECT column_id FROM sys.columns WHERE LOWER(name) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))" ,
2021-03-24 14:27:22 -04:00
tableName , strings . ReplaceAll ( cols , "`" , "'" ) )
constraints = make ( [ ] string , 0 )
if err := sess . SQL ( sql ) . Find ( & constraints ) ; err != nil {
return fmt . Errorf ( "Find constraints: %v" , err )
}
for _ , constraint := range constraints {
2021-08-08 06:34:24 -04:00
if _ , err := sess . Exec ( fmt . Sprintf ( "DROP INDEX `%[2]s` ON `%[1]s`" , tableName , constraint ) ) ; err != nil {
2021-03-24 14:27:22 -04:00
return fmt . Errorf ( "Drop index `%[2]s` on `%[1]s`: %v" , tableName , constraint , err )
}
}
2018-03-07 01:44:12 -05:00
if _ , err := sess . Exec ( fmt . Sprintf ( "ALTER TABLE `%s` DROP COLUMN %s" , tableName , cols ) ) ; err != nil {
return fmt . Errorf ( "Drop table `%s` columns %v: %v" , tableName , columnNames , err )
}
default :
2019-04-02 03:48:31 -04:00
log . Fatal ( "Unrecognized DB" )
2018-03-07 01:44:12 -05:00
}
return nil
}
2021-01-06 10:11:23 -05:00
2022-06-13 03:34:46 -04:00
// modifyColumn will modify column's type or other property. SQLITE is not supported
2021-01-06 10:11:23 -05:00
func modifyColumn ( x * xorm . Engine , tableName string , col * schemas . Column ) error {
var indexes map [ string ] * schemas . Index
var err error
// MSSQL have to remove index at first, otherwise alter column will fail
// ref. https://sqlzealots.com/2018/05/09/error-message-the-index-is-dependent-on-column-alter-table-alter-column-failed-because-one-or-more-objects-access-this-column/
if x . Dialect ( ) . URI ( ) . DBType == schemas . MSSQL {
indexes , err = x . Dialect ( ) . GetIndexes ( x . DB ( ) , context . Background ( ) , tableName )
if err != nil {
return err
}
for _ , index := range indexes {
_ , err = x . Exec ( x . Dialect ( ) . DropIndexSQL ( tableName , index ) )
if err != nil {
return err
}
}
}
defer func ( ) {
for _ , index := range indexes {
_ , err = x . Exec ( x . Dialect ( ) . CreateIndexSQL ( tableName , index ) )
if err != nil {
log . Error ( "Create index %s on table %s failed: %v" , index . Name , tableName , err )
}
}
} ( )
alterSQL := x . Dialect ( ) . ModifyColumnSQL ( tableName , col )
if _ , err := x . Exec ( alterSQL ) ; err != nil {
return err
}
return nil
}