mirror of
https://github.com/go-gitea/gitea.git
synced 2024-09-06 23:44:20 -04:00
Add proxy settings and support for migration and webhook (#16704)
* Add proxy settings and support for migration and webhook * Fix default value * Add newline for example ini * Add lfs proxy support * Fix lint * Follow @zeripath's review * Fix git clone * Fix test * missgin http requests for proxy * use empty Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
parent
422c30d315
commit
f9acad82ca
@ -2127,3 +2127,11 @@ PATH =
|
|||||||
;;
|
;;
|
||||||
;; Minio enabled ssl only available when STORAGE_TYPE is `minio`
|
;; Minio enabled ssl only available when STORAGE_TYPE is `minio`
|
||||||
;MINIO_USE_SSL = false
|
;MINIO_USE_SSL = false
|
||||||
|
|
||||||
|
;[proxy]
|
||||||
|
;; Enable the proxy, all requests to external via HTTP will be affected
|
||||||
|
;PROXY_ENABLED = false
|
||||||
|
;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy
|
||||||
|
;PROXY_URL =
|
||||||
|
;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
|
||||||
|
;PROXY_HOSTS =
|
||||||
|
@ -549,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
|
|||||||
- `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
|
- `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
|
||||||
- `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
|
- `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
|
||||||
- `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
|
- `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
|
||||||
- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
|
- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting.
|
||||||
- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
|
- `PROXY_HOSTS`: **\<empty\>`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting.
|
||||||
|
|
||||||
## Mailer (`mailer`)
|
## Mailer (`mailer`)
|
||||||
|
|
||||||
@ -950,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
|
|||||||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
|
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
|
||||||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
|
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
|
||||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
|
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
|
||||||
|
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
|
||||||
|
|
||||||
## Mirror (`mirror`)
|
## Mirror (`mirror`)
|
||||||
|
|
||||||
@ -1023,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
|
|||||||
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
|
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
|
||||||
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
|
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
|
||||||
|
|
||||||
|
## Proxy (`proxy`)
|
||||||
|
|
||||||
|
- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy
|
||||||
|
- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
|
||||||
|
- `PROXY_HOSTS`: **\<empty\>**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
|
||||||
|
|
||||||
|
i.e.
|
||||||
|
```ini
|
||||||
|
PROXY_ENABLED = true
|
||||||
|
PROXY_URL = socks://127.0.0.1:1080
|
||||||
|
PROXY_HOSTS = *.github.com
|
||||||
|
```
|
||||||
|
|
||||||
## Other (`other`)
|
## Other (`other`)
|
||||||
|
|
||||||
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
|
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
|
||||||
|
@ -332,6 +332,7 @@ IS_INPUT_FILE = false
|
|||||||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
|
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
|
||||||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
|
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
|
||||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
|
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
|
||||||
|
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
|
||||||
|
|
||||||
## LFS (`lfs`)
|
## LFS (`lfs`)
|
||||||
|
|
||||||
@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空,则此配
|
|||||||
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。
|
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。
|
||||||
- `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。
|
- `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。
|
||||||
|
|
||||||
|
## Proxy (`proxy`)
|
||||||
|
|
||||||
|
- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否,则不使用代理,环境变量中的代理也不使用
|
||||||
|
- `PROXY_URL`: **\<empty\>**: 代理服务器地址,支持 http://, https//, socks://,为空则不启用代理而使用环境变量中的 http_proxy/https_proxy
|
||||||
|
- `PROXY_HOSTS`: **\<empty\>**: 逗号分隔的多个需要代理的网址,支持 * 号匹配符号, ** 表示匹配所有网站
|
||||||
|
|
||||||
|
i.e.
|
||||||
|
```ini
|
||||||
|
PROXY_ENABLED = true
|
||||||
|
PROXY_URL = socks://127.0.0.1:1080
|
||||||
|
PROXY_HOSTS = *.github.com
|
||||||
|
```
|
||||||
|
|
||||||
## Other (`other`)
|
## Other (`other`)
|
||||||
|
|
||||||
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
|
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
|
||||||
|
@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
|
|||||||
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
|
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
|
||||||
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
|
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
|
||||||
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
|
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
|
||||||
if timeout == -1 {
|
return c.RunWithContext(&RunContext{
|
||||||
timeout = defaultCommandExecutionTimeout
|
Env: env,
|
||||||
|
Timeout: timeout,
|
||||||
|
Dir: dir,
|
||||||
|
Stdout: stdout,
|
||||||
|
Stderr: stderr,
|
||||||
|
Stdin: stdin,
|
||||||
|
PipelineFunc: fn,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunContext represents parameters to run the command
|
||||||
|
type RunContext struct {
|
||||||
|
Env []string
|
||||||
|
Timeout time.Duration
|
||||||
|
Dir string
|
||||||
|
Stdout, Stderr io.Writer
|
||||||
|
Stdin io.Reader
|
||||||
|
PipelineFunc func(context.Context, context.CancelFunc) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunWithContext run the command with context
|
||||||
|
func (c *Command) RunWithContext(rc *RunContext) error {
|
||||||
|
if rc.Timeout == -1 {
|
||||||
|
rc.Timeout = defaultCommandExecutionTimeout
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(dir) == 0 {
|
if len(rc.Dir) == 0 {
|
||||||
log.Debug("%s", c)
|
log.Debug("%s", c)
|
||||||
} else {
|
} else {
|
||||||
log.Debug("%s: %v", dir, c)
|
log.Debug("%s: %v", rc.Dir, c)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(c.parentContext, timeout)
|
ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, c.name, c.args...)
|
cmd := exec.CommandContext(ctx, c.name, c.args...)
|
||||||
if env == nil {
|
if rc.Env == nil {
|
||||||
cmd.Env = os.Environ()
|
cmd.Env = os.Environ()
|
||||||
} else {
|
} else {
|
||||||
cmd.Env = env
|
cmd.Env = rc.Env
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.Env = append(
|
cmd.Env = append(
|
||||||
@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
|
|||||||
if goVersionLessThan115 {
|
if goVersionLessThan115 {
|
||||||
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
|
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
|
||||||
}
|
}
|
||||||
cmd.Dir = dir
|
cmd.Dir = rc.Dir
|
||||||
cmd.Stdout = stdout
|
cmd.Stdout = rc.Stdout
|
||||||
cmd.Stderr = stderr
|
cmd.Stderr = rc.Stderr
|
||||||
cmd.Stdin = stdin
|
cmd.Stdin = rc.Stdin
|
||||||
if err := cmd.Start(); err != nil {
|
if err := cmd.Start(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
desc := c.desc
|
desc := c.desc
|
||||||
if desc == "" {
|
if desc == "" {
|
||||||
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
|
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
|
||||||
}
|
}
|
||||||
pid := process.GetManager().Add(desc, cancel)
|
pid := process.GetManager().Add(desc, cancel)
|
||||||
defer process.GetManager().Remove(pid)
|
defer process.GetManager().Remove(pid)
|
||||||
|
|
||||||
if fn != nil {
|
if rc.PipelineFunc != nil {
|
||||||
err := fn(ctx, cancel)
|
err := rc.PipelineFunc(ctx, cancel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cancel()
|
cancel()
|
||||||
_ = cmd.Wait()
|
_ = cmd.Wait()
|
||||||
|
@ -9,11 +9,15 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GPGSettings represents the default GPG settings for this repository
|
// GPGSettings represents the default GPG settings for this repository
|
||||||
@ -99,12 +103,12 @@ type CloneRepoOptions struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Clone clones original repository to target path.
|
// Clone clones original repository to target path.
|
||||||
func Clone(from, to string, opts CloneRepoOptions) (err error) {
|
func Clone(from, to string, opts CloneRepoOptions) error {
|
||||||
return CloneWithContext(DefaultContext, from, to, opts)
|
return CloneWithContext(DefaultContext, from, to, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloneWithContext clones original repository to target path.
|
// CloneWithContext clones original repository to target path.
|
||||||
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
|
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
|
||||||
cargs := make([]string, len(GlobalCommandArgs))
|
cargs := make([]string, len(GlobalCommandArgs))
|
||||||
copy(cargs, GlobalCommandArgs)
|
copy(cargs, GlobalCommandArgs)
|
||||||
return CloneWithArgs(ctx, from, to, cargs, opts)
|
return CloneWithArgs(ctx, from, to, cargs, opts)
|
||||||
@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
|
|||||||
opts.Timeout = -1
|
opts.Timeout = -1
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = cmd.RunTimeout(opts.Timeout)
|
var envs = os.Environ()
|
||||||
return err
|
u, err := url.Parse(from)
|
||||||
|
if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
|
||||||
|
if proxy.Match(u.Host) {
|
||||||
|
envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var stderr = new(bytes.Buffer)
|
||||||
|
if err = cmd.RunWithContext(&RunContext{
|
||||||
|
Timeout: opts.Timeout,
|
||||||
|
Env: envs,
|
||||||
|
Stdout: io.Discard,
|
||||||
|
Stderr: stderr,
|
||||||
|
}); err != nil {
|
||||||
|
return ConcatenateError(err, stderr.String())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PullRemoteOptions options when pull from remote
|
// PullRemoteOptions options when pull from remote
|
||||||
|
@ -24,9 +24,9 @@ type Client interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewClient creates a LFS client
|
// NewClient creates a LFS client
|
||||||
func NewClient(endpoint *url.URL) Client {
|
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
|
||||||
if endpoint.Scheme == "file" {
|
if endpoint.Scheme == "file" {
|
||||||
return newFilesystemClient(endpoint)
|
return newFilesystemClient(endpoint)
|
||||||
}
|
}
|
||||||
return newHTTPClient(endpoint)
|
return newHTTPClient(endpoint, skipTLSVerify)
|
||||||
}
|
}
|
||||||
|
@ -13,10 +13,10 @@ import (
|
|||||||
|
|
||||||
func TestNewClient(t *testing.T) {
|
func TestNewClient(t *testing.T) {
|
||||||
u, _ := url.Parse("file:///test")
|
u, _ := url.Parse("file:///test")
|
||||||
c := NewClient(u)
|
c := NewClient(u, true)
|
||||||
assert.IsType(t, &FilesystemClient{}, c)
|
assert.IsType(t, &FilesystemClient{}, c)
|
||||||
|
|
||||||
u, _ = url.Parse("https://test.com/lfs")
|
u, _ = url.Parse("https://test.com/lfs")
|
||||||
c = NewClient(u)
|
c = NewClient(u, true)
|
||||||
assert.IsType(t, &HTTPClient{}, c)
|
assert.IsType(t, &HTTPClient{}, c)
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,7 @@ package lfs
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -15,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"code.gitea.io/gitea/modules/json"
|
"code.gitea.io/gitea/modules/json"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
const batchSize = 20
|
const batchSize = 20
|
||||||
@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
|
|||||||
return batchSize
|
return batchSize
|
||||||
}
|
}
|
||||||
|
|
||||||
func newHTTPClient(endpoint *url.URL) *HTTPClient {
|
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
|
||||||
hc := &http.Client{}
|
hc := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
client := &HTTPClient{
|
client := &HTTPClient{
|
||||||
client: hc,
|
client: hc,
|
||||||
|
@ -6,6 +6,7 @@ package migrations
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@ -17,6 +18,8 @@ import (
|
|||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/migrations/base"
|
"code.gitea.io/gitea/modules/migrations/base"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
gitea_sdk "code.gitea.io/sdk/gitea"
|
gitea_sdk "code.gitea.io/sdk/gitea"
|
||||||
@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
|
|||||||
gitea_sdk.SetToken(token),
|
gitea_sdk.SetToken(token),
|
||||||
gitea_sdk.SetBasicAuth(username, password),
|
gitea_sdk.SetBasicAuth(username, password),
|
||||||
gitea_sdk.SetContext(ctx),
|
gitea_sdk.SetContext(ctx),
|
||||||
|
gitea_sdk.SetHTTPClient(&http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
|
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
|
||||||
@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
|
|||||||
Created: rel.CreatedAt,
|
Created: rel.CreatedAt,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
httpClient := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
for _, asset := range rel.Attachments {
|
for _, asset := range rel.Attachments {
|
||||||
size := int(asset.Size)
|
size := int(asset.Size)
|
||||||
dlCount := int(asset.DownloadCount)
|
dlCount := int(asset.DownloadCount)
|
||||||
@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// FIXME: for a private download?
|
// FIXME: for a private download?
|
||||||
resp, err := http.Get(asset.DownloadURL)
|
req, err := http.NewRequest("GET", asset.DownloadURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,7 @@ package migrations
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@ -17,6 +18,8 @@ import (
|
|||||||
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/migrations/base"
|
"code.gitea.io/gitea/modules/migrations/base"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
||||||
@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
|
|||||||
Transport: &http.Transport{
|
Transport: &http.Transport{
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||||
req.SetBasicAuth(userName, password)
|
req.SetBasicAuth(userName, password)
|
||||||
return nil, nil
|
return proxy.Proxy()(req)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
|||||||
r.Published = rel.PublishedAt.Time
|
r.Published = rel.PublishedAt.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
httpClient := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
for _, asset := range rel.Assets {
|
for _, asset := range rel.Assets {
|
||||||
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
|
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
|
||||||
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
||||||
@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
resp, err := http.DefaultClient.Do(req)
|
resp, err := httpClient.Do(req)
|
||||||
err1 := g.RefreshRate()
|
err1 := g.RefreshRate()
|
||||||
if err1 != nil {
|
if err1 != nil {
|
||||||
log.Error("g.client.RateLimits: %s", err1)
|
log.Error("g.client.RateLimits: %s", err1)
|
||||||
|
@ -6,6 +6,7 @@ package migrations
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@ -17,6 +18,8 @@ import (
|
|||||||
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/migrations/base"
|
"code.gitea.io/gitea/modules/migrations/base"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
"github.com/xanzy/go-gitlab"
|
"github.com/xanzy/go-gitlab"
|
||||||
@ -77,7 +80,12 @@ type GitlabDownloader struct {
|
|||||||
// Use either a username/password, personal token entered into the username field, or anonymous/public access
|
// Use either a username/password, personal token entered into the username field, or anonymous/public access
|
||||||
// Note: Public access only allows very basic access
|
// Note: Public access only allows very basic access
|
||||||
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
|
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
|
||||||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
|
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}))
|
||||||
// Only use basic auth if token is blank and password is NOT
|
// Only use basic auth if token is blank and password is NOT
|
||||||
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
|
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
|
||||||
if token == "" && password != "" {
|
if token == "" && password != "" {
|
||||||
@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
|
|||||||
PublisherName: rel.Author.Username,
|
PublisherName: rel.Author.Username,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
httpClient := &http.Client{
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
|
Proxy: proxy.Proxy(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
for k, asset := range rel.Assets.Links {
|
for k, asset := range rel.Assets.Links {
|
||||||
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
||||||
ID: int64(asset.ID),
|
ID: int64(asset.ID),
|
||||||
@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req = req.WithContext(g.ctx)
|
req = req.WithContext(g.ctx)
|
||||||
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,7 @@ package migrations
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
@ -14,6 +15,8 @@ import (
|
|||||||
|
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/migrations/base"
|
"code.gitea.io/gitea/modules/migrations/base"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
|
||||||
"github.com/gogs/go-gogs-client"
|
"github.com/gogs/go-gogs-client"
|
||||||
@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
|
|||||||
downloader.userName = token
|
downloader.userName = token
|
||||||
} else {
|
} else {
|
||||||
downloader.transport = &http.Transport{
|
downloader.transport = &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
|
||||||
Proxy: func(req *http.Request) (*url.URL, error) {
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
||||||
req.SetBasicAuth(userName, password)
|
req.SetBasicAuth(userName, password)
|
||||||
return nil, nil
|
return proxy.Proxy()(req)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
83
modules/proxy/proxy.go
Normal file
83
modules/proxy/proxy.go
Normal file
@ -0,0 +1,83 @@
|
|||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package proxy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
|
||||||
|
"github.com/gobwas/glob"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
once sync.Once
|
||||||
|
hostMatchers []glob.Glob
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetProxyURL returns proxy url
|
||||||
|
func GetProxyURL() string {
|
||||||
|
if !setting.Proxy.Enabled {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if setting.Proxy.ProxyURL == "" {
|
||||||
|
if os.Getenv("http_proxy") != "" {
|
||||||
|
return os.Getenv("http_proxy")
|
||||||
|
}
|
||||||
|
return os.Getenv("https_proxy")
|
||||||
|
}
|
||||||
|
return setting.Proxy.ProxyURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match return true if url needs to be proxied
|
||||||
|
func Match(u string) bool {
|
||||||
|
if !setting.Proxy.Enabled {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// enforce do once
|
||||||
|
Proxy()
|
||||||
|
|
||||||
|
for _, v := range hostMatchers {
|
||||||
|
if v.Match(u) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proxy returns the system proxy
|
||||||
|
func Proxy() func(req *http.Request) (*url.URL, error) {
|
||||||
|
if !setting.Proxy.Enabled {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if setting.Proxy.ProxyURL == "" {
|
||||||
|
return http.ProxyFromEnvironment
|
||||||
|
}
|
||||||
|
|
||||||
|
once.Do(func() {
|
||||||
|
for _, h := range setting.Proxy.ProxyHosts {
|
||||||
|
if g, err := glob.Compile(h); err == nil {
|
||||||
|
hostMatchers = append(hostMatchers, g)
|
||||||
|
} else {
|
||||||
|
log.Error("glob.Compile %s failed: %v", h, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return func(req *http.Request) (*url.URL, error) {
|
||||||
|
for _, v := range hostMatchers {
|
||||||
|
if v.Match(req.URL.Host) {
|
||||||
|
return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return http.ProxyFromEnvironment(req)
|
||||||
|
}
|
||||||
|
}
|
@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
|
|||||||
|
|
||||||
if opts.LFS {
|
if opts.LFS {
|
||||||
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
|
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
|
||||||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil {
|
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
|
||||||
log.Error("Failed to store missing LFS objects for repository: %v", err)
|
log.Error("Failed to store missing LFS objects for repository: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
|
|||||||
}
|
}
|
||||||
|
|
||||||
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
|
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
|
||||||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error {
|
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
|
||||||
client := lfs.NewClient(endpoint)
|
client := lfs.NewClient(endpoint, skipTLSVerify)
|
||||||
contentStore := lfs.NewContentStore()
|
contentStore := lfs.NewContentStore()
|
||||||
|
|
||||||
pointerChan := make(chan lfs.PointerBlob)
|
pointerChan := make(chan lfs.PointerBlob)
|
||||||
|
@ -16,6 +16,7 @@ var (
|
|||||||
AllowedDomains []string
|
AllowedDomains []string
|
||||||
BlockedDomains []string
|
BlockedDomains []string
|
||||||
AllowLocalNetworks bool
|
AllowLocalNetworks bool
|
||||||
|
SkipTLSVerify bool
|
||||||
}{
|
}{
|
||||||
MaxAttempts: 3,
|
MaxAttempts: 3,
|
||||||
RetryBackoff: 3,
|
RetryBackoff: 3,
|
||||||
@ -37,4 +38,5 @@ func newMigrationsService() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
|
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
|
||||||
|
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
|
||||||
}
|
}
|
||||||
|
40
modules/setting/proxy.go
Normal file
40
modules/setting/proxy.go
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package setting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Proxy settings
|
||||||
|
Proxy = struct {
|
||||||
|
Enabled bool
|
||||||
|
ProxyURL string
|
||||||
|
ProxyURLFixed *url.URL
|
||||||
|
ProxyHosts []string
|
||||||
|
}{
|
||||||
|
Enabled: false,
|
||||||
|
ProxyURL: "",
|
||||||
|
ProxyHosts: []string{},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func newProxyService() {
|
||||||
|
sec := Cfg.Section("proxy")
|
||||||
|
Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
|
||||||
|
Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
|
||||||
|
if Proxy.ProxyURL != "" {
|
||||||
|
var err error
|
||||||
|
Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Global PROXY_URL is not valid")
|
||||||
|
Proxy.ProxyURL = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
|
||||||
|
}
|
@ -1195,6 +1195,7 @@ func NewServices() {
|
|||||||
newMailService()
|
newMailService()
|
||||||
newRegisterMailService()
|
newRegisterMailService()
|
||||||
newNotifyMailService()
|
newNotifyMailService()
|
||||||
|
newProxyService()
|
||||||
newWebhookService()
|
newWebhookService()
|
||||||
newMigrationsService()
|
newMigrationsService()
|
||||||
newIndexerService()
|
newIndexerService()
|
||||||
|
@ -196,7 +196,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool)
|
|||||||
if m.LFS && setting.LFS.StartServer {
|
if m.LFS && setting.LFS.StartServer {
|
||||||
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
|
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
|
||||||
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
|
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
|
||||||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil {
|
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil {
|
||||||
log.Error("Failed to synchronize LFS objects for repository: %v", err)
|
log.Error("Failed to synchronize LFS objects for repository: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -134,7 +134,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
|
|||||||
defer gitRepo.Close()
|
defer gitRepo.Close()
|
||||||
|
|
||||||
ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
|
ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
|
||||||
if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil {
|
if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil {
|
||||||
return util.NewURLSanitizedError(err, remoteAddr, true)
|
return util.NewURLSanitizedError(err, remoteAddr, true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -176,8 +176,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error {
|
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
|
||||||
client := lfs.NewClient(endpoint)
|
client := lfs.NewClient(endpoint, skipTLSVerify)
|
||||||
contentStore := lfs.NewContentStore()
|
contentStore := lfs.NewContentStore()
|
||||||
|
|
||||||
pointerChan := make(chan lfs.PointerBlob)
|
pointerChan := make(chan lfs.PointerBlob)
|
||||||
|
@ -25,6 +25,7 @@ import (
|
|||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/graceful"
|
"code.gitea.io/gitea/modules/graceful"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/proxy"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"github.com/gobwas/glob"
|
"github.com/gobwas/glob"
|
||||||
)
|
)
|
||||||
@ -260,7 +261,7 @@ var (
|
|||||||
|
|
||||||
func webhookProxy() func(req *http.Request) (*url.URL, error) {
|
func webhookProxy() func(req *http.Request) (*url.URL, error) {
|
||||||
if setting.Webhook.ProxyURL == "" {
|
if setting.Webhook.ProxyURL == "" {
|
||||||
return http.ProxyFromEnvironment
|
return proxy.Proxy()
|
||||||
}
|
}
|
||||||
|
|
||||||
once.Do(func() {
|
once.Do(func() {
|
||||||
|
Loading…
Reference in New Issue
Block a user