mirror of
https://github.com/go-gitea/gitea.git
synced 2024-08-27 22:04:28 -04:00
## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
334 lines
9.8 KiB
Go
334 lines
9.8 KiB
Go
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package graceful
|
|
|
|
import (
|
|
"context"
|
|
"runtime/pprof"
|
|
"sync"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/process"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
)
|
|
|
|
type state uint8
|
|
|
|
const (
|
|
stateInit state = iota
|
|
stateRunning
|
|
stateShuttingDown
|
|
stateTerminate
|
|
)
|
|
|
|
// There are some places that could inherit sockets:
|
|
//
|
|
// * HTTP or HTTPS main listener
|
|
// * HTTP or HTTPS install listener
|
|
// * HTTP redirection fallback
|
|
// * Builtin SSH listener
|
|
//
|
|
// If you add a new place you must increment this number
|
|
// and add a function to call manager.InformCleanup if it's not going to be used
|
|
const numberOfServersToCreate = 4
|
|
|
|
// Manager represents the graceful server manager interface
|
|
var manager *Manager
|
|
|
|
var initOnce = sync.Once{}
|
|
|
|
// GetManager returns the Manager
|
|
func GetManager() *Manager {
|
|
InitManager(context.Background())
|
|
return manager
|
|
}
|
|
|
|
// InitManager creates the graceful manager in the provided context
|
|
func InitManager(ctx context.Context) {
|
|
initOnce.Do(func() {
|
|
manager = newGracefulManager(ctx)
|
|
|
|
// Set the process default context to the HammerContext
|
|
process.DefaultContext = manager.HammerContext()
|
|
})
|
|
}
|
|
|
|
// WithCallback is a runnable to call when the caller has finished
|
|
type WithCallback func(callback func())
|
|
|
|
// RunnableWithShutdownFns is a runnable with functions to run at shutdown and terminate
|
|
// After the callback to atShutdown is called and is complete, the main function must return.
|
|
// Similarly the callback function provided to atTerminate must return once termination is complete.
|
|
// Please note that use of the atShutdown and atTerminate callbacks will create go-routines that will wait till their respective signals
|
|
// - users must therefore be careful to only call these as necessary.
|
|
type RunnableWithShutdownFns func(atShutdown, atTerminate func(func()))
|
|
|
|
// RunWithShutdownFns takes a function that has both atShutdown and atTerminate callbacks
|
|
// After the callback to atShutdown is called and is complete, the main function must return.
|
|
// Similarly the callback function provided to atTerminate must return once termination is complete.
|
|
// Please note that use of the atShutdown and atTerminate callbacks will create go-routines that will wait till their respective signals
|
|
// - users must therefore be careful to only call these as necessary.
|
|
func (g *Manager) RunWithShutdownFns(run RunnableWithShutdownFns) {
|
|
g.runningServerWaitGroup.Add(1)
|
|
defer g.runningServerWaitGroup.Done()
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunWithShutdownFns: %v\nStacktrace: %s", err, log.Stack(2))
|
|
g.doShutdown()
|
|
}
|
|
}()
|
|
run(func(atShutdown func()) {
|
|
g.lock.Lock()
|
|
defer g.lock.Unlock()
|
|
g.toRunAtShutdown = append(g.toRunAtShutdown,
|
|
func() {
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunWithShutdownFns: %v\nStacktrace: %s", err, log.Stack(2))
|
|
g.doShutdown()
|
|
}
|
|
}()
|
|
atShutdown()
|
|
})
|
|
}, func(atTerminate func()) {
|
|
g.RunAtTerminate(atTerminate)
|
|
})
|
|
}
|
|
|
|
// RunWithShutdownContext takes a function that has a context to watch for shutdown.
|
|
// After the provided context is Done(), the main function must return once shutdown is complete.
|
|
// (Optionally the HammerContext may be obtained and waited for however, this should be avoided if possible.)
|
|
func (g *Manager) RunWithShutdownContext(run func(context.Context)) {
|
|
g.runningServerWaitGroup.Add(1)
|
|
defer g.runningServerWaitGroup.Done()
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunWithShutdownContext: %v\nStacktrace: %s", err, log.Stack(2))
|
|
g.doShutdown()
|
|
}
|
|
}()
|
|
ctx := g.ShutdownContext()
|
|
pprof.SetGoroutineLabels(ctx) // We don't have a label to restore back to but I think this is fine
|
|
run(ctx)
|
|
}
|
|
|
|
// RunAtTerminate adds to the terminate wait group and creates a go-routine to run the provided function at termination
|
|
func (g *Manager) RunAtTerminate(terminate func()) {
|
|
g.terminateWaitGroup.Add(1)
|
|
g.lock.Lock()
|
|
defer g.lock.Unlock()
|
|
g.toRunAtTerminate = append(g.toRunAtTerminate,
|
|
func() {
|
|
defer g.terminateWaitGroup.Done()
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunAtTerminate: %v\nStacktrace: %s", err, log.Stack(2))
|
|
}
|
|
}()
|
|
terminate()
|
|
})
|
|
}
|
|
|
|
// RunAtShutdown creates a go-routine to run the provided function at shutdown
|
|
func (g *Manager) RunAtShutdown(ctx context.Context, shutdown func()) {
|
|
g.lock.Lock()
|
|
defer g.lock.Unlock()
|
|
g.toRunAtShutdown = append(g.toRunAtShutdown,
|
|
func() {
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunAtShutdown: %v\nStacktrace: %s", err, log.Stack(2))
|
|
}
|
|
}()
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
shutdown()
|
|
}
|
|
})
|
|
}
|
|
|
|
// RunAtHammer creates a go-routine to run the provided function at shutdown
|
|
func (g *Manager) RunAtHammer(hammer func()) {
|
|
g.lock.Lock()
|
|
defer g.lock.Unlock()
|
|
g.toRunAtHammer = append(g.toRunAtHammer,
|
|
func() {
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Critical("PANIC during RunAtHammer: %v\nStacktrace: %s", err, log.Stack(2))
|
|
}
|
|
}()
|
|
hammer()
|
|
})
|
|
}
|
|
|
|
func (g *Manager) doShutdown() {
|
|
if !g.setStateTransition(stateRunning, stateShuttingDown) {
|
|
g.DoImmediateHammer()
|
|
return
|
|
}
|
|
g.lock.Lock()
|
|
g.shutdownCtxCancel()
|
|
atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "post-shutdown"))
|
|
pprof.SetGoroutineLabels(atShutdownCtx)
|
|
for _, fn := range g.toRunAtShutdown {
|
|
go fn()
|
|
}
|
|
g.lock.Unlock()
|
|
|
|
if setting.GracefulHammerTime >= 0 {
|
|
go g.doHammerTime(setting.GracefulHammerTime)
|
|
}
|
|
go func() {
|
|
g.WaitForServers()
|
|
// Mop up any remaining unclosed events.
|
|
g.doHammerTime(0)
|
|
<-time.After(1 * time.Second)
|
|
g.doTerminate()
|
|
g.WaitForTerminate()
|
|
g.lock.Lock()
|
|
g.managerCtxCancel()
|
|
g.lock.Unlock()
|
|
}()
|
|
}
|
|
|
|
func (g *Manager) doHammerTime(d time.Duration) {
|
|
time.Sleep(d)
|
|
g.lock.Lock()
|
|
select {
|
|
case <-g.hammerCtx.Done():
|
|
default:
|
|
log.Warn("Setting Hammer condition")
|
|
g.hammerCtxCancel()
|
|
atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "post-hammer"))
|
|
pprof.SetGoroutineLabels(atHammerCtx)
|
|
for _, fn := range g.toRunAtHammer {
|
|
go fn()
|
|
}
|
|
}
|
|
g.lock.Unlock()
|
|
}
|
|
|
|
func (g *Manager) doTerminate() {
|
|
if !g.setStateTransition(stateShuttingDown, stateTerminate) {
|
|
return
|
|
}
|
|
g.lock.Lock()
|
|
select {
|
|
case <-g.terminateCtx.Done():
|
|
default:
|
|
log.Warn("Terminating")
|
|
g.terminateCtxCancel()
|
|
atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "post-terminate"))
|
|
pprof.SetGoroutineLabels(atTerminateCtx)
|
|
|
|
for _, fn := range g.toRunAtTerminate {
|
|
go fn()
|
|
}
|
|
}
|
|
g.lock.Unlock()
|
|
}
|
|
|
|
// IsChild returns if the current process is a child of previous Gitea process
|
|
func (g *Manager) IsChild() bool {
|
|
return g.isChild
|
|
}
|
|
|
|
// IsShutdown returns a channel which will be closed at shutdown.
|
|
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
|
|
func (g *Manager) IsShutdown() <-chan struct{} {
|
|
return g.shutdownCtx.Done()
|
|
}
|
|
|
|
// IsHammer returns a channel which will be closed at hammer
|
|
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
|
|
// Servers running within the running server wait group should respond to IsHammer
|
|
// if not shutdown already
|
|
func (g *Manager) IsHammer() <-chan struct{} {
|
|
return g.hammerCtx.Done()
|
|
}
|
|
|
|
// IsTerminate returns a channel which will be closed at terminate
|
|
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
|
|
// IsTerminate will only close once all running servers have stopped
|
|
func (g *Manager) IsTerminate() <-chan struct{} {
|
|
return g.terminateCtx.Done()
|
|
}
|
|
|
|
// ServerDone declares a running server done and subtracts one from the
|
|
// running server wait group. Users probably do not want to call this
|
|
// and should use one of the RunWithShutdown* functions
|
|
func (g *Manager) ServerDone() {
|
|
g.runningServerWaitGroup.Done()
|
|
}
|
|
|
|
// WaitForServers waits for all running servers to finish. Users should probably
|
|
// instead use AtTerminate or IsTerminate
|
|
func (g *Manager) WaitForServers() {
|
|
g.runningServerWaitGroup.Wait()
|
|
}
|
|
|
|
// WaitForTerminate waits for all terminating actions to finish.
|
|
// Only the main go-routine should use this
|
|
func (g *Manager) WaitForTerminate() {
|
|
g.terminateWaitGroup.Wait()
|
|
}
|
|
|
|
func (g *Manager) getState() state {
|
|
g.lock.RLock()
|
|
defer g.lock.RUnlock()
|
|
return g.state
|
|
}
|
|
|
|
func (g *Manager) setStateTransition(old, new state) bool {
|
|
if old != g.getState() {
|
|
return false
|
|
}
|
|
g.lock.Lock()
|
|
if g.state != old {
|
|
g.lock.Unlock()
|
|
return false
|
|
}
|
|
g.state = new
|
|
g.lock.Unlock()
|
|
return true
|
|
}
|
|
|
|
func (g *Manager) setState(st state) {
|
|
g.lock.Lock()
|
|
defer g.lock.Unlock()
|
|
|
|
g.state = st
|
|
}
|
|
|
|
// InformCleanup tells the cleanup wait group that we have either taken a listener or will not be taking a listener.
|
|
// At the moment the total number of servers (numberOfServersToCreate) are pre-defined as a const before global init,
|
|
// so this function MUST be called if a server is not used.
|
|
func (g *Manager) InformCleanup() {
|
|
g.createServerWaitGroup.Done()
|
|
}
|
|
|
|
// Done allows the manager to be viewed as a context.Context, it returns a channel that is closed when the server is finished terminating
|
|
func (g *Manager) Done() <-chan struct{} {
|
|
return g.managerCtx.Done()
|
|
}
|
|
|
|
// Err allows the manager to be viewed as a context.Context done at Terminate
|
|
func (g *Manager) Err() error {
|
|
return g.managerCtx.Err()
|
|
}
|
|
|
|
// Value allows the manager to be viewed as a context.Context done at Terminate
|
|
func (g *Manager) Value(key interface{}) interface{} {
|
|
return g.managerCtx.Value(key)
|
|
}
|
|
|
|
// Deadline returns nil as there is no fixed Deadline for the manager, it allows the manager to be viewed as a context.Context
|
|
func (g *Manager) Deadline() (deadline time.Time, ok bool) {
|
|
return g.managerCtx.Deadline()
|
|
}
|