Serve pre-defined files in "public", add "security.txt", add CORS header for ".well-known" (#25974)

Replace #25892

Close  #21942
Close  #25464

Major changes:

1. Serve "robots.txt" and ".well-known/security.txt" in the "public"
custom path
* All files in "public/.well-known" can be served, just like
"public/assets"
3. Add a test for ".well-known/security.txt"
4. Simplify the "FileHandlerFunc" logic, now the paths are consistent so
the code can be simpler
5. Add CORS header for ".well-known" endpoints
6. Add logs to tell users they should move some of their legacy custom
public files

```
2023/07/19 13:00:37 cmd/web.go:178:serveInstalled() [E] Found legacy public asset "img" in CustomPath. Please move it to /work/gitea/custom/public/assets/img
2023/07/19 13:00:37 cmd/web.go:182:serveInstalled() [E] Found legacy public asset "robots.txt" in CustomPath. Please move it to /work/gitea/custom/public/robots.txt
```
This PR is not breaking.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>

cmd/web.go

@@ -15,9 +15,11 @@ import (
 
 	_ "net/http/pprof" // Used for debugging if enabled and a web server is running
 
+	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/public"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/routers/install"
@@ -175,6 +177,20 @@ func serveInstalled(ctx *cli.Context) error {
 		}
 	}
 
+	// in old versions, user's custom web files are placed in "custom/public", and they were served as "http://domain.com/assets/xxx"
+	// now, Gitea only serves pre-defined files in the "custom/public" folder basing on the web root, the user should move their custom files to "custom/public/assets"
+	publicFiles, _ := public.AssetFS().ListFiles(".")
+	publicFilesSet := container.SetOf(publicFiles...)
+	publicFilesSet.Remove(".well-known")
+	publicFilesSet.Remove("assets")
+	publicFilesSet.Remove("robots.txt")
+	for _, fn := range publicFilesSet.Values() {
+		log.Error("Found legacy public asset %q in CustomPath. Please move it to %s/public/assets/%s", fn, setting.CustomPath, fn)
+	}
+	if _, err := os.Stat(filepath.Join(setting.CustomPath, "robots.txt")); err == nil {
+		log.Error(`Found legacy public asset "robots.txt" in CustomPath. Please move it to %s/public/robots.txt`, setting.CustomPath)
+	}
+
 	routers.InitWebInstalled(graceful.GetManager().HammerContext())
 
 	// We check that AppDataPath exists here (it should have been created during installation)

docs/content/doc/administration/customizing-gitea.en-us.md

@@ -56,7 +56,11 @@ is set under the "Configuration" tab on the site administration page.
 
 To make Gitea serve custom public files (like pages and images), use the folder
 `$GITEA_CUSTOM/public/` as the webroot. Symbolic links will be followed.
-At the moment, only files in the `public/assets/` folder are served.
+At the moment, only the following files are served:
+
+- `public/robots.txt`
+- files in the `public/.well-known/` folder
+- files in the `public/assets/` folder
 
 For example, a file `image.png` stored in `$GITEA_CUSTOM/public/assets/`, can be accessed with
 the url `http://gitea.domain.tld/assets/image.png`.

modules/public/public.go

@@ -28,27 +28,15 @@ func AssetFS() *assetfs.LayeredFS {
 	return assetfs.Layered(CustomAssets(), BuiltinAssets())
 }
 
-// AssetsHandlerFunc implements the static handler for serving custom or original assets.
+// FileHandlerFunc implements the static handler for serving files in "public" assets
-func AssetsHandlerFunc(prefix string) http.HandlerFunc {
+func FileHandlerFunc() http.HandlerFunc {
 	assetFS := AssetFS()
-	prefix = strings.TrimSuffix(prefix, "/") + "/"
 	return func(resp http.ResponseWriter, req *http.Request) {
-		subPath := req.URL.Path
-		if !strings.HasPrefix(subPath, prefix) {
-			return
-		}
-		subPath = strings.TrimPrefix(subPath, prefix)
-
 		if req.Method != "GET" && req.Method != "HEAD" {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
-
+		handleRequest(resp, req, assetFS, req.URL.Path)
-		if handleRequest(resp, req, assetFS, subPath) {
-			return
-		}
-
-		resp.WriteHeader(http.StatusNotFound)
 	}
 }
 
@@ -71,16 +59,17 @@ func setWellKnownContentType(w http.ResponseWriter, file string) {
 	}
 }
 
-func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool {
+func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) {
 	// actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here
-	f, err := fs.Open(util.PathJoinRelX("assets", file))
+	f, err := fs.Open(util.PathJoinRelX(file))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return false
+			w.WriteHeader(http.StatusNotFound)
+			return
 		}
 		w.WriteHeader(http.StatusInternalServerError)
 		log.Error("[Static] Open %q failed: %v", file, err)
-		return true
+		return
 	}
 	defer f.Close()
 
@@ -88,17 +77,16 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem,
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		log.Error("[Static] %q exists, but fails to open: %v", file, err)
-		return true
+		return
 	}
 
-	// Try to serve index file
+	// need to serve index file? (no at the moment)
 	if fi.IsDir() {
 		w.WriteHeader(http.StatusNotFound)
-		return true
+		return
 	}
 
 	serveContent(w, req, fi, fi.ModTime(), f)
-	return true
 }
 
 type GzipBytesProvider interface {

modules/setting/server.go

@@ -349,9 +349,4 @@ func loadServerFrom(rootCfg ConfigProvider) {
 	default:
 		LandingPageURL = LandingPage(landingPage)
 	}
-
-	HasRobotsTxt, err = util.IsFile(path.Join(CustomPath, "robots.txt"))
-	if err != nil {
-		log.Error("Unable to check if %s is a file. Error: %v", path.Join(CustomPath, "robots.txt"), err)
-	}
 }

public/.well-known/security.txt

@@ -0,0 +1,6 @@
+# This site is running a Gitea instance.
+# Gitea related security problems could be reported to Gitea community.
+# Site related security problems should be reported to this site's admin.
+Contact: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
+Policy: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
+Preferred-Languages: en

routers/install/routes.go

@@ -20,7 +20,7 @@ import (
 func Routes() *web.Route {
 	base := web.NewRoute()
 	base.Use(common.ProtocolMiddlewares()...)
-	base.Methods("GET, HEAD", "/assets/*", public.AssetsHandlerFunc("/assets/"))
+	base.Methods("GET, HEAD", "/assets/*", public.FileHandlerFunc())
 
 	r := web.NewRoute()
 	r.Use(common.Sessioner(), Contexter())

routers/web/misc/misc.go

@@ -34,9 +34,12 @@ func DummyOK(w http.ResponseWriter, req *http.Request) {
 }
 
 func RobotsTxt(w http.ResponseWriter, req *http.Request) {
-	filePath := util.FilePathJoinAbs(setting.CustomPath, "robots.txt")
+	robotsTxt := util.FilePathJoinAbs(setting.CustomPath, "public/robots.txt")
+	if ok, _ := util.IsExist(robotsTxt); !ok {
+		robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt"
+	}
 	httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
-	http.ServeFile(w, req, filePath)
+	http.ServeFile(w, req, robotsTxt)
 }
 
 func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) {

routers/web/web.go

@@ -108,7 +108,7 @@ func Routes() *web.Route {
 	routes := web.NewRoute()
 
 	routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler
-	routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.AssetsHandlerFunc("/assets/"))
+	routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.FileHandlerFunc())
 	routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
 	routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars))
 	routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png"))
@@ -132,15 +132,12 @@ func Routes() *web.Route {
 		routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...)
 	}
 
-	if setting.HasRobotsTxt {
-		routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
-	}
-
 	if setting.Metrics.Enabled {
 		prometheus.MustRegister(metrics.NewCollector())
 		routes.Get("/metrics", append(mid, Metrics)...)
 	}
 
+	routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
 	routes.Get("/ssh_info", misc.SSHInfo)
 	routes.Get("/api/healthz", healthcheck.Check)
 
@@ -336,8 +333,7 @@ func registerRoutes(m *web.Route) {
 
 	// FIXME: not all routes need go through same middleware.
 	// Especially some AJAX requests, we can reduce middleware number to improve performance.
-	// Routers.
+
-	// for health check
 	m.Get("/", Home)
 	m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap)
 	m.Group("/.well-known", func() {
@@ -349,7 +345,8 @@ func registerRoutes(m *web.Route) {
 		m.Get("/change-password", func(ctx *context.Context) {
 			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
 		})
-	})
+		m.Any("/*", CorsHandler(), public.FileHandlerFunc())
+	}, CorsHandler())
 
 	m.Group("/explore", func() {
 		m.Get("", func(ctx *context.Context) {

tests/integration/links_test.go

@@ -38,6 +38,7 @@ func TestLinksNoLogin(t *testing.T) {
 		"/user2/repo1/projects/1",
 		"/assets/img/404.png",
 		"/assets/img/500.png",
+		"/.well-known/security.txt",
 	}
 
 	for _, link := range links {