zs/main.go
2023-03-18 13:06:33 +10:00

630 lines
16 KiB
Go

// Package main is a command-lint tool `zs` called Zen Static for generating static websites
package main
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strings"
"syscall"
"text/template"
"time"
ignore "github.com/sabhiram/go-gitignore"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer/html"
"go.mills.io/static"
"golang.org/x/sync/errgroup"
"gopkg.in/yaml.v2"
)
const (
// ZSDIR is the default directory for storing layouts and extensions
ZSDIR = ".zs"
// ZSIGNORE is the default ignore file
ZSIGNORE = ".zsignore"
// PUBDIR is the default directory for publishing final built content
PUBDIR = ".pub"
// DefaultIgnore is the default set of ignore patterns if no .zsignore
DefaultIgnore = `*~
*.bak
COPYING
Dockerfile
LICENSE
Makefile
README.md`
)
// Ignore holds a set of patterns to ignore from parsing a .zsignore file
var Ignore *ignore.GitIgnore
var (
// Production is a global variable to conditionally build for production
Production = false
)
// Vars holds a map of global variables
type Vars map[string]string
// NewTicker is a function that wraps a time.Ticker and ticks immediately instead of waiting for the first interval
func NewTicker(d time.Duration) *time.Ticker {
ticker := time.NewTicker(d)
oc := ticker.C
nc := make(chan time.Time, 1)
go func() {
nc <- time.Now()
for tm := range oc {
nc <- tm
}
}()
ticker.C = nc
return ticker
}
// RootCmd is the base command when called without any subcommands
var RootCmd = &cobra.Command{
Use: "zs",
Version: FullVersion(),
Short: "zs the zen static site generator",
Long: `zs is an extremely minimal static site generator written in Go.
- Keep your texts in markdown, or HTML format right in the main directory of your blog/site.
- Keep all service files (extensions, layout pages, deployment scripts etc) in the .zs subdirectory.
- Define variables in the header of the content files using YAML front matter:
- Use placeholders for variables and plugins in your markdown or html files, e.g. {{ title }} or {{ command arg1 arg2 }}.
- Write extensions in any language you like and put them into the .zs sub-directory.
- Everything the extensions prints to stdout becomes the value of the placeholder.
`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
debug, err := cmd.Flags().GetBool("debug")
if err != nil {
return fmt.Errorf("error getting debug flag: %w", err)
}
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
}
return nil
},
}
// BuildCmd is the build sub-command that performs whole builds or single builds
var BuildCmd = &cobra.Command{
Use: "build [<file>]",
Short: "Builds the whole site or a single file",
Long: `The build command builds the entire site or a single file if specified.`,
Args: cobra.RangeArgs(0, 1),
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
ctx := context.Background()
if err := buildAll(ctx, false); err != nil {
return fmt.Errorf("error building site: %w", err)
}
return nil
}
if err := build(args[0], os.Stdout, globals()); err != nil {
return fmt.Errorf("error building file %q", args[0])
}
return nil
},
}
// ServeCmd is the serve sub-command that performs whole builds or single builds
var ServeCmd = &cobra.Command{
Use: "serve [flags]",
Short: "Serves the site and rebuilds automatically",
Long: `The serve command serves the site and watches for rebuilds automatically`,
Args: cobra.RangeArgs(0, 1),
RunE: func(cmd *cobra.Command, args []string) error {
var wg errgroup.Group
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
bind, err := cmd.Flags().GetString("bind")
if err != nil {
return fmt.Errorf("error getting bind flag: %w", err)
}
root, err := cmd.Flags().GetString("root")
if err != nil {
return fmt.Errorf("error getting root flag: %w", err)
}
wg.Go(func() error {
if err := serve(ctx, bind, root); err != nil {
return fmt.Errorf("error serving site: %w", err)
}
return nil
})
if err := wg.Wait(); err != nil {
return fmt.Errorf("error running serve: %w", err)
}
return nil
},
}
// VarCmd is the var sub-command that performs whole builds or single builds
var VarCmd = &cobra.Command{
Use: "var <file> [<var>...]",
Aliases: []string{"vars"},
Short: "Display variables for the specified file",
Long: `The var command extracts and display sll teh variables defined in a file.
If the name of variables (optional) are passed as additional arguments, only those variables
are display instead of all variables (the default behavior).`,
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
s := ""
vars, _, err := getVars(args[0], Vars{})
if err != nil {
return fmt.Errorf("error getting variables from %s: %w", args[0], err)
}
if len(args) > 1 {
for _, a := range args[1:] {
s = s + vars[a] + "\n"
}
} else {
for k, v := range vars {
s = s + k + ":" + v + "\n"
}
}
fmt.Println(strings.TrimSpace(s))
return nil
},
}
// WatchCmd is the watch sub-command that performs whole builds or single builds
var WatchCmd = &cobra.Command{
Use: "watch",
Short: "Watches for file changes and rebuilds modified files",
Long: `The watch command watches for any changes to files and rebuilds them automatically`,
Args: cobra.RangeArgs(0, 1),
RunE: func(cmd *cobra.Command, args []string) error {
var wg errgroup.Group
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
wg.Go(func() error {
if err := buildAll(ctx, true); err != nil {
return fmt.Errorf("error watching for changes: %w", err)
}
return nil
})
if err := wg.Wait(); err != nil {
return fmt.Errorf("error running watch: %w", err)
}
return nil
},
}
// renameExt renames extension (if any) from oldext to newext
// If oldext is an empty string - extension is extracted automatically.
// If path has no extension - new extension is appended
func renameExt(path, oldext, newext string) string {
if oldext == "" {
oldext = filepath.Ext(path)
}
if oldext == "" || strings.HasSuffix(path, oldext) {
return strings.TrimSuffix(path, oldext) + newext
}
return path
}
// globals returns list of global OS environment variables that start
// with ZS_ prefix as Vars, so the values can be used inside templates
func globals() Vars {
vars := Vars{}
if Production {
vars["production"] = "1"
}
for _, e := range os.Environ() {
pair := strings.Split(e, "=")
if strings.HasPrefix(pair[0], "ZS_") {
vars[strings.ToLower(pair[0][3:])] = pair[1]
}
}
return vars
}
// run executes a command or a script. Vars define the command environment,
// each zs var is converted into OS environemnt variable with ZS_ prefix
// prepended. Additional variable $ZS contains path to the zs binary. Command
// stderr is printed to zs stderr, command output is returned as a string.
func run(vars Vars, cmd string, args ...string) (string, error) {
// First check if partial exists (.html)
if b, err := ioutil.ReadFile(filepath.Join(ZSDIR, cmd+".html")); err == nil {
return string(b), nil
}
var errbuf, outbuf bytes.Buffer
c := exec.Command(cmd, args...)
env := []string{"ZS=" + os.Args[0], "ZS_OUTDIR=" + PUBDIR}
env = append(env, os.Environ()...)
for k, v := range vars {
env = append(env, "ZS_"+strings.ToUpper(k)+"="+v)
}
c.Env = env
c.Stdout = &outbuf
c.Stderr = &errbuf
err := c.Run()
if errbuf.Len() > 0 {
log.Errorf("error running command: %s", cmd)
log.Error(errbuf.String())
}
if err != nil {
return "", err
}
return string(outbuf.Bytes()), nil
}
// getVars returns list of variables defined in a text file and actual file
// content following the variables declaration. Header is separated from
// content by an empty line. Header can be either YAML or JSON.
// If no empty newline is found - file is treated as content-only.
func getVars(path string, globals Vars) (Vars, string, error) {
if Ignore.MatchesPath(path) {
return nil, "", nil
}
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, "", err
}
s := string(b)
// Pick some default values for content-dependent variables
v := Vars{}
title := strings.Replace(strings.Replace(path, "_", " ", -1), "-", " ", -1)
v["title"] = strings.ToTitle(title)
v["description"] = ""
v["file"] = path
v["url"] = path[:len(path)-len(filepath.Ext(path))] + ".html"
v["output"] = filepath.Join(PUBDIR, v["url"])
// Override default values with globals
for name, value := range globals {
v[name] = value
}
// Add layout if none is specified
if _, ok := v["layout"]; !ok {
v["layout"] = "layout.html"
}
delim := "\n---\n"
sep := strings.Index(s, delim)
if sep == -1 {
return v, s, nil
}
header := s[:sep]
body := s[sep+len(delim):]
vars := Vars{}
if err := yaml.Unmarshal([]byte(header), &vars); err != nil {
log.WithError(err).Warn("failed to parse header")
return v, s, nil
}
// Override default values + globals with the ones defines in the file
for key, value := range vars {
v[key] = value
}
if strings.HasPrefix(v["url"], "./") {
v["url"] = v["url"][2:]
}
return v, body, nil
}
// Render expanding zs plugins and variables
func render(s string, vars Vars) (string, error) {
openingDelimiter := "{{"
closingDelimiter := "}}"
out := &bytes.Buffer{}
for {
from := strings.Index(s, openingDelimiter)
if from == -1 {
out.WriteString(s)
return out.String(), nil
}
to := strings.Index(s, closingDelimiter)
if to == -1 {
return "", fmt.Errorf("closing delimiter not found")
}
out.WriteString(s[:from])
cmd := s[from+len(openingDelimiter) : to]
s = s[to+len(closingDelimiter):]
m := strings.Fields(strings.TrimSpace(cmd))
if len(m) == 1 {
log.Debugf("vars: #%v", vars)
if v, ok := vars[m[0]]; ok {
out.WriteString(v)
continue
}
}
if _, err := exec.LookPath(m[0]); err == nil {
if res, err := run(vars, m[0], m[1:]...); err == nil {
out.WriteString(res)
} else {
log.WithError(err).Warnf("error running command: %s", m[0])
}
}
}
}
// Renders markdown with the given layout into html expanding all the macros
func buildMarkdown(path string, w io.Writer, vars Vars) error {
v, body, err := getVars(path, vars)
if err != nil {
return err
}
content, err := render(body, v)
if err != nil {
return err
}
buf := &bytes.Buffer{}
gm := goldmark.New(
goldmark.WithExtensions(
extension.GFM,
extension.Linkify,
),
goldmark.WithParserOptions(
parser.WithAutoHeadingID(),
),
goldmark.WithRendererOptions(
html.WithHardWraps(),
html.WithXHTML(),
html.WithUnsafe(),
),
)
if err := gm.Convert([]byte(content), buf); err != nil {
return err
}
v["content"] = buf.String()
if w == nil {
out, err := os.Create(filepath.Join(PUBDIR, renameExt(path, "", ".html")))
if err != nil {
return err
}
defer out.Close()
w = out
}
return buildHTML(filepath.Join(ZSDIR, v["layout"]), w, v)
}
// Renders text file expanding all variable macros inside it
func buildHTML(path string, w io.Writer, vars Vars) error {
v, body, err := getVars(path, vars)
if err != nil {
return err
}
if body, err = render(body, v); err != nil {
return err
}
tmpl, err := template.New("").Delims("<%", "%>").Parse(body)
if err != nil {
return err
}
if w == nil {
f, err := os.Create(filepath.Join(PUBDIR, path))
if err != nil {
return err
}
defer f.Close()
w = f
}
return tmpl.Execute(w, vars)
}
// Copies file as is from path to writer
func buildRaw(path string, w io.Writer) error {
in, err := os.Open(path)
if err != nil {
return err
}
defer in.Close()
if w == nil {
out, err := os.Create(filepath.Join(PUBDIR, path))
if err != nil {
return err
}
defer out.Close()
w = out
}
_, err = io.Copy(w, in)
return err
}
func build(path string, w io.Writer, vars Vars) error {
if Ignore.MatchesPath(path) {
return nil
}
ext := filepath.Ext(path)
if ext == ".md" || ext == ".mkd" {
return buildMarkdown(path, w, vars)
} else if ext == ".html" || ext == ".xml" {
return buildHTML(path, w, vars)
} else {
return buildRaw(path, w)
}
}
func buildAll(ctx context.Context, watch bool) error {
ticker := NewTicker(time.Second)
defer ticker.Stop()
lastModified := time.Unix(0, 0)
modified := false
vars := globals()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
os.Mkdir(PUBDIR, 0755)
filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
// rebuild if changes to .zs/ or .zsignore
if (filepath.Base(path) == ZSIGNORE || filepath.Dir(path) == ZSDIR) && info.ModTime().After(lastModified) {
Ignore = ParseIgnoreFile(path)
// reset lastModified to 0 so everything rebuidls
lastModified = time.Unix(0, 0)
return nil
}
// ignore hidden files and directories and ignored patterns
if filepath.Base(path)[0] == '.' || strings.HasPrefix(path, ".") || Ignore.MatchesPath(path) {
return nil
}
// inform user about fs walk errors, but continue iteration
if err != nil {
log.WithError(err).Warn("error walking directory")
return nil
}
if info.IsDir() {
os.Mkdir(filepath.Join(PUBDIR, path), 0755)
return nil
} else if info.ModTime().After(lastModified) {
if !modified {
// First file in this build cycle is about to be modified
if _, err := exec.LookPath("prehook"); err == nil {
if _, err := run(vars, "prehook"); err != nil {
log.WithError(err).Warn("error running prehook")
}
modified = true
}
}
log.Debugf("build: %s", path)
return build(path, nil, vars)
}
return nil
})
if modified {
// At least one file in this build cycle has been modified
if _, err := exec.LookPath("posthook"); err == nil {
if _, err := run(vars, "posthook"); err != nil {
log.WithError(err).Warn("error running posthook")
}
modified = false
}
}
if !watch {
return nil
}
lastModified = time.Now()
}
}
}
// serve runs a static web server and builds and continuously watches for changes to rebuild
func serve(ctx context.Context, bind, root string) error {
os.Mkdir(root, 0755)
svr, err := static.NewServer(
static.WithBind(bind),
static.WithDir(true),
static.WithRoot(root),
static.WithSPA(true),
)
if err != nil {
return err
}
go svr.Run(ctx)
go buildAll(ctx, true)
<-ctx.Done()
return nil
}
func ensureFirstPath(p string) {
paths := strings.Split(os.Getenv("PATH"), string(os.PathListSeparator))
if len(paths) > 0 && paths[0] != p {
paths = append([]string{p}, paths...)
os.Setenv("PATH", strings.Join(paths, string(os.PathListSeparator)))
}
}
func init() {
RootCmd.PersistentFlags().BoolP("debug", "d", false, "enable debug logging $($ZS_DEBUG)")
RootCmd.PersistentFlags().BoolVarP(&Production, "production", "p", false, "enable production mode ($ZS_PRODUCTION)")
ServeCmd.Flags().StringP("bind", "b", ":8000", "set the [<address>]:<port> to listen on")
ServeCmd.Flags().StringP("root", "r", PUBDIR, "set the root directory to serve")
RootCmd.AddCommand(BuildCmd)
RootCmd.AddCommand(ServeCmd)
RootCmd.AddCommand(VarCmd)
RootCmd.AddCommand(WatchCmd)
// prepend .zs to $PATH, so plugins will be found before OS commands
w, _ := os.Getwd()
ensureFirstPath(filepath.Join(w, ZSDIR))
}
func ParseIgnoreFile(fn string) *ignore.GitIgnore {
obj, err := ignore.CompileIgnoreFile(ZSIGNORE)
if err != nil {
log.WithError(err).Warnf("error parsing .zsignore: % (using defaults)s", fn)
return ignore.CompileIgnoreLines(DefaultIgnore)
}
return obj
}
func main() {
// prepend .zs to $PATH, so plugins will be found before OS commands
w, _ := os.Getwd()
ensureFirstPath(filepath.Join(w, ZSDIR))
// initialise Ignore (.zsignore) patterns
Ignore = ParseIgnoreFile(ZSIGNORE)
if err := RootCmd.Execute(); err != nil {
log.WithError(err).Error("error executing command")
os.Exit(1)
}
}