First pass integrated file server with cached SRI

.gitignore

@@ -1,2 +1,3 @@
 go.work*
 xtemplate*
+templates

TODO.md

@@ -3,8 +3,6 @@
 ### Features
 
 - [ ] Split xtemplate from caddy so it can be used standalone
-    - [ ] Integrate a static file server based on `caddy.caddyhttp.file_server`
-        - Serve static files from tempalates dir that are not .html files
 - [ ] Add "Why?" section to readme.
 
 ### Automation
@@ -29,6 +27,8 @@
 - [ ] Demo how to use standalone
 - [ ] Build a way to send live updates to a page by rendering a template to an SSE stream. Maybe backed by NATS.io?
 - [ ] Consider using the functional options pattern for configuring XTemplate
+- [ ] Convert *runtime to an `atomic.Pointer[T]`
+- [ ] Allow .ServeFile to serve files from contextfs
 
 
 # DONE
@@ -40,5 +40,6 @@
     - [x] Split into separate packages `xtemplate` and `xtemplate/caddy`, rename repo to `xtemplate`
     - [x] Write basic server based on net/http
     - [x] Update docs describe the separate packages
+    - [x] Integrate a static file server
 - [x] Add github automation
     - [x] Build and upload binaries

go.mod

@@ -21,6 +21,7 @@ require (
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect

go.sum

@@ -15,6 +15,8 @@ github.com/alecthomas/chroma/v2 v2.9.1/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF
 github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

serve.go

@@ -53,17 +53,15 @@ func (t *XTemplate) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	var headers = http.Header{}
 	context := &TemplateContext{
-		Req:     r,
-		Params:  params,
+		Req:    r,
+		Params: params,
+
 		Headers: WrappedHeader{headers},
-		Config:  t.Config,
-
-		status: http.StatusOK,
-		tmpl:   runtime.tmpl,
-		funcs:  runtime.funcs,
-		fs:     t.ContextFS,
-		log:    log,
-		tx:     tx,
+		status:  http.StatusOK,
+
+		log:     log,
+		tx:      tx,
+		runtime: runtime,
 	}
 
 	r.ParseForm()

templates.go

@@ -3,21 +3,28 @@
 package xtemplate
 
 import (
+	"compress/gzip"
+	"crypto/sha512"
 	"database/sql"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"html/template"
 	"io"
 	"io/fs"
 	"log/slog"
+	"net/http"
 	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"text/template/parse"
+	"time"
 
 	"github.com/Masterminds/sprig/v3"
+	"github.com/andybalholm/brotli"
 	"github.com/infogulch/pathmatcher"
+	"github.com/klauspost/compress/zstd"
 )
 
 type XTemplate struct {
@@ -33,24 +40,47 @@ type XTemplate struct {
 }
 
 type runtime struct {
-	funcs  template.FuncMap
-	tmpl   *template.Template
-	router *pathmatcher.HttpMatcher[*template.Template]
+	templateFS fs.FS
+	contextFS  fs.FS
+	config     map[string]string
+	funcs      template.FuncMap
+	templates  *template.Template
+	router     *pathmatcher.HttpMatcher[*template.Template]
+	files      map[string]fileInfo
+}
+
+type fileInfo struct {
+	hash, contentType string
+	encodings         []encodingInfo
+}
+
+type encodingInfo struct {
+	encoding, path string
+	size           int64
+	modtime        time.Time
 }
 
 func (t *XTemplate) Reload() error {
 	log := t.Log.WithGroup("reload")
 
+	r := &runtime{
+		templateFS: t.TemplateFS,
+		contextFS:  t.ContextFS,
+		config:     t.Config,
+		funcs:      make(template.FuncMap),
+		files:      make(map[string]fileInfo),
+		router:     pathmatcher.NewHttpMatcher[*template.Template](),
+	}
+
 	// Init funcs
-	funcs := make(template.FuncMap)
 	for _, fm := range append(t.ExtraFuncs, sprig.GenericFuncMap(), xtemplateFuncs) {
 		for n, f := range fm {
-			funcs[n] = f
+			r.funcs[n] = f
 		}
 	}
 
 	// Define the template instance that will accumulate all template definitions.
-	templates := template.New(".").Delims(t.Delims.L, t.Delims.R).Funcs(funcs)
+	r.templates = template.New(".").Delims(t.Delims.L, t.Delims.R).Funcs(r.funcs)
 
 	// Find all files and send the ones that match *.html into a channel. Will check walkErr later.
 	files := make(chan string)
@@ -60,32 +90,105 @@ func (t *XTemplate) Reload() error {
 			if err != nil {
 				return err
 			}
-			if ext := filepath.Ext(path); ext == ".html" {
-				files <- path
-			} else {
-				log.Debug("file ignored", "path", path, "ext", ext)
+			if d.IsDir() {
+				return nil
 			}
-			return err
+			files <- path
+			return nil
 		})
 		close(files)
 	}()
 
 	// Ingest all templates; add GET handlers for template files that don't start with '_'
 	for path_ := range files {
+
+		if ext := filepath.Ext(path_); ext != ".html" {
+			fsfile, err := r.templateFS.Open(path_)
+			if err != nil {
+				return fmt.Errorf("could not open raw file '%s': %w", path_, err)
+			}
+			defer fsfile.Close()
+			seeker := fsfile.(io.ReadSeeker)
+			stat, err := fsfile.Stat()
+			if err != nil {
+				return fmt.Errorf("could not stat file '%s': %w", path_, err)
+			}
+			size := stat.Size()
+
+			basepath := strings.TrimSuffix(path.Clean("/"+path_), ext)
+			var sri string
+			var reader io.Reader = fsfile
+			var encoding string = "identity"
+			file, exists := r.files[basepath]
+			if exists {
+				switch ext {
+				case ".gz":
+					reader, err = gzip.NewReader(seeker)
+					encoding = "gzip"
+				case ".zst":
+					reader, err = zstd.NewReader(seeker)
+					encoding = "zstd"
+				case ".br":
+					reader = brotli.NewReader(seeker)
+					encoding = "br"
+				}
+				if err != nil {
+					return fmt.Errorf("could not create decompressor for file `%s`: %w", path_, err)
+				}
+			} else {
+				basepath = path.Clean("/" + path_)
+			}
+			{
+				hash := sha512.New384()
+				_, err = io.Copy(hash, reader)
+				if err != nil {
+					return fmt.Errorf("could not hash file %w", err)
+				}
+				sri = "sha384-" + base64.StdEncoding.EncodeToString(hash.Sum(nil))
+			}
+			if encoding == "identity" {
+				// note: identity file will always be found first because fs.WalkDir sorts files in lexical order
+				file.hash = sri
+				if ctype, ok := extensionContentTypes[ext]; ok {
+					file.contentType = ctype
+				} else {
+					content := make([]byte, 512)
+					seeker.Seek(0, io.SeekStart)
+					count, err := seeker.Read(content)
+					if err != nil && err != io.EOF {
+						return fmt.Errorf("failed to read file to guess content type '%s': %w", path_, err)
+					}
+					file.contentType = http.DetectContentType(content[:count])
+				}
+				file.encodings = []encodingInfo{{encoding: encoding, path: path_, size: size, modtime: stat.ModTime()}}
+				r.templates.AddParseTree("GET "+basepath, serveFileTemplate)
+				r.templates.AddParseTree("HEAD "+basepath, serveFileTemplate)
+				log.Debug("added new direct serve file handler", slog.String("requestpath", basepath), slog.String("filepath", path_), slog.String("contenttype", file.contentType), slog.String("hash", sri), slog.Int64("size", size))
+			} else {
+				if file.hash != sri {
+					return fmt.Errorf("encoded file contents did not match original file '%s': expected %s, got %s", path_, file.hash, sri)
+				}
+				file.encodings = append(file.encodings, encodingInfo{encoding: encoding, path: path_, size: size, modtime: stat.ModTime()})
+				log.Debug("added new encoding to serve file", slog.String("requestpath", basepath), slog.String("filepath", path_), slog.String("encoding", encoding), slog.Int64("size", size), slog.Time("modtime", stat.ModTime()))
+			}
+			r.files[basepath] = file
+			continue
+		}
+
 		content, err := fs.ReadFile(t.TemplateFS, path_)
 		if err != nil {
 			return fmt.Errorf("could not read template file '%s': %v", path_, err)
 		}
-		path_ = filepath.Clean("/" + path_)
+		path_ = path.Clean("/" + path_)
 		// parse each template file manually to have more control over its final
 		// names in the template namespace.
-		newtemplates, err := parse.Parse(path_, string(content), t.Delims.L, t.Delims.R, funcs, buliltinsSkeleton)
+		newtemplates, err := parse.Parse(path_, string(content), t.Delims.L, t.Delims.R, r.funcs, buliltinsSkeleton)
 		if err != nil {
 			return fmt.Errorf("could not parse template file '%s': %v", path_, err)
 		}
 		// add all templates
 		for name, tree := range newtemplates {
-			_, err = templates.AddParseTree(name, tree)
+			_, err = r.templates.AddParseTree(name, tree)
 			if err != nil {
 				return fmt.Errorf("could not add template '%s' from '%s': %v", name, path_, err)
 			}
@@ -98,7 +201,7 @@ func (t *XTemplate) Reload() error {
 			}
 			route := "GET " + routePath
 			log.Debug("adding filename route template", "route", route, "routePath", routePath, "path", path_)
-			_, err = templates.AddParseTree(route, newtemplates[path_])
+			_, err = r.templates.AddParseTree(route, newtemplates[path_])
 			if err != nil {
 				return fmt.Errorf("could not add parse tree from '%s': %v", path_, err)
 			}
@@ -110,7 +213,7 @@ func (t *XTemplate) Reload() error {
 	}
 
 	// Invoke all initilization templates, aka any template whose name starts with "INIT "
-	for _, tmpl := range templates.Templates() {
+	for _, tmpl := range r.templates.Templates() {
 		if strings.HasPrefix(tmpl.Name(), "INIT ") {
 			var tx *sql.Tx
 			var err error
@@ -121,12 +224,9 @@ func (t *XTemplate) Reload() error {
 				}
 			}
 			err = tmpl.Execute(io.Discard, &TemplateContext{
-				tmpl:   templates,
-				funcs:  funcs,
-				fs:     t.ContextFS,
-				log:    log,
-				tx:     tx,
-				Config: t.Config,
+				runtime: r,
+				log:     log,
+				tx:      tx,
 			})
 			if err != nil {
 				if tx != nil {
@@ -147,27 +247,35 @@ func (t *XTemplate) Reload() error {
 	}
 
 	// Add all routing templates to the internal router
-	router := pathmatcher.NewHttpMatcher[*template.Template]()
 	matcher, _ := regexp.Compile("^(GET|POST|PUT|PATCH|DELETE) (.*)$")
 	count := 0
-	for _, tmpl := range templates.Templates() {
+	for _, tmpl := range r.templates.Templates() {
 		matches := matcher.FindStringSubmatch(tmpl.Name())
 		if len(matches) != 3 {
 			continue
 		}
 		method, path_ := matches[1], matches[2]
 		log.Debug("adding route handler", "method", method, "path", path_, "template_name", tmpl.Name())
 		tmpl := tmpl // create unique variable for closure
-		router.Add(method, path_, tmpl)
+		r.router.Add(method, path_, tmpl)
 		count += 1
 	}
 
 	// Set runtime in one pointer assignment, avoiding race conditions where the
 	// inner fields don't match.
-	t.runtime = &runtime{
-		funcs,
-		templates,
-		router,
-	}
+	t.runtime = r
 	return nil
 }
+
+var serveFileTemplate *parse.Tree
+
+func init() {
+	serveFiles, _ := parse.Parse("servefile", "{{.ServeFile}}", "{{", "}}")
+	serveFileTemplate = serveFiles["servefile"]
+}
+
+var extensionContentTypes map[string]string = map[string]string{
+	".css": "text/css; charset=utf-8",
+	".js":  "text/javascript; charset=utf-8",
+	".csv": "text/csv",
+}