implmented fetch on cache miss

config.go

@@ -0,0 +1,13 @@
+package main
+
+type Config struct {
+	RepoPath  string
+	MirrorURL string
+}
+
+func NewConfig() *Config {
+	return &Config{
+		RepoPath:  "/home/ewpt3ch/dev/pacman-cache-server/tmprepo",
+		MirrorURL: "https://us.mirrors.cicku.me/archlinux/",
+	}
+}

go.mod

@@ -1,3 +1,5 @@
 module gitea.ewpt3ch.dev/ewpt3ch/pkgstash
 
 go 1.26.2
+
+require golang.org/x/sync v0.20.0

go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=

handlerPkgs.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"errors"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"gitea.ewpt3ch.dev/ewpt3ch/pkgstash/internal/cache"
+)
+
+func handlePackage(w http.ResponseWriter, req *http.Request, c *cache.Cache) {
+	// build file paths from the request, they follow archlinux repo
+	// <reporoot>/[core, extra, etc]/os/[x86_64, arm, etc]/package.pkg.tar.zst[.sig]
+	repo := req.PathValue("repo")
+	arch := req.PathValue("arch")
+	file := req.PathValue("file")
+	relPath := filepath.Join(repo, "os", arch, file) //path from repo root to pkg or db file
+	pkgPath := filepath.Join(repoRoot, relPath)      //path for local read of the file
+
+	if _, err := os.Stat(pkgPath); err != nil {
+		err = c.Fetch(relPath)
+		if err != nil {
+			var upstreamErr *cache.UpstreamError
+			if errors.As(err, &upstreamErr) {
+				http.Error(w, "Not found upstream", upstreamErr.StatusCode)
+				return
+			}
+			http.Error(w, "Failed to fetch from upstream", http.StatusBadGateway)
+
+		}
+	}
+
+	http.ServeFile(w, req, pkgPath)
+}

internal/cache/cache.go

@@ -0,0 +1,76 @@
+package cache
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sync/singleflight"
+)
+
+type Cache struct {
+	repo      string
+	mirrorURL string
+	sf        singleflight.Group
+}
+
+func NewCache(repo string, mirrorURL string) *Cache {
+	return &Cache{
+		repo:      repo,
+		mirrorURL: mirrorURL,
+	}
+}
+
+func (c *Cache) Fetch(pkgPath string) error {
+	_, err, _ := c.sf.Do(pkgPath, func() (any, error) {
+		return nil, c.fetch(pkgPath)
+	})
+	return err
+}
+
+type UpstreamError struct {
+	StatusCode int
+}
+
+func (e *UpstreamError) Error() string {
+	return fmt.Sprintf("upstream returned %d", e.StatusCode)
+}
+
+func (c *Cache) fetch(pkgPath string) error {
+	// pkgPath is relative to the repo or mirror root
+	tempPkgPath := pkgPath + ".tmp"
+	tempPkgName := filepath.Join(c.repo, tempPkgPath) //full tmp write path
+	outPkg := filepath.Join(c.repo, pkgPath)
+	pkgURL := c.mirrorURL + pkgPath
+
+	resp, err := http.Get(pkgURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return &UpstreamError{StatusCode: resp.StatusCode}
+	}
+
+	outFile, err := os.Create(tempPkgName)
+	if err != nil {
+		return err
+	}
+	defer outFile.Close()
+
+	_, err = io.Copy(outFile, resp.Body)
+	if err != nil {
+		os.Remove(tempPkgName)
+		return err
+	}
+
+	if err := os.Rename(tempPkgName, outPkg); err != nil {
+		os.Remove(tempPkgName)
+		return err
+	}
+	return nil
+}

main.go

@@ -3,17 +3,21 @@ package main
 import (
 	"log"
 	"net/http"
-	"os"
+
-	"path/filepath"
+	"gitea.ewpt3ch.dev/ewpt3ch/pkgstash/internal/cache"
 )
 
 const repoRoot = "/home/ewpt3ch/dev/pacman-cache-server/tmprepo"
 
 func main() {
 	const port = "8090"
+	cfg := NewConfig()
+	c := cache.NewCache(cfg.RepoPath, cfg.MirrorURL)
 
 	mux := http.NewServeMux()
-	mux.HandleFunc("GET /{repo}/os/{arch}/{file}", handlePackage)
+	mux.HandleFunc("GET /{repo}/os/{arch}/{file}", func(w http.ResponseWriter, req *http.Request) {
+		handlePackage(w, req, c)
+	})
 
 	httpServe := &http.Server{
 		Addr:    ":" + port,
@@ -23,21 +27,3 @@ func main() {
 	log.Fatal(httpServe.ListenAndServe())
 
 }
-
-func handlePackage(w http.ResponseWriter, req *http.Request) {
-	repo := req.PathValue("repo")
-	arch := req.PathValue("arch")
-	file := req.PathValue("file")
-
-	filePath := filepath.Join(repoRoot, repo, "os", arch, file)
-	// is where we handle cache misses and fetch the file
-	// from a mirror, for now we send a 404
-	if _, err := os.Stat(filePath); err != nil {
-		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-		w.WriteHeader(404)
-		w.Write([]byte("No such file"))
-		return
-	}
-
-	http.ServeFile(w, req, filePath)
-}