db update requires own download logic for prefetch, previous logic caused check for updates to use old database and all databases were being updated before all repos were checked for cached pkgs

cmd/server/main.go

@@ -77,11 +77,13 @@ func main() {
 	mux.HandleFunc("POST /api/refresh", srv.handlerRefresh)
 	mux.HandleFunc("POST /api/loglevel", srv.handlerLogLevel)
 
-	if err := srv.c.FetchDB(); err != nil {
+	for _, repo := range cfg.MirroredRepos {
-		slog.Error("failed to refesh db files", "err", err)
+		if err := srv.c.FetchDB(repo); err != nil {
-		//nolint:errcheck //already exiting
+			slog.Error("failed to refesh db file", "repo", repo, "err", err)
-		_ = c.Close() // best effort cleanup on exit
+			//nolint:errcheck //already exiting
-		os.Exit(1)
+			_ = c.Close() // best effort cleanup on exit
+			os.Exit(1)
+		}
 	}
 
 	httpServe := &http.Server{

internal/cache/refresh.go

@@ -1,27 +1,57 @@
 package cache
 
-import "path/filepath"
+import (
+	"io"
+	"log/slog"
+	"net/http"
+	"path/filepath"
+)
 
-func (c *Cache) FetchDB() error {
+func (c *Cache) FetchDB(repo string) error {
-	if !c.refreshMu.TryLock() {
-		return nil
-	}
-	defer c.refreshMu.Unlock()
-
-	for _, repo := range c.cfg.mirroredRepos {
-		if err := c.fetchDB(repo); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (c *Cache) fetchDB(repo string) error {
 	dbFile := repo + ".db.tar.gz"
 	dbPath := filepath.Join(repo, "os/x86_64", dbFile)
-	_, _, err := c.getStream(dbPath)
+	url := c.nextMirror() + dbPath
+
+	// set the user agent
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		slog.Error("failed create request", "err", err)
+		return err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		slog.Warn("fetch failed", "url", url, "err", err)
+		return err
+	}
+	if resp.StatusCode != 200 {
+		slog.Info("fetch returned", "url", url, "status", resp.StatusCode)
+		return &UpstreamError{StatusCode: resp.StatusCode}
+	}
+	defer resp.Body.Close()
+
+	tmpPath := dbPath + ".tmp"
+	tmpFile, err := c.cr.Create(tmpPath)
 	if err != nil {
 		return err
 	}
+
+	if _, err := io.Copy(tmpFile, resp.Body); err != nil {
+		tmpFile.Close()
+		c.cr.Remove(tmpPath)
+		return err
+	}
+
+	if err := tmpFile.Close(); err != nil {
+		c.cr.Remove(tmpPath)
+		return err
+	}
+	if err := c.cr.Rename(tmpPath, dbPath); err != nil {
+		c.cr.Remove(tmpPath)
+		slog.Error("failed to move tmpfile to permanent file", "file", tmpFile, "err", err)
+		return err
+	}
+
 	return nil
 }

internal/repomaint/repomaint.go

@@ -16,7 +16,7 @@ const (
 )
 
 type CacheClient interface {
-	FetchDB() error
+	FetchDB(repo string) error
 	Fetch(relpath string) (*cache.CacheFile, error)
 }
 
@@ -53,7 +53,7 @@ func (r *RepoSync) Sync() error {
 
 		// call cache db fetch
 		slog.Info("refreshing databases")
-		if err := r.c.FetchDB(); err != nil {
+		if err := r.c.FetchDB(repo); err != nil {
 			return err
 		}
 

internal/repomaint/repomaint_test.go

@@ -19,7 +19,7 @@ type mockCache struct {
 	fetchDBCalled bool
 }
 
-func (m *mockCache) FetchDB() error {
+func (m *mockCache) FetchDB(repo string) error {
 	m.fetchDBCalled = true
 	return nil
 }