added retry feature on failed fetches from mirror

handlerPkgs.go

@@ -6,11 +6,19 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"gitea.ewpt3ch.dev/ewpt3ch/pkgstash/internal/cache"
 )
 
 func (s *Server) handlePackage(w http.ResponseWriter, req *http.Request) {
+
+	// most mirrors don't have a *db.sig so we 404 it here instead of spamming the mirror
+	if strings.HasSuffix(req.PathValue("file"), ".db.sig") {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
 	// build file paths from the request, they follow archlinux repo
 	// <mirrorroot>/[core, extra, etc]/os/[x86_64, arm, etc]/package.pkg.tar.zst[.sig]
 	repo := req.PathValue("repo")

internal/cache/cache.go

@@ -84,17 +84,37 @@ func (c *Cache) fetch(pkgName string) error {
 
 	// final file name and path
 	outPkg := filepath.Join(c.cfg.cacheRoot, pkgName)
-	pkgURL := c.nextMirror() + pkgName
 
-	log.Printf("fetching %v", pkgURL)
+	// declare vars outside loop
-
+	var resp *http.Response
-	resp, err := c.client.Get(pkgURL)
+	var err error
-	if err != nil {
+	// fetch pkgs from mirror with retry logic
-		return err
+	for range len(c.cfg.mirrorURLs) {
+		pkgURL := c.nextMirror() + pkgName
+		log.Printf("fetching %v", pkgURL)
+		resp, err = c.client.Get(pkgURL)
+		if err != nil {
+			log.Printf("error fetching %s: %v", pkgURL, err)
+			continue
+		}
+		if resp.StatusCode == http.StatusOK {
+			break
+		}
+		log.Printf("retrying on code %v", resp.StatusCode)
+		resp.Body.Close()
+	}
+	if resp == nil {
+		return fmt.Errorf("all mirrors exhausted")
 	}
 	defer resp.Body.Close()
 
+	if err != nil {
+		log.Printf("exhauted all mirrors error: %v", err)
+		return err
+	}
+
 	if resp.StatusCode != http.StatusOK {
+		log.Printf("exhauted all mirrors %v", resp.StatusCode)
 		return &UpstreamError{StatusCode: resp.StatusCode}
 	}
 

internal/cache/cache_test.go

@@ -19,10 +19,10 @@ func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
 	return svr
 }
 
-func newTestCache(t *testing.T, mirrorURL []string) *Cache {
+func newTestCache(t *testing.T, mirrorURLs []string) *Cache {
 	t.Helper()
 	mirroredRepos := []string{"core", "extra"}
-	c := NewCache(t.TempDir(), mirrorURL, mirroredRepos)
+	c := NewCache(t.TempDir(), mirrorURLs, mirroredRepos)
 	c.client.Timeout = 500 * time.Millisecond
 	return c
 }
@@ -62,7 +62,7 @@ func TestFetchNotFound(t *testing.T) {
 	err := c.Fetch("fakefile")
 	var upstreamErr *UpstreamError
 	if !errors.As(err, &upstreamErr) {
-		t.Fatalf("expected UpstreamError fot %v", err)
+		t.Fatalf("expected UpstreamError got %v", err)
 	}
 	if upstreamErr.StatusCode != http.StatusNotFound {
 		t.Errorf("expected 404 got %d", upstreamErr.StatusCode)
@@ -109,3 +109,62 @@ func TestFetchSrvDead(t *testing.T) {
 		t.Error("expected network error not UpstreamError")
 	}
 }
+
+func TestFetchRetryExists(t *testing.T) {
+	const expected = "This is fake file contents"
+
+	svr1 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+
+	svr2 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, expected)
+	}))
+	fakeURLs := []string{
+		svr1.URL + "/",
+		svr2.URL + "/",
+	}
+
+	c := newTestCache(t, fakeURLs)
+
+	err := c.Fetch("fakefile")
+	if err != nil {
+		t.Fatalf("fetch failed: %v", err)
+	}
+
+	fakefilepath := filepath.Join(c.cfg.cacheRoot, "fakefile")
+	data, err := os.ReadFile(fakefilepath)
+	if err != nil {
+		t.Fatalf("error reading file back: %v", err)
+	}
+	if !bytes.Equal(data, []byte(expected)) {
+		t.Errorf("expected file to contain %s got %s", expected, data)
+	}
+}
+
+func TestFetchRetryNonExist(t *testing.T) {
+
+	svr1 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+
+	svr2 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+
+	fakeURLs := []string{
+		svr1.URL + "/",
+		svr2.URL + "/",
+	}
+
+	c := newTestCache(t, fakeURLs)
+
+	err := c.Fetch("fakefile")
+	var upstreamErr *UpstreamError
+	if !errors.As(err, &upstreamErr) {
+		t.Errorf("expected UpstreamError got %v", err)
+	}
+	if upstreamErr.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 got %d", upstreamErr.StatusCode)
+	}
+}