diff --git a/handlerPkgs.go b/handlerPkgs.go index ea40540..e76fd01 100644 --- a/handlerPkgs.go +++ b/handlerPkgs.go @@ -6,11 +6,19 @@ import ( "net/http" "os" "path/filepath" + "strings" "gitea.ewpt3ch.dev/ewpt3ch/pkgstash/internal/cache" ) func (s *Server) handlePackage(w http.ResponseWriter, req *http.Request) { + + // most mirrors don't have a *db.sig so we 404 it here instead of spamming the mirror + if strings.HasSuffix(req.PathValue("file"), ".db.sig") { + w.WriteHeader(http.StatusNotFound) + return + } + // build file paths from the request, they follow archlinux repo // /[core, extra, etc]/os/[x86_64, arm, etc]/package.pkg.tar.zst[.sig] repo := req.PathValue("repo") diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 675e75c..2fc1294 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -84,17 +84,37 @@ func (c *Cache) fetch(pkgName string) error { // final file name and path outPkg := filepath.Join(c.cfg.cacheRoot, pkgName) - pkgURL := c.nextMirror() + pkgName - log.Printf("fetching %v", pkgURL) - - resp, err := c.client.Get(pkgURL) - if err != nil { - return err + // declare vars outside loop + var resp *http.Response + var err error + // fetch pkgs from mirror with retry logic + for range len(c.cfg.mirrorURLs) { + pkgURL := c.nextMirror() + pkgName + log.Printf("fetching %v", pkgURL) + resp, err = c.client.Get(pkgURL) + if err != nil { + log.Printf("error fetching %s: %v", pkgURL, err) + continue + } + if resp.StatusCode == http.StatusOK { + break + } + log.Printf("retrying on code %v", resp.StatusCode) + resp.Body.Close() + } + if resp == nil { + return fmt.Errorf("all mirrors exhausted") } defer resp.Body.Close() + if err != nil { + log.Printf("exhauted all mirrors error: %v", err) + return err + } + if resp.StatusCode != http.StatusOK { + log.Printf("exhauted all mirrors %v", resp.StatusCode) return &UpstreamError{StatusCode: resp.StatusCode} } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index f61e8f9..d635ace 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -19,10 +19,10 @@ func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { return svr } -func newTestCache(t *testing.T, mirrorURL []string) *Cache { +func newTestCache(t *testing.T, mirrorURLs []string) *Cache { t.Helper() mirroredRepos := []string{"core", "extra"} - c := NewCache(t.TempDir(), mirrorURL, mirroredRepos) + c := NewCache(t.TempDir(), mirrorURLs, mirroredRepos) c.client.Timeout = 500 * time.Millisecond return c } @@ -62,7 +62,7 @@ func TestFetchNotFound(t *testing.T) { err := c.Fetch("fakefile") var upstreamErr *UpstreamError if !errors.As(err, &upstreamErr) { - t.Fatalf("expected UpstreamError fot %v", err) + t.Fatalf("expected UpstreamError got %v", err) } if upstreamErr.StatusCode != http.StatusNotFound { t.Errorf("expected 404 got %d", upstreamErr.StatusCode) @@ -109,3 +109,62 @@ func TestFetchSrvDead(t *testing.T) { t.Error("expected network error not UpstreamError") } } + +func TestFetchRetryExists(t *testing.T) { + const expected = "This is fake file contents" + + svr1 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + svr2 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, expected) + })) + fakeURLs := []string{ + svr1.URL + "/", + svr2.URL + "/", + } + + c := newTestCache(t, fakeURLs) + + err := c.Fetch("fakefile") + if err != nil { + t.Fatalf("fetch failed: %v", err) + } + + fakefilepath := filepath.Join(c.cfg.cacheRoot, "fakefile") + data, err := os.ReadFile(fakefilepath) + if err != nil { + t.Fatalf("error reading file back: %v", err) + } + if !bytes.Equal(data, []byte(expected)) { + t.Errorf("expected file to contain %s got %s", expected, data) + } +} + +func TestFetchRetryNonExist(t *testing.T) { + + svr1 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + svr2 := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + fakeURLs := []string{ + svr1.URL + "/", + svr2.URL + "/", + } + + c := newTestCache(t, fakeURLs) + + err := c.Fetch("fakefile") + var upstreamErr *UpstreamError + if !errors.As(err, &upstreamErr) { + t.Errorf("expected UpstreamError got %v", err) + } + if upstreamErr.StatusCode != http.StatusNotFound { + t.Errorf("expected 404 got %d", upstreamErr.StatusCode) + } +}