From 9fa2b93330526b41f0850e131943a8396257b58b Mon Sep 17 00:00:00 2001 From: Eric Phillips Date: Mon, 20 Apr 2026 22:18:22 -0600 Subject: [PATCH] added multiple mirrors to round robin downloads from --- config.go | 8 ++++---- deploy/pkgstash.toml.example | 4 +++- internal/cache/cache.go | 20 ++++++++++++++++---- main.go | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/config.go b/config.go index 2a2f985..0bb41da 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ import ( type Config struct { CacheRoot string `toml:"cache_root"` - MirrorURL string `toml:"mirror_url"` + MirrorURLs []string `toml:"mirror_url"` MirroredRepos []string `toml:"mirrored_repos"` Port string `toml:"port"` Auth AuthConfig `toml:"auth"` @@ -22,7 +22,7 @@ type AuthConfig struct { func NewConfig() *Config { return &Config{ CacheRoot: "/home/ewpt3ch/dev/pacman-cache-server/tmprepo", - MirrorURL: "https://us.mirrors.cicku.me/archlinux/", + MirrorURLs: "https://us.mirrors.cicku.me/archlinux/", Port: "8090", Auth: AuthConfig{Token: "FakeToken"}, } @@ -48,8 +48,8 @@ func (c *Config) validate() error { if c.CacheRoot == "" { return fmt.Errorf("cache root is required") } - if c.MirrorURL == "" { - return fmt.Errorf("mirror url is required") + if len(c.MirrorURLs) == 0 { + return fmt.Errorf("at least one mirror is required") } if c.Port == "" { c.Port = "8090" diff --git a/deploy/pkgstash.toml.example b/deploy/pkgstash.toml.example index dd1d326..e675559 100644 --- a/deploy/pkgstash.toml.example +++ b/deploy/pkgstash.toml.example @@ -1,5 +1,7 @@ cache_root = "/home/ewpt3ch/dev/pacman-cache-server/tmprepo" -mirror_url = "https://us.mirrors.cicku.me/archlinux/" +mirror_url = ["https://us.mirrors.cicku.me/archlinux/", + "https://losangeles.mirror.pkgbuild.com/", + "https://mirror.givebytes.net/archlinux/"] # array of upstream repos this server caches see pacman.conf # or pacman docs for more info mirrored_repos = ["core", "extra"] diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 032d210..123cecc 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -3,11 +3,13 @@ package cache import ( "fmt" "io" + "log" "net" "net/http" "os" "path/filepath" "sync" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -15,14 +17,15 @@ import ( type Cache struct { cacheRoot string - mirrorURL string + mirrorURLs []string mirroredRepos []string + mirrorIdx atomic.Uint32 sf singleflight.Group //prevents duplicate downloads mu sync.Mutex client http.Client } -func NewCache(cacheRoot string, mirrorURL string, mirroredRepos []string) *Cache { +func NewCache(cacheRoot string, mirrorURLs []string, mirroredRepos []string) *Cache { transport := &http.Transport{ DialContext: (&net.Dialer{ Timeout: 5 * time.Second, @@ -32,7 +35,7 @@ func NewCache(cacheRoot string, mirrorURL string, mirroredRepos []string) *Cache return &Cache{ cacheRoot: cacheRoot, - mirrorURL: mirrorURL, + mirrorURLs: mirrorURLs, mirroredRepos: mirroredRepos, client: http.Client{ Timeout: 15 * time.Second, @@ -42,7 +45,9 @@ func NewCache(cacheRoot string, mirrorURL string, mirroredRepos []string) *Cache } func (c *Cache) Fetch(pkgPath string) error { + log.Printf("pkgPath from Fetch %v", pkgPath) _, err, _ := c.sf.Do(pkgPath, func() (any, error) { + log.Print("calling fetch") return nil, c.fetch(pkgPath) }) return err @@ -62,7 +67,9 @@ func (c *Cache) fetch(pkgName string) error { tempPkgName := pkgName + ".tmp" tempPkgPath := filepath.Join(c.cacheRoot, tempPkgName) //full tmp write path outPkg := filepath.Join(c.cacheRoot, pkgName) - pkgURL := c.mirrorURL + pkgName + pkgURL := c.nextMirror() + pkgName + + log.Printf("fetching %v", pkgURL) resp, err := c.client.Get(pkgURL) if err != nil { @@ -92,3 +99,8 @@ func (c *Cache) fetch(pkgName string) error { } return nil } + +func (c *Cache) nextMirror() string { + idx := c.mirrorIdx.Add(1) - 1 + return c.mirrorURLs[idx%uint32(len(c.mirrorURLs))] +} diff --git a/main.go b/main.go index b833d8c..a34f15e 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ func main() { log.Fatal(err) } - c := cache.NewCache(cfg.CacheRoot, cfg.MirrorURL, cfg.MirroredRepos) + c := cache.NewCache(cfg.CacheRoot, cfg.MirrorURLs, cfg.MirroredRepos) srv := &Server{cfg: cfg, c: c} mux := http.NewServeMux()