From 5ca42e11cdc4adc5f2320ef6d9a19c8a00653ab3 Mon Sep 17 00:00:00 2001 From: Eric Phillips Date: Sat, 23 May 2026 20:12:13 -0600 Subject: [PATCH] prefetch complete --- internal/repomaint/db.go | 47 ++++++++++++++++++++++++++-- internal/repomaint/repomaint.go | 27 +++++++++++++--- internal/repomaint/repomaint_test.go | 47 ++++++++++++++++++++++++++-- 3 files changed, 113 insertions(+), 8 deletions(-) diff --git a/internal/repomaint/db.go b/internal/repomaint/db.go index 049a734..15454aa 100644 --- a/internal/repomaint/db.go +++ b/internal/repomaint/db.go @@ -15,7 +15,7 @@ import ( func (r *RepoSync) buildMap(repo string) (map[string]string, error) { // create slice all filenames in repo - pkgs, err := r.cachedPkgs(repo) + pkgs, err := r.getCachedPkgs(repo) if err != nil { return nil, err } @@ -56,7 +56,50 @@ func (r *RepoSync) buildMap(repo string) (map[string]string, error) { return pkgMap, nil } -func (r *RepoSync) cachedPkgs(repo string) ([]string, error) { +func (r *RepoSync) updatablePkgs(repo string, pkgs map[string]string) ([]string, error) { + + // open db file + db, fr, gzr, err := r.openDb(repo) + if err != nil { + return nil, err + } + defer fr.Close() + defer gzr.Close() + + files := []string{} + // for entry in db + for { + hdr, err := db.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if filepath.Base(hdr.Name) != "desc" { + continue + } + pkgName, currentFileName, err := parseDesc(db) + if err != nil { + slog.Warn("failed to parse desc file", "pkg", hdr.Name, "err", err) + continue + } + // check if pkg is cached, continue if not + cachedFileName, ok := pkgs[pkgName] + if !ok { + continue + } + // pkg is cached, check if needs update + if cachedFileName == currentFileName { + continue + } + files = append(files, currentFileName) + + } + return files, nil +} + +func (r *RepoSync) getCachedPkgs(repo string) ([]string, error) { repoPath := filepath.Join(repo, repoArch) f, err := r.root.Open(repoPath) if err != nil { diff --git a/internal/repomaint/repomaint.go b/internal/repomaint/repomaint.go index ebf42da..35dbbc7 100644 --- a/internal/repomaint/repomaint.go +++ b/internal/repomaint/repomaint.go @@ -3,6 +3,7 @@ package repomaint import ( "log/slog" "os" + "path/filepath" "github.com/ewpt3ch/pkgstash/internal/cache" ) @@ -42,16 +43,34 @@ func (r *RepoSync) Sync() error { for _, repo := range r.repos { // create map of pkgname to filenames from current db cachedPkgs, err := r.buildMap(repo) - slog.Warn("failed to read current db", "err", err) + if err != nil { + slog.Warn("failed to read current db", "err", err) + return err + } // call cache db fetch if err := r.c.FetchDB(); err != nil { return err } - } - // compare and fetch + // pkgsToUpdate slice contains relative paths like + // /os//filename + pkgsToUpdate, err := r.updatablePkgs(repo, cachedPkgs) + if err != nil { + slog.Warn("failed to get updatable pkgs list", "err", err) + return err + } - // call cache cleanup + for _, fileName := range pkgsToUpdate { + path := filepath.Join(repo, repoArch, fileName) + _, err := r.c.Fetch(path) + if err != nil { + slog.Warn("failed to update pkg", "pkg", fileName, "err", err) + } + + } + + // call cache cleanup + } return nil } diff --git a/internal/repomaint/repomaint_test.go b/internal/repomaint/repomaint_test.go index dd590d9..3a4890d 100644 --- a/internal/repomaint/repomaint_test.go +++ b/internal/repomaint/repomaint_test.go @@ -61,13 +61,13 @@ func TestCachedFiles(t *testing.T) { rs.root.WriteFile(filepath.Join(repoPath, f), []byte{}, 0644) } - cachedFiles, err := rs.cachedPkgs("core") + cachedFiles, err := rs.getCachedPkgs("core") assert.Contains(t, cachedFiles, "linux") assert.Contains(t, cachedFiles, "gcc-rust") assert.Contains(t, cachedFiles, "linux-api-headers") } -func TestSyncMapBuild(t *testing.T) { +func TestBuildMap(t *testing.T) { testFiles := make(map[string]string) testFiles["linux"] = "linux-7.0.9.arch1-1-x86_64.pkg.tar.zst" testFiles["linux-api-headers"] = "linux-api-headers-6.19-1-x86_64.pkg.tar.zst" @@ -113,3 +113,46 @@ func TestSyncMapBuild(t *testing.T) { }) } + +func TestUpdatablePkgs(t *testing.T) { + repo := "core" + testMap := make(map[string]string) + testMap["linux"] = "linux-7.0.9.arch1-1" + testMap["linux-api-headers"] = "linux-a" + testMap["gcc-rust"] = "gcc-rust-16.1.1+r12+g301eb08fa2c5-1-x86_64.pkg.tar.zst" + expectedFileName1 := "linux-7.0.9.arch1-1-x86_64.pkg.tar.zst" + expectedFileName2 := "linux-api-headers-6.19-1-x86_64.pkg.tar.zst" + unexpectedFileName := "gcc-rust-16.1.1+r12+g301eb08fa2c5-1-x86_64.pkg.tar.zst" + + // create test structs + c := &mockCache{ + fetched: make([]string, 5), + } + rs, err := NewRepoSync(c, t.TempDir(), []string{"core"}) + if err != nil { + t.Fatalf("failed to create RepoMaint: %v", err) + } + repoPath := filepath.Join(rs.repos[0], repoArch) + rs.root.MkdirAll(repoPath, 0750) + + src, err := os.Open(filepath.Join("testdata", "pre.core.db.tar.gz")) + if err != nil { + t.Fatalf("unable to open src db for copy: %v", err) + } + defer src.Close() + dst, err := rs.root.Create(filepath.Join(repoPath, "core.db.tar.gz")) + if err != nil { + t.Fatalf("unable to open dst db for copy: %s", err) + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + t.Fatalf("failed to copy db: %v", err) + } + + files, err := rs.updatablePkgs(repo, testMap) + require.NoError(t, err) + assert.Contains(t, files, expectedFileName1) + assert.Contains(t, files, expectedFileName2) + assert.NotContains(t, files, unexpectedFileName) + +}