prefetch complete

internal/repomaint/db.go

@@ -15,7 +15,7 @@ import (
 
 func (r *RepoSync) buildMap(repo string) (map[string]string, error) {
 	// create slice all filenames in repo
-	pkgs, err := r.cachedPkgs(repo)
+	pkgs, err := r.getCachedPkgs(repo)
 	if err != nil {
 		return nil, err
 	}
@@ -56,7 +56,50 @@ func (r *RepoSync) buildMap(repo string) (map[string]string, error) {
 	return pkgMap, nil
 }
 
-func (r *RepoSync) cachedPkgs(repo string) ([]string, error) {
+func (r *RepoSync) updatablePkgs(repo string, pkgs map[string]string) ([]string, error) {
+
+	// open db file
+	db, fr, gzr, err := r.openDb(repo)
+	if err != nil {
+		return nil, err
+	}
+	defer fr.Close()
+	defer gzr.Close()
+
+	files := []string{}
+	// for entry in db
+	for {
+		hdr, err := db.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if filepath.Base(hdr.Name) != "desc" {
+			continue
+		}
+		pkgName, currentFileName, err := parseDesc(db)
+		if err != nil {
+			slog.Warn("failed to parse desc file", "pkg", hdr.Name, "err", err)
+			continue
+		}
+		// check if pkg is cached, continue if not
+		cachedFileName, ok := pkgs[pkgName]
+		if !ok {
+			continue
+		}
+		// pkg is cached, check if needs update
+		if cachedFileName == currentFileName {
+			continue
+		}
+		files = append(files, currentFileName)
+
+	}
+	return files, nil
+}
+
+func (r *RepoSync) getCachedPkgs(repo string) ([]string, error) {
 	repoPath := filepath.Join(repo, repoArch)
 	f, err := r.root.Open(repoPath)
 	if err != nil {

internal/repomaint/repomaint.go

@@ -3,6 +3,7 @@ package repomaint
 import (
 	"log/slog"
 	"os"
+	"path/filepath"
 
 	"github.com/ewpt3ch/pkgstash/internal/cache"
 )
@@ -42,16 +43,34 @@ func (r *RepoSync) Sync() error {
 	for _, repo := range r.repos {
 		// create map of pkgname to filenames from current db
 		cachedPkgs, err := r.buildMap(repo)
+		if err != nil {
 			slog.Warn("failed to read current db", "err", err)
+			return err
+		}
 
 		// call cache db fetch
 		if err := r.c.FetchDB(); err != nil {
 			return err
 		}
 
+		// pkgsToUpdate slice contains relative paths like
+		// <repo>/os/<arch>/filename
+		pkgsToUpdate, err := r.updatablePkgs(repo, cachedPkgs)
+		if err != nil {
+			slog.Warn("failed to get updatable pkgs list", "err", err)
+			return err
+		}
+
+		for _, fileName := range pkgsToUpdate {
+			path := filepath.Join(repo, repoArch, fileName)
+			_, err := r.c.Fetch(path)
+			if err != nil {
+				slog.Warn("failed to update pkg", "pkg", fileName, "err", err)
+			}
+
 		}
-	// compare and fetch
 
 		// call cache cleanup
+	}
 	return nil
 }

internal/repomaint/repomaint_test.go

@@ -61,13 +61,13 @@ func TestCachedFiles(t *testing.T) {
 		rs.root.WriteFile(filepath.Join(repoPath, f), []byte{}, 0644)
 	}
 
-	cachedFiles, err := rs.cachedPkgs("core")
+	cachedFiles, err := rs.getCachedPkgs("core")
 	assert.Contains(t, cachedFiles, "linux")
 	assert.Contains(t, cachedFiles, "gcc-rust")
 	assert.Contains(t, cachedFiles, "linux-api-headers")
 }
 
-func TestSyncMapBuild(t *testing.T) {
+func TestBuildMap(t *testing.T) {
 	testFiles := make(map[string]string)
 	testFiles["linux"] = "linux-7.0.9.arch1-1-x86_64.pkg.tar.zst"
 	testFiles["linux-api-headers"] = "linux-api-headers-6.19-1-x86_64.pkg.tar.zst"
@@ -113,3 +113,46 @@ func TestSyncMapBuild(t *testing.T) {
 	})
 
 }
+
+func TestUpdatablePkgs(t *testing.T) {
+	repo := "core"
+	testMap := make(map[string]string)
+	testMap["linux"] = "linux-7.0.9.arch1-1"
+	testMap["linux-api-headers"] = "linux-a"
+	testMap["gcc-rust"] = "gcc-rust-16.1.1+r12+g301eb08fa2c5-1-x86_64.pkg.tar.zst"
+	expectedFileName1 := "linux-7.0.9.arch1-1-x86_64.pkg.tar.zst"
+	expectedFileName2 := "linux-api-headers-6.19-1-x86_64.pkg.tar.zst"
+	unexpectedFileName := "gcc-rust-16.1.1+r12+g301eb08fa2c5-1-x86_64.pkg.tar.zst"
+
+	// create test structs
+	c := &mockCache{
+		fetched: make([]string, 5),
+	}
+	rs, err := NewRepoSync(c, t.TempDir(), []string{"core"})
+	if err != nil {
+		t.Fatalf("failed to create RepoMaint: %v", err)
+	}
+	repoPath := filepath.Join(rs.repos[0], repoArch)
+	rs.root.MkdirAll(repoPath, 0750)
+
+	src, err := os.Open(filepath.Join("testdata", "pre.core.db.tar.gz"))
+	if err != nil {
+		t.Fatalf("unable to open src db for copy: %v", err)
+	}
+	defer src.Close()
+	dst, err := rs.root.Create(filepath.Join(repoPath, "core.db.tar.gz"))
+	if err != nil {
+		t.Fatalf("unable to open dst db for copy: %s", err)
+	}
+	defer dst.Close()
+	if _, err := io.Copy(dst, src); err != nil {
+		t.Fatalf("failed to copy db: %v", err)
+	}
+
+	files, err := rs.updatablePkgs(repo, testMap)
+	require.NoError(t, err)
+	assert.Contains(t, files, expectedFileName1)
+	assert.Contains(t, files, expectedFileName2)
+	assert.NotContains(t, files, unexpectedFileName)
+
+}