Handle symlinks in populate: * Accept but ignore Utimens() on directory nodes. This is for symlinks to include/ directories * Return added and changed files separately. This is so we don't touch everything for a sloth-populate call into an empty workspace. * Cleanup e2e test code, and add a case for a changing symlink. Change-Id: I743b9887182077bf9c63d660031b92823fbeb1f8
diff --git a/cmd/slothfs-populate/main.go b/cmd/slothfs-populate/main.go index 5c50d79..c130edf 100644 --- a/cmd/slothfs-populate/main.go +++ b/cmd/slothfs-populate/main.go
@@ -34,16 +34,24 @@ log.Fatal("too many arguments.") } - changed, err := populate.Checkout(*mount, dir) + added, changed, err := populate.Checkout(*mount, dir) if err != nil { log.Fatalf("populate.Checkout: %v", err) } - now := time.Now() - for _, c := range changed { - if err := os.Chtimes(c, now, now); err != nil { - log.Fatalf("Chtimes(%s): %v", c, err) + if len(changed) > 0 { + now := time.Now() + n := 0 + for _, slice := range [][]string{added, changed} { + for _, c := range slice { + if err := os.Chtimes(c, now, now); err != nil { + log.Fatalf("Chtimes(%s): %v", c, err) + } + n++ + } } + log.Printf("touched %d files", n) + } else { + log.Printf("no files were changed, %d were added; assuming fresh checkout.", len(added)) } - log.Printf("touched %d files", len(changed)) }
diff --git a/fs/gitilesfs.go b/fs/gitilesfs.go index de4661c..a43c5fd 100644 --- a/fs/gitilesfs.go +++ b/fs/gitilesfs.go
@@ -330,6 +330,12 @@ nodefs.Node } +// Implement Utimens so we don't create spurious "not implemented" +// messages when directory targets for symlinks are touched. +func (n *dirNode) Utimens(file nodefs.File, atime *time.Time, mtime *time.Time, context *fuse.Context) (code fuse.Status) { + return fuse.OK +} + func (n *dirNode) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) (code fuse.Status) { out.Mode = fuse.S_IFDIR | 0755 t := time.Unix(1, 0)
diff --git a/populate/e2e_test.go b/populate/e2e_test.go index 47223ff..9f141cf 100644 --- a/populate/e2e_test.go +++ b/populate/e2e_test.go
@@ -7,12 +7,14 @@ "net" "os" "path/filepath" + "reflect" "testing" "github.com/google/slothfs/cache" "github.com/google/slothfs/fs" "github.com/google/slothfs/gitiles" "github.com/google/slothfs/manifest" + "github.com/hanwen/go-fuse/fuse" "github.com/hanwen/go-fuse/fuse/nodefs" git "github.com/libgit2/git2go" @@ -38,58 +40,110 @@ return &i } +func newString(s string) *string { + return &s +} + func abortListener(l net.Listener) { - _, err := l.Accept() - if err == nil { - log.Panicf("got incoming connection") + for { + conn, err := l.Accept() + if err != nil { + break + } + conn.Close() } } -func TestFUSE(t *testing.T) { +type fixture struct { + dir string + cache *cache.Cache + fsServer *fuse.Server + abortGitiles net.Listener +} + +func (f *fixture) Cleanup() { + if f.abortGitiles != nil { + f.abortGitiles.Close() + } + if f.fsServer != nil { + if err := f.fsServer.Unmount(); err != nil { + return + } + } + os.RemoveAll(f.dir) +} + +func (f *fixture) addWorkspace(name string, mf *manifest.Manifest) error { + bytes1, err := mf.MarshalXML() + if err != nil { + return err + } + + dir := f.dir + + if err := ioutil.WriteFile(filepath.Join(dir, name+".xml"), bytes1, 0644); err != nil { + return err + } + if err := os.Symlink(filepath.Join(dir, name+".xml"), filepath.Join(dir, "mnt", "config", name)); err != nil { + return err + } + return nil +} + +func newFixture() (*fixture, error) { dir, err := ioutil.TempDir("", "") if err != nil { - t.Fatal(err) + return nil, err } - defer os.RemoveAll(dir) + + fix := fixture{dir: dir} for _, d := range []string{"mnt", "ws", "cache"} { if err := os.MkdirAll(filepath.Join(dir, d), 0755); err != nil { - t.Fatal(err) + return nil, err } } - cache, err := cache.NewCache(filepath.Join(dir, "cache"), cache.Options{}) + fix.cache, err = cache.NewCache(filepath.Join(dir, "cache"), cache.Options{}) if err != nil { - t.Fatal(err) + return nil, err } // Setup a fake gitiles; make sure we never talk to it. - l, err := net.Listen("tcp", ":0") + fix.abortGitiles, err = net.Listen("tcp", ":0") if err != nil { - t.Fatal(err) + return nil, err } - go abortListener(l) - defer l.Close() + go abortListener(fix.abortGitiles) - service, err := gitiles.NewService(fmt.Sprintf("http://%s/", l.Addr()), gitiles.Options{}) + service, err := gitiles.NewService(fmt.Sprintf("http://%s/", fix.abortGitiles.Addr()), gitiles.Options{}) if err != nil { log.Printf("NewService: %v", err) } opts := fs.MultiFSOptions{} - root := fs.NewMultiFS(service, cache, opts) + root := fs.NewMultiFS(service, fix.cache, opts) fuseOpts := nodefs.NewOptions() server, _, err := nodefs.MountRoot(filepath.Join(dir, "mnt"), root, fuseOpts) if err != nil { - t.Fatal(err) + return nil, err } go server.Serve() - defer server.Unmount() + + return &fix, nil +} + +func TestFUSESymlink(t *testing.T) { + fixture, err := newFixture() + if err != nil { + t.Fatal(err) + } + defer fixture.Cleanup() // We avoid talking to gitiles by inserting entries into the // cache manually. - if err := cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{ + if err := fixture.cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{ ID: ids[0], Entries: []gitiles.TreeEntry{ { @@ -100,6 +154,98 @@ Size: newInt(42), }, { + Mode: 0100644, + Name: "link", + Type: "blob", + ID: ids[2], + Size: newInt(1), + Target: newString("non-existent"), + }, + }, + }); err != nil { + t.Fatal(err) + } + + // We avoid talking to gitiles by inserting entries into the + // cache manually. + if err := fixture.cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{ + ID: ids[1], + Entries: []gitiles.TreeEntry{ + { + Mode: 0100644, + Name: "a", + Type: "blob", + ID: ids[1], + Size: newInt(42), + }, + { + Mode: 0100644, + Name: "link", + Type: "blob", + ID: ids[3], + Size: newInt(1), + Target: newString("a"), + }, + }, + }); err != nil { + t.Fatal(err) + } + + for i := 0; i <= 1; i++ { + if err := fixture.addWorkspace(fmt.Sprintf("m%d", i), &manifest.Manifest{ + Project: []manifest.Project{{ + Name: "platform/project", + Path: "p", + Revision: ids[i], + }}}); err != nil { + t.Fatalf("addWorkspace(%d): %v", i, err) + } + } + + ws := filepath.Join(fixture.dir, "ws") + m0 := filepath.Join(fixture.dir, "mnt", "m0") + added, changed, err := Checkout(m0, ws) + if err != nil { + t.Fatalf("Checkout m0: %v", err) + } + if len(changed) > 0 { + t.Errorf("got changed files %v on fresh checkout", changed) + } + if want := []string{filepath.Join(m0, "p/a"), filepath.Join(m0, "p/link")}; !reflect.DeepEqual(added, want) { + t.Errorf("got added %v want %v on fresh checkout", added, want) + } + + m1 := filepath.Join(fixture.dir, "mnt", "m1") + added, changed, err = Checkout(m1, ws) + if len(added) > 0 { + t.Errorf("got added files %v on sync", added) + } + if want := []string{filepath.Join(m1, "p/link")}; !reflect.DeepEqual(changed, want) { + t.Errorf("got changed files %v, want %v", changed, want) + } +} + +func TestBasic(t *testing.T) { + fixture, err := newFixture() + if err != nil { + t.Fatal(err) + } + defer fixture.Cleanup() + dir := fixture.dir + + // We avoid talking to gitiles by inserting entries into the + // cache manually. + if err := fixture.cache.Tree.Add(gitID(ids[0]), &gitiles.Tree{ + ID: ids[0], + Entries: []gitiles.TreeEntry{ + { + Mode: 0100644, + Name: "a", + Type: "blob", + ID: ids[1], + Size: newInt(1), + }, + { Mode: 0100644, Name: "b/c", Type: "blob", @@ -110,7 +256,7 @@ }); err != nil { t.Fatal(err) } - if err := cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{ + if err := fixture.cache.Tree.Add(gitID(ids[1]), &gitiles.Tree{ ID: ids[1], Entries: []gitiles.TreeEntry{ { @@ -118,7 +264,7 @@ Name: "a", Type: "blob", ID: ids[2], - Size: newInt(42), + Size: newInt(1), }, { Mode: 0100644, @@ -132,13 +278,14 @@ Name: "new", Type: "blob", ID: ids[3], - Size: newInt(42), + Size: newInt(1), }, }, }); err != nil { t.Fatal(err) } - if err := cache.Tree.Add(gitID(ids[2]), &gitiles.Tree{ + + if err := fixture.cache.Tree.Add(gitID(ids[2]), &gitiles.Tree{ ID: ids[2], Entries: []gitiles.TreeEntry{ { @@ -146,21 +293,23 @@ Name: "d", Type: "blob", ID: ids[3], - Size: newInt(42), + Size: newInt(1), }, }, }); err != nil { t.Fatal(err) } - mf1 := manifest.Manifest{ + if err := fixture.addWorkspace("m1", &manifest.Manifest{ Project: []manifest.Project{{ Name: "platform/project", Path: "project", Revision: ids[0], - }}} + }}}); err != nil { + t.Fatalf("addWorkspace(m1): %v", err) + } - mf2 := manifest.Manifest{ + if err := fixture.addWorkspace("m2", &manifest.Manifest{ Project: []manifest.Project{ { Name: "platform/project", @@ -171,42 +320,20 @@ Path: "sub", Revision: ids[2], }}, - } - - bytes1, err := mf1.MarshalXML() - if err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(dir, "m1.xml"), bytes1, 0644); err != nil { - t.Fatal(err) - } - - bytes2, err := mf2.MarshalXML() - if err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(dir, "m2.xml"), bytes2, 0644); err != nil { - t.Fatal(err) - } - - if err := os.Symlink(filepath.Join(dir, "m1.xml"), filepath.Join(dir, "mnt", "config", "m1")); err != nil { - t.Fatal(err) + }); err != nil { + t.Fatalf("addWorkspace(m2): %v", err) } testFile := filepath.Join(dir, "mnt", "m1", "project", "b/c") if fi, err := os.Lstat(testFile); err != nil { t.Fatalf("Lstat(%s): %v", testFile, err) } else if fi.Size() != 1 { - t.Fatalf("%s has size %d", fi.Size()) - } - - if err := os.Symlink(filepath.Join(dir, "m2.xml"), filepath.Join(dir, "mnt", "config", "m2")); err != nil { - t.Fatal(err) + t.Fatalf("%s has size %d", testFile, fi.Size()) } ws := filepath.Join(dir, "ws") - if _, err := Checkout(filepath.Join(dir, "mnt", "m1"), ws); err != nil { + if _, _, err := Checkout(filepath.Join(dir, "mnt", "m1"), ws); err != nil { t.Fatal("Checkout m1:", err) } @@ -220,27 +347,25 @@ // the test setup that no blobs are shared with newly // appearing files, or they'll be touched for being new files. - changed, err := Checkout(filepath.Join(dir, "mnt", "m2"), ws) + added, changed, err := Checkout(filepath.Join(dir, "mnt", "m2"), ws) if err != nil { t.Fatal(err) } - for _, f := range []string{"project/a", "project/new"} { - found := false - for _, c := range changed { - if c == filepath.Join(dir, "mnt", "m2", f) { - found = true - break - } - } - if !found { - t.Errorf("file %s was not changed.", f) - } + if want := []string{filepath.Join(dir, "mnt", "m2", "project/a")}; !reflect.DeepEqual(changed, want) { + t.Errorf("got changed %v, want %v", changed, want) + } + + if want := []string{ + filepath.Join(dir, "mnt", "m2", "project/new"), + filepath.Join(dir, "mnt", "m2", "sub/d"), + }; !reflect.DeepEqual(added, want) { + t.Errorf("got added %v, want %v", added, want) } if dest, err := os.Readlink(filepath.Join(ws, "sub")); err != nil { t.Fatal(err) } else if want := filepath.Join(dir, "mnt", "m2", "sub"); dest != want { - t.Fatalf("got %q, want %q", dest, want) + t.Fatalf("Readlink(ws/sub): got %q, want %q", dest, want) } }
diff --git a/populate/populate.go b/populate/populate.go index e8b91a3..37cd82c 100644 --- a/populate/populate.go +++ b/populate/populate.go
@@ -159,16 +159,11 @@ // Returns the filenames (as relative paths) in newDir that have // changed relative to the files in oldDir. -func changedFiles(oldInfos map[string]*fileInfo, newInfos map[string]*fileInfo) ([]string, error) { - var changed []string +func changedFiles(oldInfos map[string]*fileInfo, newInfos map[string]*fileInfo) (added, changed []string, err error) { for path, info := range newInfos { old, ok := oldInfos[path] if !ok { - changed = append(changed, path) - continue - } - if info.isLink { - // TODO(hanwen): maybe we should we deref the link? + added = append(added, path) continue } @@ -182,16 +177,17 @@ } } sort.Strings(changed) - return changed, nil + sort.Strings(added) + return added, changed, nil } // Checkout updates a RW dir with new symlinks to the given RO dir. // Returns the files that should be touched. -func Checkout(ro, rw string) ([]string, error) { +func Checkout(ro, rw string) (added, changed []string, err error) { ro = filepath.Clean(ro) wsName, err := clearLinks(filepath.Dir(ro), rw) if err != nil { - return nil, err + return nil, nil, err } oldRoot := filepath.Join(filepath.Dir(ro), wsName) @@ -227,23 +223,27 @@ for i := 0; i < cap(errs); i++ { err := <-errs if err != nil { - return nil, err + return nil, nil, err } } if err := createLinks(roTree, rwTree, ro, rw); err != nil { - return nil, err + return nil, nil, err } newInfos := roTree.allFiles() - changed, err := changedFiles(oldInfos, newInfos) + added, changed, err = changedFiles(oldInfos, newInfos) if err != nil { - return nil, fmt.Errorf("changedFiles: %v", err) + return nil, nil, fmt.Errorf("changedFiles: %v", err) } for i, p := range changed { changed[i] = filepath.Join(ro, p) } - return changed, nil + for i, p := range added { + added[i] = filepath.Join(ro, p) + } + + return added, changed, nil }