diff --git a/Dockerfile b/Dockerfile index 86732d662..085e9b0c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ COPY --from=xx / / RUN apk add --no-cache \ docker \ file \ + findutils \ git \ make \ protoc \ diff --git a/go.mod b/go.mod index 9c3b7e057..992ad91f0 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/theupdateframework/notary v0.7.0 + github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 go.opentelemetry.io/otel v1.12.0 golang.org/x/sync v0.1.0 gopkg.in/yaml.v2 v2.4.0 @@ -92,7 +93,7 @@ require ( github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/patternmatcher v0.5.0 github.com/moby/spdystream v0.2.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect @@ -152,6 +153,8 @@ require ( require go.uber.org/goleak v1.1.12 +require github.com/fsnotify/fsevents v0.1.1 + replace ( // Override for e2e tests github.com/cucumber/godog => github.com/laurazard/godog v0.0.0-20220922095256-4c4b17abdae7 diff --git a/go.sum b/go.sum index aa0dc1521..a6e9d8a18 100644 --- a/go.sum +++ b/go.sum @@ -205,6 +205,8 @@ github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fsnotify/fsevents v0.1.1 h1:/125uxJvvoSDDBPen6yUZbil8J9ydKZnnl3TWWmvnkw= +github.com/fsnotify/fsevents v0.1.1/go.mod h1:+d+hS27T6k5J8CRaPLKFgwKYcpS7GwW3Ule9+SC2ZRc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -649,6 +651,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= +github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA= +github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tonistiigi/fsutil v0.0.0-20220930225714-4638ad635be5 h1:NJ1nZs4j4XcBJKIY5sAwTGp9w5b78Zxr3+r0zXRuKnA= github.com/tonistiigi/fsutil v0.0.0-20220930225714-4638ad635be5/go.mod h1:F83XRhNblQsKQH9hcKEE45GAOkL9590mtw9KsD0Q4fE= @@ -904,6 +908,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/watch/dockerignore.go b/pkg/watch/dockerignore.go new file mode 100644 index 000000000..6b29aaa5a --- /dev/null +++ b/pkg/watch/dockerignore.go @@ -0,0 +1,146 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package watch + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/moby/buildkit/frontend/dockerfile/dockerignore" + "github.com/moby/patternmatcher" +) + +type dockerPathMatcher struct { + repoRoot string + matcher *patternmatcher.PatternMatcher +} + +func (i dockerPathMatcher) Matches(f string) (bool, error) { + if !filepath.IsAbs(f) { + f = filepath.Join(i.repoRoot, f) + } + return i.matcher.MatchesOrParentMatches(f) +} + +func (i dockerPathMatcher) MatchesEntireDir(f string) (bool, error) { + matches, err := i.Matches(f) + if !matches || err != nil { + return matches, err + } + + // We match the dir, but we might exclude files underneath it. + if i.matcher.Exclusions() { + for _, pattern := range i.matcher.Patterns() { + if !pattern.Exclusion() { + continue + } + if IsChild(f, pattern.String()) { + // Found an exclusion match -- we don't match this whole dir + return false, nil + } + } + return true, nil + } + return true, nil +} + +func NewDockerIgnoreTester(repoRoot string) (*dockerPathMatcher, error) { + absRoot, err := filepath.Abs(repoRoot) + if err != nil { + return nil, err + } + + patterns, err := readDockerignorePatterns(absRoot) + if err != nil { + return nil, err + } + + return NewDockerPatternMatcher(absRoot, patterns) +} + +// Make all the patterns use absolute paths. +func absPatterns(absRoot string, patterns []string) []string { + absPatterns := make([]string, 0, len(patterns)) + for _, p := range patterns { + // The pattern parsing here is loosely adapted from fileutils' NewPatternMatcher + p = strings.TrimSpace(p) + if p == "" { + continue + } + p = filepath.Clean(p) + + pPath := p + isExclusion := false + if p[0] == '!' { + pPath = p[1:] + isExclusion = true + } + + if !filepath.IsAbs(pPath) { + pPath = filepath.Join(absRoot, pPath) + } + absPattern := pPath + if isExclusion { + absPattern = fmt.Sprintf("!%s", pPath) + } + absPatterns = append(absPatterns, absPattern) + } + return absPatterns +} + +func NewDockerPatternMatcher(repoRoot string, patterns []string) (*dockerPathMatcher, error) { + absRoot, err := filepath.Abs(repoRoot) + if err != nil { + return nil, err + } + + pm, err := patternmatcher.New(absPatterns(absRoot, patterns)) + if err != nil { + return nil, err + } + + return &dockerPathMatcher{ + repoRoot: absRoot, + matcher: pm, + }, nil +} + +func readDockerignorePatterns(repoRoot string) ([]string, error) { + var excludes []string + + f, err := os.Open(filepath.Join(repoRoot, ".dockerignore")) + switch { + case os.IsNotExist(err): + return excludes, nil + case err != nil: + return nil, err + } + defer func() { _ = f.Close() }() + + return dockerignore.ReadAll(f) +} + +func DockerIgnoreTesterFromContents(repoRoot string, contents string) (*dockerPathMatcher, error) { + patterns, err := dockerignore.ReadAll(strings.NewReader(contents)) + if err != nil { + return nil, err + } + + return NewDockerPatternMatcher(repoRoot, patterns) +} diff --git a/pkg/watch/notify.go b/pkg/watch/notify.go index 5d7da04ec..d6d3087d7 100644 --- a/pkg/watch/notify.go +++ b/pkg/watch/notify.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( @@ -8,8 +24,6 @@ import ( "runtime" "strconv" "strings" - - "github.com/tilt-dev/tilt/pkg/logger" ) var ( @@ -68,8 +82,8 @@ func (EmptyMatcher) MatchesEntireDir(f string) (bool, error) { return false, nil var _ PathMatcher = EmptyMatcher{} -func NewWatcher(paths []string, ignore PathMatcher, l logger.Logger) (Notify, error) { - return newWatcher(paths, ignore, l) +func NewWatcher(paths []string, ignore PathMatcher) (Notify, error) { + return newWatcher(paths, ignore) } const WindowsBufferSizeEnvVar = "TILT_WATCH_WINDOWS_BUFFER_SIZE" diff --git a/pkg/watch/notify_test.go b/pkg/watch/notify_test.go index 1834a1ae6..eed784631 100644 --- a/pkg/watch/notify_test.go +++ b/pkg/watch/notify_test.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( @@ -13,10 +29,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/tilt-dev/tilt/internal/dockerignore" - "github.com/tilt-dev/tilt/internal/testutils/tempdir" - "github.com/tilt-dev/tilt/pkg/logger" ) // Each implementation of the notify interface should have the same basic @@ -24,15 +36,18 @@ import ( func TestWindowsBufferSize(t *testing.T) { orig := os.Getenv(WindowsBufferSizeEnvVar) - defer os.Setenv(WindowsBufferSizeEnvVar, orig) + defer os.Setenv(WindowsBufferSizeEnvVar, orig) //nolint:errcheck - os.Setenv(WindowsBufferSizeEnvVar, "") + err := os.Setenv(WindowsBufferSizeEnvVar, "") + assert.Nil(t, err) assert.Equal(t, defaultBufferSize, DesiredWindowsBufferSize()) - os.Setenv(WindowsBufferSizeEnvVar, "a") + err = os.Setenv(WindowsBufferSizeEnvVar, "a") + assert.Nil(t, err) assert.Equal(t, defaultBufferSize, DesiredWindowsBufferSize()) - os.Setenv(WindowsBufferSizeEnvVar, "10") + err = os.Setenv(WindowsBufferSizeEnvVar, "10") + assert.Nil(t, err) assert.Equal(t, 10, DesiredWindowsBufferSize()) } @@ -71,7 +86,7 @@ func TestEventOrdering(t *testing.T) { for i, dir := range dirs { base := fmt.Sprintf("%d.txt", i) p := filepath.Join(dir, base) - err := os.WriteFile(p, []byte(base), os.FileMode(0777)) + err := os.WriteFile(p, []byte(base), os.FileMode(0o777)) if err != nil { t.Fatal(err) } @@ -174,7 +189,7 @@ func TestNewDirectoriesAreRecursivelyWatched(t *testing.T) { // change something inside sub directory changeFilePath := filepath.Join(subPath, "change") - file, err := os.OpenFile(changeFilePath, os.O_RDONLY|os.O_CREATE, 0666) + file, err := os.OpenFile(changeFilePath, os.O_RDONLY|os.O_CREATE, 0o666) if err != nil { t.Fatal(err) } @@ -236,7 +251,7 @@ func TestRemoveAndAddBack(t *testing.T) { path := filepath.Join(f.paths[0], "change") d1 := []byte("hello\ngo\n") - err := os.WriteFile(path, d1, 0644) + err := os.WriteFile(path, d1, 0o644) if err != nil { t.Fatal(err) } @@ -251,7 +266,7 @@ func TestRemoveAndAddBack(t *testing.T) { f.assertEvents(path) f.events = nil - err = os.WriteFile(path, d1, 0644) + err = os.WriteFile(path, d1, 0o644) if err != nil { t.Fatal(err) } @@ -272,7 +287,7 @@ func TestSingleFile(t *testing.T) { f.fsync() d2 := []byte("hello\nworld\n") - err := os.WriteFile(path, d2, 0644) + err := os.WriteFile(path, d2, 0o644) if err != nil { t.Fatal(err) } @@ -302,7 +317,7 @@ func TestWriteGoodLink(t *testing.T) { f := newNotifyFixture(t) goodFile := filepath.Join(f.paths[0], "goodFile") - err := os.WriteFile(goodFile, []byte("hello"), 0644) + err := os.WriteFile(goodFile, []byte("hello"), 0o644) if err != nil { t.Fatal(err) } @@ -387,7 +402,7 @@ func TestWatchNonexistentFileInNonexistentDirectoryCreatedSimultaneously(t *test f := newNotifyFixture(t) root := f.JoinPath("root") - err := os.Mkdir(root, 0777) + err := os.Mkdir(root, 0o777) if err != nil { t.Fatal(err) } @@ -404,7 +419,7 @@ func TestWatchNonexistentDirectory(t *testing.T) { f := newNotifyFixture(t) root := f.JoinPath("root") - err := os.Mkdir(root, 0777) + err := os.Mkdir(root, 0o777) if err != nil { t.Fatal(err) } @@ -415,12 +430,12 @@ func TestWatchNonexistentDirectory(t *testing.T) { f.fsync() f.events = nil - err = os.Mkdir(parent, 0777) + err = os.Mkdir(parent, 0o777) if err != nil { t.Fatal(err) } - // for directories that were the root of an Add, we don't report creation, cf. watcher_darwin.go + // for directories that were the root of an Add, we don't report creation, cf. watcher_fsevent.go f.assertEvents() f.events = nil @@ -433,7 +448,7 @@ func TestWatchNonexistentFileInNonexistentDirectory(t *testing.T) { f := newNotifyFixture(t) root := f.JoinPath("root") - err := os.Mkdir(root, 0777) + err := os.Mkdir(root, 0o777) if err != nil { t.Fatal(err) } @@ -443,7 +458,7 @@ func TestWatchNonexistentFileInNonexistentDirectory(t *testing.T) { f.watch(file) f.assertEvents() - err = os.Mkdir(parent, 0777) + err = os.Mkdir(parent, 0o777) if err != nil { t.Fatal(err) } @@ -474,7 +489,7 @@ func TestWatchCountInnerFileWithIgnore(t *testing.T) { f := newNotifyFixture(t) root := f.paths[0] - ignore, _ := dockerignore.NewDockerPatternMatcher(root, []string{ + ignore, _ := NewDockerPatternMatcher(root, []string{ "a", "!a/b", }) @@ -497,7 +512,7 @@ func TestIgnoreCreatedDir(t *testing.T) { f := newNotifyFixture(t) root := f.paths[0] - ignore, _ := dockerignore.NewDockerPatternMatcher(root, []string{"a/b"}) + ignore, _ := NewDockerPatternMatcher(root, []string{"a/b"}) f.setIgnore(ignore) a := f.JoinPath(root, "a") @@ -517,7 +532,7 @@ func TestIgnoreCreatedDirWithExclusions(t *testing.T) { f := newNotifyFixture(t) root := f.paths[0] - ignore, _ := dockerignore.NewDockerPatternMatcher(root, + ignore, _ := NewDockerPatternMatcher(root, []string{ "a/b", "c", @@ -542,7 +557,7 @@ func TestIgnoreInitialDir(t *testing.T) { f := newNotifyFixture(t) root := f.TempDir("root") - ignore, _ := dockerignore.NewDockerPatternMatcher(root, []string{"a/b"}) + ignore, _ := NewDockerPatternMatcher(root, []string{"a/b"}) f.setIgnore(ignore) a := f.JoinPath(root, "a") @@ -568,7 +583,7 @@ type notifyFixture struct { ctx context.Context cancel func() out *bytes.Buffer - *tempdir.TempDirFixture + *TempDirFixture notify Notify ignore PathMatcher paths []string @@ -581,7 +596,7 @@ func newNotifyFixture(t *testing.T) *notifyFixture { nf := ¬ifyFixture{ ctx: ctx, cancel: cancel, - TempDirFixture: tempdir.NewTempDirFixture(t), + TempDirFixture: NewTempDirFixture(t), paths: []string{}, ignore: EmptyMatcher{}, out: out, @@ -609,7 +624,7 @@ func (f *notifyFixture) rebuildWatcher() { } // create a new watcher - notify, err := NewWatcher(f.paths, f.ignore, logger.NewTestLogger(f.out)) + notify, err := NewWatcher(f.paths, f.ignore) if err != nil { f.T().Fatal(err) } @@ -674,7 +689,7 @@ func (f *notifyFixture) fsyncWithRetryCount(retryCount int) { syncPathBase := fmt.Sprintf("sync-%d.txt", time.Now().UnixNano()) syncPath := filepath.Join(f.paths[0], syncPathBase) anySyncPath := filepath.Join(f.paths[0], "sync-") - timeout := time.After(250 * time.Millisecond) + timeout := time.After(250 * time.Second) f.WriteFile(syncPath, time.Now().String()) diff --git a/pkg/watch/paths.go b/pkg/watch/paths.go index b2817cdc1..ef33a751a 100644 --- a/pkg/watch/paths.go +++ b/pkg/watch/paths.go @@ -1,13 +1,28 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" - - "github.com/tilt-dev/tilt/internal/ospath" ) func greatestExistingAncestor(path string) (string, error) { @@ -37,13 +52,13 @@ func dedupePathsForRecursiveWatcher(paths []string) []string { hasRemovals := false for i, existing := range result { - if ospath.IsChild(existing, current) { + if IsChild(existing, current) { // The path is already covered, so there's no need to include it isCovered = true break } - if ospath.IsChild(current, existing) { + if IsChild(current, existing) { // Mark the element empty fo removal. result[i] = "" hasRemovals = true @@ -67,3 +82,58 @@ func dedupePathsForRecursiveWatcher(paths []string) []string { } return result } + +func IsChild(dir string, file string) bool { + if dir == "" { + return false + } + + dir = filepath.Clean(dir) + current := filepath.Clean(file) + child := "." + for { + if strings.EqualFold(dir, current) { + // If the two paths are exactly equal, then they must be the same. + if dir == current { + return true + } + + // If the two paths are equal under case-folding, but not exactly equal, + // then the only way to check if they're truly "equal" is to check + // to see if we're on a case-insensitive file system. + // + // This is a notoriously tricky problem. See how dep solves it here: + // https://github.com/golang/dep/blob/v0.5.4/internal/fs/fs.go#L33 + // + // because you can mount case-sensitive filesystems onto case-insensitive + // file-systems, and vice versa :scream: + // + // We want to do as much of this check as possible with strings-only + // (to avoid a file system read and error handling), so we only + // do this check if we have no other choice. + dirInfo, err := os.Stat(dir) + if err != nil { + return false + } + + currentInfo, err := os.Stat(current) + if err != nil { + return false + } + + if !os.SameFile(dirInfo, currentInfo) { + return false + } + return true + } + + if len(current) <= len(dir) || current == "." { + return false + } + + cDir := filepath.Dir(current) + cBase := filepath.Base(current) + child = filepath.Join(cBase, child) + current = cDir + } +} diff --git a/pkg/watch/paths_test.go b/pkg/watch/paths_test.go index 49872f632..d906b7031 100644 --- a/pkg/watch/paths_test.go +++ b/pkg/watch/paths_test.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( @@ -5,12 +21,10 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/tilt-dev/tilt/internal/testutils/tempdir" ) func TestGreatestExistingAncestor(t *testing.T) { - f := tempdir.NewTempDirFixture(t) + f := NewTempDirFixture(t) p, err := greatestExistingAncestor(f.Path()) assert.NoError(t, err) diff --git a/pkg/watch/temp.go b/pkg/watch/temp.go index d6b094aaa..4e5c52535 100644 --- a/pkg/watch/temp.go +++ b/pkg/watch/temp.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( @@ -50,7 +66,7 @@ func (d *TempDir) NewDir(prefix string) (*TempDir, error) { func (d *TempDir) NewDeterministicDir(name string) (*TempDir, error) { d2 := filepath.Join(d.dir, name) - err := os.Mkdir(d2, 0700) + err := os.Mkdir(d2, 0o700) if os.IsExist(err) { return nil, err } else if err != nil { diff --git a/pkg/watch/temp_dir_fixture.go b/pkg/watch/temp_dir_fixture.go new file mode 100644 index 000000000..a0855e875 --- /dev/null +++ b/pkg/watch/temp_dir_fixture.go @@ -0,0 +1,199 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package watch + +import ( + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +type TempDirFixture struct { + t testing.TB + dir *TempDir + oldDir string +} + +// everything not listed in this character class will get replaced by _, so that it's a safe filename +var sanitizeForFilenameRe = regexp.MustCompile("[^a-zA-Z0-9.]") + +func SanitizeFileName(name string) string { + return sanitizeForFilenameRe.ReplaceAllString(name, "_") +} + +func NewTempDirFixture(t testing.TB) *TempDirFixture { + dir, err := NewDir(SanitizeFileName(t.Name())) + if err != nil { + t.Fatalf("Error making temp dir: %v", err) + } + + ret := &TempDirFixture{ + t: t, + dir: dir, + } + + t.Cleanup(ret.tearDown) + + return ret +} + +func (f *TempDirFixture) T() testing.TB { + return f.t +} + +func (f *TempDirFixture) Path() string { + return f.dir.Path() +} + +func (f *TempDirFixture) Chdir() { + cwd, err := os.Getwd() + if err != nil { + f.t.Fatal(err) + } + + f.oldDir = cwd + + err = os.Chdir(f.Path()) + if err != nil { + f.t.Fatal(err) + } +} + +func (f *TempDirFixture) JoinPath(path ...string) string { + p := []string{} + isAbs := len(path) > 0 && filepath.IsAbs(path[0]) + if isAbs { + if !strings.HasPrefix(path[0], f.Path()) { + f.t.Fatalf("Path outside fixture tempdir are forbidden: %s", path[0]) + } + } else { + p = append(p, f.Path()) + } + + p = append(p, path...) + return filepath.Join(p...) +} + +func (f *TempDirFixture) JoinPaths(paths []string) []string { + joined := make([]string, len(paths)) + for i, p := range paths { + joined[i] = f.JoinPath(p) + } + return joined +} + +// Returns the full path to the file written. +func (f *TempDirFixture) WriteFile(path string, contents string) string { + fullPath := f.JoinPath(path) + base := filepath.Dir(fullPath) + err := os.MkdirAll(base, os.FileMode(0o777)) + if err != nil { + f.t.Fatal(err) + } + err = os.WriteFile(fullPath, []byte(contents), os.FileMode(0o777)) + if err != nil { + f.t.Fatal(err) + } + return fullPath +} + +// Returns the full path to the file written. +func (f *TempDirFixture) CopyFile(originalPath, newPath string) { + contents, err := os.ReadFile(originalPath) + if err != nil { + f.t.Fatal(err) + } + f.WriteFile(newPath, string(contents)) +} + +// Read the file. +func (f *TempDirFixture) ReadFile(path string) string { + fullPath := f.JoinPath(path) + contents, err := os.ReadFile(fullPath) + if err != nil { + f.t.Fatal(err) + } + return string(contents) +} + +func (f *TempDirFixture) WriteSymlink(linkContents, destPath string) { + fullDestPath := f.JoinPath(destPath) + err := os.MkdirAll(filepath.Dir(fullDestPath), os.FileMode(0o777)) + if err != nil { + f.t.Fatal(err) + } + err = os.Symlink(linkContents, fullDestPath) + if err != nil { + f.t.Fatal(err) + } +} + +func (f *TempDirFixture) MkdirAll(path string) { + fullPath := f.JoinPath(path) + err := os.MkdirAll(fullPath, os.FileMode(0o777)) + if err != nil { + f.t.Fatal(err) + } +} + +func (f *TempDirFixture) TouchFiles(paths []string) { + for _, p := range paths { + f.WriteFile(p, "") + } +} + +func (f *TempDirFixture) Rm(pathInRepo string) { + fullPath := f.JoinPath(pathInRepo) + err := os.RemoveAll(fullPath) + if err != nil { + f.t.Fatal(err) + } +} + +func (f *TempDirFixture) NewFile(prefix string) (*os.File, error) { + return os.CreateTemp(f.dir.Path(), prefix) +} + +func (f *TempDirFixture) TempDir(prefix string) string { + name, err := os.MkdirTemp(f.dir.Path(), prefix) + if err != nil { + f.t.Fatal(err) + } + return name +} + +func (f *TempDirFixture) tearDown() { + if f.oldDir != "" { + err := os.Chdir(f.oldDir) + if err != nil { + f.t.Fatal(err) + } + } + + err := f.dir.TearDown() + if err != nil && runtime.GOOS == "windows" && + (strings.Contains(err.Error(), "The process cannot access the file") || + strings.Contains(err.Error(), "Access is denied")) { + // NOTE(nick): I'm not convinced that this is a real problem. + // I think it might just be clean up of file notification I/O. + } else if err != nil { + f.t.Fatal(err) + } +} diff --git a/pkg/watch/watcher_darwin.go b/pkg/watch/watcher_fsevent.go similarity index 68% rename from pkg/watch/watcher_darwin.go rename to pkg/watch/watcher_fsevent.go index 3c9b6393b..b4061dbf5 100644 --- a/pkg/watch/watcher_darwin.go +++ b/pkg/watch/watcher_fsevent.go @@ -1,19 +1,36 @@ +//go:build darwin +// +build darwin + +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import ( "path/filepath" "time" + "github.com/fsnotify/fsevents" "github.com/pkg/errors" - - "github.com/tilt-dev/tilt/pkg/logger" - - "github.com/tilt-dev/fsevents" + "github.com/sirupsen/logrus" ) // A file watcher optimized for Darwin. -// Uses FSEvents to avoid the terrible perf characteristics of kqueue. -type darwinNotify struct { +// Uses FSEvents to avoid the terrible perf characteristics of kqueue. Requires CGO +type fseventNotify struct { stream *fsevents.EventStream events chan FileEvent errors chan error @@ -21,11 +38,10 @@ type darwinNotify struct { pathsWereWatching map[string]interface{} ignore PathMatcher - logger logger.Logger sawAnyHistoryDone bool } -func (d *darwinNotify) loop() { +func (d *fseventNotify) loop() { for { select { case <-d.stop: @@ -58,7 +74,7 @@ func (d *darwinNotify) loop() { ignore, err := d.ignore.Matches(e.Path) if err != nil { - d.logger.Infof("Error matching path %q: %v", e.Path, err) + logrus.Infof("Error matching path %q: %v", e.Path, err) } else if ignore { continue } @@ -70,7 +86,7 @@ func (d *darwinNotify) loop() { } // Add a path to be watched. Should only be called during initialization. -func (d *darwinNotify) initAdd(name string) { +func (d *fseventNotify) initAdd(name string) { d.stream.Paths = append(d.stream.Paths, name) if d.pathsWereWatching == nil { @@ -79,7 +95,7 @@ func (d *darwinNotify) initAdd(name string) { d.pathsWereWatching[name] = struct{}{} } -func (d *darwinNotify) Start() error { +func (d *fseventNotify) Start() error { if len(d.stream.Paths) == 0 { return nil } @@ -93,7 +109,7 @@ func (d *darwinNotify) Start() error { return nil } -func (d *darwinNotify) Close() error { +func (d *fseventNotify) Close() error { numberOfWatches.Add(int64(-len(d.stream.Paths))) d.stream.Stop() @@ -103,18 +119,17 @@ func (d *darwinNotify) Close() error { return nil } -func (d *darwinNotify) Events() chan FileEvent { +func (d *fseventNotify) Events() chan FileEvent { return d.events } -func (d *darwinNotify) Errors() chan error { +func (d *fseventNotify) Errors() chan error { return d.errors } -func newWatcher(paths []string, ignore PathMatcher, l logger.Logger) (*darwinNotify, error) { - dw := &darwinNotify{ +func newFSEventWatcher(paths []string, ignore PathMatcher) (*fseventNotify, error) { + dw := &fseventNotify{ ignore: ignore, - logger: l, stream: &fsevents.EventStream{ Latency: 1 * time.Millisecond, Flags: fsevents.FileEvents, @@ -139,4 +154,4 @@ func newWatcher(paths []string, ignore PathMatcher, l logger.Logger) (*darwinNot return dw, nil } -var _ Notify = &darwinNotify{} +var _ Notify = &fseventNotify{} diff --git a/pkg/watch/watcher_naive.go b/pkg/watch/watcher_naive.go index 00ca01174..a81e96dff 100644 --- a/pkg/watch/watcher_naive.go +++ b/pkg/watch/watcher_naive.go @@ -1,5 +1,18 @@ -//go:build !darwin -// +build !darwin +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ package watch @@ -12,10 +25,9 @@ import ( "strings" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/tilt-dev/fsnotify" - "github.com/tilt-dev/tilt/internal/ospath" - "github.com/tilt-dev/tilt/pkg/logger" ) // A naive file watcher that uses the plain fsnotify API. @@ -33,7 +45,6 @@ type naiveNotify struct { notifyList map[string]bool ignore PathMatcher - log logger.Logger isWatcherRecursive bool watcher *fsnotify.Watcher @@ -71,7 +82,9 @@ func (d *naiveNotify) Start() error { // we should have caught that above, let's just skip it. if os.IsNotExist(err) { continue - } else if fi.IsDir() { + } + + if fi.IsDir() { err = d.watchRecursively(name) if err != nil { return errors.Wrapf(err, "notify.Add(%q)", name) @@ -141,7 +154,7 @@ func (d *naiveNotify) Errors() chan error { return d.errors } -func (d *naiveNotify) loop() { +func (d *naiveNotify) loop() { //nolint:gocyclo defer close(d.wrappedEvents) for e := range d.events { // The Windows fsnotify event stream sometimes gets events with empty names @@ -202,13 +215,13 @@ func (d *naiveNotify) loop() { if shouldWatch { err := d.add(path) if err != nil && !os.IsNotExist(err) { - d.log.Infof("Error watching path %s: %s", e.Name, err) + logrus.Infof("Error watching path %s: %s", e.Name, err) } } return nil }) if err != nil && !os.IsNotExist(err) { - d.log.Infof("Error walking directory %s: %s", e.Name, err) + logrus.Infof("Error walking directory %s: %s", e.Name, err) } } } @@ -216,7 +229,7 @@ func (d *naiveNotify) loop() { func (d *naiveNotify) shouldNotify(path string) bool { ignore, err := d.ignore.Matches(path) if err != nil { - d.log.Infof("Error matching path %q: %v", path, err) + logrus.Infof("Error matching path %q: %v", path, err) } else if ignore { return false } @@ -225,14 +238,11 @@ func (d *naiveNotify) shouldNotify(path string) bool { // We generally don't care when directories change at the root of an ADD stat, err := os.Lstat(path) isDir := err == nil && stat.IsDir() - if isDir { - return false - } - return true + return !isDir } for root := range d.notifyList { - if ospath.IsChild(root, path) { + if IsChild(root, path) { return true } } @@ -267,7 +277,7 @@ func (d *naiveNotify) shouldSkipDir(path string) (bool, error) { // - A parent of a directory that's in our notify list // (i.e., to cover the "path doesn't exist" case). for root := range d.notifyList { - if ospath.IsChild(root, path) || ospath.IsChild(path, root) { + if IsChild(root, path) || IsChild(path, root) { return false, nil } } @@ -284,7 +294,7 @@ func (d *naiveNotify) add(path string) error { return nil } -func newWatcher(paths []string, ignore PathMatcher, l logger.Logger) (*naiveNotify, error) { +func newWatcher(paths []string, ignore PathMatcher) (*naiveNotify, error) { if ignore == nil { return nil, fmt.Errorf("newWatcher: ignore is nil") } @@ -319,7 +329,6 @@ func newWatcher(paths []string, ignore PathMatcher, l logger.Logger) (*naiveNoti wmw := &naiveNotify{ notifyList: notifyList, ignore: ignore, - log: l, watcher: fsw, events: fsw.Events, wrappedEvents: wrappedEvents, diff --git a/pkg/watch/watcher_naive_test.go b/pkg/watch/watcher_naive_test.go index a54228efe..de0e80fe9 100644 --- a/pkg/watch/watcher_naive_test.go +++ b/pkg/watch/watcher_naive_test.go @@ -1,5 +1,18 @@ -//go:build !darwin -// +build !darwin +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ package watch @@ -39,7 +52,7 @@ func TestDontWatchEachFile(t *testing.T) { f.WriteFile(f.JoinPath(watched, "initial.txt"), "initial data") initialDir := f.JoinPath(watched, "initial_dir") - if err := os.Mkdir(initialDir, 0777); err != nil { + if err := os.Mkdir(initialDir, 0o777); err != nil { t.Fatal(err) } @@ -56,13 +69,13 @@ func TestDontWatchEachFile(t *testing.T) { // inplace inplace := f.JoinPath(watched, "inplace") - if err := os.Mkdir(inplace, 0777); err != nil { + if err := os.Mkdir(inplace, 0o777); err != nil { t.Fatal(err) } f.WriteFile(f.JoinPath(inplace, "inplace.txt"), "inplace data") inplaceDir := f.JoinPath(inplace, "inplace_dir") - if err := os.Mkdir(inplaceDir, 0777); err != nil { + if err := os.Mkdir(inplaceDir, 0o777); err != nil { t.Fatal(err) } @@ -81,7 +94,7 @@ func TestDontWatchEachFile(t *testing.T) { f.WriteFile(f.JoinPath(staged, "staged.txt"), "staged data") stagedDir := f.JoinPath(staged, "staged_dir") - if err := os.Mkdir(stagedDir, 0777); err != nil { + if err := os.Mkdir(stagedDir, 0o777); err != nil { t.Fatal(err) } @@ -109,10 +122,10 @@ func TestDontWatchEachFile(t *testing.T) { func inotifyNodes() (int, error) { pid := os.Getpid() - output, err := exec.Command("bash", "-c", fmt.Sprintf( + output, err := exec.Command("/bin/sh", "-c", fmt.Sprintf( "find /proc/%d/fd -lname anon_inode:inotify -printf '%%hinfo/%%f\n' | xargs cat | grep -c '^inotify'", pid)).Output() if err != nil { - return 0, fmt.Errorf("error running command to determine number of watched files: %v", err) + return 0, fmt.Errorf("error running command to determine number of watched files: %v\n %s", err, output) } n, err := strconv.Atoi(strings.TrimSpace(string(output))) diff --git a/pkg/watch/watcher_nonwin.go b/pkg/watch/watcher_nonwin.go index 021108664..360713318 100644 --- a/pkg/watch/watcher_nonwin.go +++ b/pkg/watch/watcher_nonwin.go @@ -1,6 +1,22 @@ //go:build !windows // +build !windows +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import "github.com/tilt-dev/fsnotify" diff --git a/pkg/watch/watcher_windows.go b/pkg/watch/watcher_windows.go index 6e05150c3..69af1ea5d 100644 --- a/pkg/watch/watcher_windows.go +++ b/pkg/watch/watcher_windows.go @@ -1,6 +1,22 @@ //go:build windows // +build windows +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package watch import (