From da1ca578b5c687480b4510fd9b81056d8c1fead0 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 24 Feb 2023 11:10:15 -0500 Subject: [PATCH] watch: ignore ephemeral files & minor output tweaks Big change here is to import the ephemeral ignore set from Tilt. The `.git` directory is also ignored for now: this restriction should probably be lifted and made configurable in the future, but it's not generally important to watch and triggers a LOT of events (e.g. Git creates `index.lock` files that will appear and disappear rapidly as terminals/IDEs/etc interact with Git, even for read-only operations). The Tilt-provided ephemeral file set has been slowly devised over time based on temporary files that can cause trouble. We can also look at a more robust/configurable solution here in the future, but thse provide a reasonable out-of-the-box configuration for the moment. There's also some small tweaks to the output to add missing newlines in a few edge cases and such. Signed-off-by: Milas Bowman --- pkg/compose/watch.go | 32 +++++++++++++++++++++++--------- pkg/watch/ephemeral.go | 27 +++++++++++++++++++-------- pkg/watch/notify.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/watch/watcher_darwin.go | 3 +-- pkg/watch/watcher_naive.go | 2 ++ 5 files changed, 81 insertions(+), 19 deletions(-) diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 8b53583d2..6b0588c1a 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -22,14 +22,15 @@ import ( "time" "github.com/compose-spec/compose-go/types" - "github.com/docker/compose/v2/pkg/api" - "github.com/docker/compose/v2/pkg/utils" - "github.com/docker/compose/v2/pkg/watch" "github.com/jonboulle/clockwork" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + + "github.com/docker/compose/v2/pkg/api" + "github.com/docker/compose/v2/pkg/utils" + "github.com/docker/compose/v2/pkg/watch" ) type DevelopmentConfig struct { @@ -82,11 +83,24 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv } bc := service.Build.Context - ignore, err := watch.LoadDockerIgnore(bc) + dockerIgnores, err := watch.LoadDockerIgnore(bc) if err != nil { return err } + // add a hardcoded set of ignores on top of what came from .dockerignore + // some of this should likely be configurable (e.g. there could be cases + // where you want `.git` to be synced) but this is suitable for now + dotGitIgnore, err := watch.NewDockerPatternMatcher("/", []string{".git/"}) + if err != nil { + return err + } + ignore := watch.NewCompositeMatcher( + dockerIgnores, + watch.EphemeralPathMatcher, + dotGitIgnore, + ) + watcher, err := watch.NewWatcher([]string{bc}, ignore) if err != nil { return err @@ -109,7 +123,7 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv path := event.Path() for _, trigger := range config.Watch { - logrus.Debugf("change deteced on %s - comparing with %s", path, trigger.Path) + logrus.Debugf("change detected on %s - comparing with %s", path, trigger.Path) if watch.IsChild(trigger.Path, path) { fmt.Fprintf(s.stderr(), "change detected on %s\n", path) @@ -126,7 +140,7 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv Destination: fmt.Sprintf("%s:%s", name, dest), } case WatchActionRebuild: - logrus.Debugf("modified file %s require image to be rebuilt", path) + logrus.Debugf("modified file %s requires image to be rebuilt", path) needRebuild <- name default: return fmt.Errorf("watch action %q is not supported", trigger) @@ -176,7 +190,7 @@ func (s *composeService) makeRebuildFn(ctx context.Context, project *types.Proje Services: services, }) if err != nil { - fmt.Fprintf(s.stderr(), "Build failed") + fmt.Fprintf(s.stderr(), "Build failed\n") } for i, service := range project.Services { if id, ok := imageIds[service.Name]; ok { @@ -196,7 +210,7 @@ func (s *composeService) makeRebuildFn(ctx context.Context, project *types.Proje }, }) if err != nil { - fmt.Fprintf(s.stderr(), "Application failed to start after update") + fmt.Fprintf(s.stderr(), "Application failed to start after update\n") } } } @@ -212,7 +226,7 @@ func (s *composeService) makeSyncFn(ctx context.Context, project *types.Project, if err != nil { return err } - fmt.Fprintf(s.stderr(), "%s updated\n", opt.Source) + fmt.Fprintf(s.stderr(), "%s updated\n", opt.Destination) } } } diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index cc000a81e..2aa271503 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -1,9 +1,20 @@ -package ignore +/* + Copyright 2020 Docker Compose CLI authors -import ( - "github.com/tilt-dev/tilt/internal/dockerignore" - "github.com/tilt-dev/tilt/pkg/model" -) + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package watch // EphemeralPathMatcher filters out spurious changes that we don't want to // rebuild on, like IDE temp/lock files. @@ -16,7 +27,7 @@ import ( // https://app.clubhouse.io/windmill/story/691/filter-out-ephemeral-file-changes var EphemeralPathMatcher = initEphemeralPathMatcher() -func initEphemeralPathMatcher() model.PathMatcher { +func initEphemeralPathMatcher() PathMatcher { golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} emacsPatterns := []string{"**/.#*", "**/#*#"} // if .swp is taken (presumably because multiple vims are running in that dir), @@ -34,14 +45,14 @@ func initEphemeralPathMatcher() model.PathMatcher { // https://github.com/golang/go/blob/0b5218cf4e3e5c17344ea113af346e8e0836f6c4/src/cmd/go/internal/work/exec.go#L1764 goPatterns := []string{"**/*-go-tmp-umask"} - allPatterns := []string{} + var allPatterns []string allPatterns = append(allPatterns, golandPatterns...) allPatterns = append(allPatterns, emacsPatterns...) allPatterns = append(allPatterns, vimPatterns...) allPatterns = append(allPatterns, katePatterns...) allPatterns = append(allPatterns, goPatterns...) - matcher, err := dockerignore.NewDockerPatternMatcher("/", allPatterns) + matcher, err := NewDockerPatternMatcher("/", allPatterns) if err != nil { panic(err) } diff --git a/pkg/watch/notify.go b/pkg/watch/notify.go index f09ac98f7..3a6398a0e 100644 --- a/pkg/watch/notify.go +++ b/pkg/watch/notify.go @@ -106,3 +106,39 @@ func DesiredWindowsBufferSize() int { func IsWindowsShortReadError(err error) bool { return runtime.GOOS == "windows" && !errors.Is(err, fsnotify.ErrEventOverflow) } + +type CompositePathMatcher struct { + Matchers []PathMatcher +} + +func NewCompositeMatcher(matchers ...PathMatcher) PathMatcher { + if len(matchers) == 0 { + return EmptyMatcher{} + } + return CompositePathMatcher{Matchers: matchers} +} + +func (c CompositePathMatcher) Matches(f string) (bool, error) { + for _, t := range c.Matchers { + ret, err := t.Matches(f) + if err != nil { + return false, err + } + if ret { + return true, nil + } + } + return false, nil +} + +func (c CompositePathMatcher) MatchesEntireDir(f string) (bool, error) { + for _, t := range c.Matchers { + matches, err := t.MatchesEntireDir(f) + if matches || err != nil { + return matches, err + } + } + return false, nil +} + +var _ PathMatcher = CompositePathMatcher{} diff --git a/pkg/watch/watcher_darwin.go b/pkg/watch/watcher_darwin.go index 9329a294b..558134555 100644 --- a/pkg/watch/watcher_darwin.go +++ b/pkg/watch/watcher_darwin.go @@ -20,7 +20,6 @@ package watch import ( - "fmt" "path/filepath" "time" @@ -53,7 +52,6 @@ func (d *fseventNotify) loop() { } for _, e := range events { - fmt.Println(e) e.Path = filepath.Join("/", e.Path) _, isPathWereWatching := d.pathsWereWatching[e.Path] @@ -67,6 +65,7 @@ func (d *fseventNotify) loop() { if err != nil { logrus.Infof("Error matching path %q: %v", e.Path, err) } else if ignore { + logrus.Tracef("Ignoring event for path: %v", e.Path) continue } diff --git a/pkg/watch/watcher_naive.go b/pkg/watch/watcher_naive.go index 60056df1a..e8c7e855f 100644 --- a/pkg/watch/watcher_naive.go +++ b/pkg/watch/watcher_naive.go @@ -129,6 +129,7 @@ func (d *naiveNotify) watchRecursively(dir string) error { } if shouldSkipDir { + logrus.Debugf("Ignoring directory and its contents (recursively): %s", path) return filepath.SkipDir } @@ -234,6 +235,7 @@ func (d *naiveNotify) shouldNotify(path string) bool { if err != nil { logrus.Infof("Error matching path %q: %v", path, err) } else if ignore { + logrus.Tracef("Ignoring event for path: %v", path) return false }