From 14bec211aae63b9f878680a12f2b434af97ad023 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 14 Jun 2013 13:38:24 +0530 Subject: [PATCH] Various performance improvements for the vcs backend 1) Fix a bug where watching a directory would also report the directory as changed if any files inside it were changed. This would causes excessive polling by the branch segment 2) Fix a bug where the watch for the repo dir in the branch segment was being continually recreated, again causing excessive polling. This was particularly noticeable with the patch to fix fugitive file status updates. 3) Improve the performance of branch coloring for git repos by ignoring change events for the index.lock file. This file is created/deleted every time git access the index and would cause the entire working tree status to be re-queried every time. --- powerline/lib/file_watcher.py | 18 +++++++++++++----- powerline/lib/tree_watcher.py | 15 ++++++++------- powerline/lib/vcs/__init__.py | 22 ++++++++++++++++------ powerline/lib/vcs/git.py | 13 ++++++++++--- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/powerline/lib/file_watcher.py b/powerline/lib/file_watcher.py index 0750a5fa..bf78b7bb 100644 --- a/powerline/lib/file_watcher.py +++ b/powerline/lib/file_watcher.py @@ -84,14 +84,18 @@ class INotifyWatch(INotify): eno = ctypes.get_errno() if eno != errno.ENOTDIR: self.handle_error() - # Try watching path as a file - flags |= (self.MODIFY | self.ATTRIB) - wd = self._add_watch(self._inotify_fd, buf, flags) - if wd == -1: - self.handle_error() + # Try watching path as a file + flags |= (self.MODIFY | self.ATTRIB) + wd = self._add_watch(self._inotify_fd, buf, flags) + if wd == -1: + self.handle_error() self.watches[path] = wd self.modified[path] = False + def is_watched(self, path): + with self.lock: + return realpath(path) in self.watches + def __call__(self, path): ''' Return True if path has been modified since the last call. Can raise OSError if the path does not exist. ''' @@ -141,6 +145,10 @@ class StatWatch(object): with self.lock: self.watches.pop(path, None) + def is_watched(self, path): + with self.lock: + return realpath(path) in self.watches + def __call__(self, path): path = realpath(path) with self.lock: diff --git a/powerline/lib/tree_watcher.py b/powerline/lib/tree_watcher.py index b2f11fb7..77389f61 100644 --- a/powerline/lib/tree_watcher.py +++ b/powerline/lib/tree_watcher.py @@ -31,11 +31,12 @@ class INotifyTreeWatcher(INotify): is_dummy = False - def __init__(self, basedir): + def __init__(self, basedir, ignore_event=None): super(INotifyTreeWatcher, self).__init__() self.basedir = realpath(basedir) self.watch_tree() self.modified = True + self.ignore_event = (lambda path, name: False) if ignore_event is None else ignore_event def watch_tree(self): self.watched_dirs = {} @@ -93,7 +94,7 @@ class INotifyTreeWatcher(INotify): self.MODIFY | self.CREATE | self.DELETE | self.MOVE_SELF | self.MOVED_FROM | self.MOVED_TO | - self.ATTRIB | self.MOVE_SELF | self.DELETE_SELF) + self.ATTRIB | self.DELETE_SELF) if wd == -1: eno = ctypes.get_errno() if eno == errno.ENOTDIR: @@ -112,7 +113,7 @@ class INotifyTreeWatcher(INotify): return path = self.watched_rmap.get(wd, None) if path is not None: - self.modified = True + self.modified = not self.ignore_event(path, name) if mask & self.CREATE: # A new sub-directory might have been created, monitor it. try: @@ -152,10 +153,10 @@ class TreeWatcher(object): self.last_query_times = {} self.expire_time = expire_time * 60 - def watch(self, path, logger=None): + def watch(self, path, logger=None, ignore_event=None): path = realpath(path) try: - w = INotifyTreeWatcher(path) + w = INotifyTreeWatcher(path, ignore_event=ignore_event) except (INotifyError, DirTooLarge) as e: if logger is not None: logger.warn('Failed to watch path: {0} with error: {1}'.format(path, e)) @@ -176,14 +177,14 @@ class TreeWatcher(object): for path in pop: del self.last_query_times[path] - def __call__(self, path, logger=None): + def __call__(self, path, logger=None, ignore_event=None): path = realpath(path) self.expire_old_queries() self.last_query_times[path] = monotonic() w = self.watches.get(path, None) if w is None: try: - self.watch(path) + self.watch(path, logger=logger, ignore_event=ignore_event) except NoSuchDir: pass return True diff --git a/powerline/lib/vcs/__init__.py b/powerline/lib/vcs/__init__.py index e5e07f8d..848543a7 100644 --- a/powerline/lib/vcs/__init__.py +++ b/powerline/lib/vcs/__init__.py @@ -39,8 +39,15 @@ def get_branch_name(directory, config_file, get_func): global branch_name_cache with branch_lock: # Check if the repo directory was moved/deleted + # We cannot use the file_watcher for this as the inotify based file + # watcher will mark a directory as changed if any files inside it have + # changed, this is a big performance hit in vim, which continuously + # changes files inside the repo dir (backup/swap files, for instance). + # Check if the repo directory was moved/deleted + fw = file_watcher() + is_watched = fw.is_watched(directory) try: - changed = file_watcher()(directory) + changed = fw(directory) except OSError as e: if getattr(e, 'errno', None) != errno.ENOENT: raise @@ -48,12 +55,13 @@ def get_branch_name(directory, config_file, get_func): if changed: branch_name_cache.pop(config_file, None) # Remove the watches for this repo - file_watcher().unwatch(directory) - file_watcher().unwatch(config_file) + if is_watched: + fw.unwatch(directory) + fw.unwatch(config_file) else: # Check if the config file has changed try: - changed = file_watcher()(config_file) + changed = fw(config_file) except OSError as e: if getattr(e, 'errno', None) != errno.ENOENT: raise @@ -176,7 +184,7 @@ class TreeStatusCache(dict): def __call__(self, repo, logger): key = repo.directory try: - if self.tw(key): + if self.tw(key, logger=logger, ignore_event=getattr(repo, 'ignore_event', None)): self.pop(key, None) except OSError as e: logger.warn('Failed to check %s for changes, with error: %s'% key, e) @@ -209,7 +217,7 @@ def debug(): ''' To use run python -c "from powerline.lib.vcs import debug; debug()" some_file_to_watch ''' import sys dest = sys.argv[-1] - repo = guess(dest) + repo = guess(os.path.abspath(dest)) if repo is None: print ('%s is not a recognized vcs repo' % dest) raise SystemExit(1) @@ -224,3 +232,5 @@ def debug(): raw_input('Press Enter to check again: ') except KeyboardInterrupt: pass + except EOFError: + pass diff --git a/powerline/lib/vcs/git.py b/powerline/lib/vcs/git.py index 3c32db8f..09c029c6 100644 --- a/powerline/lib/vcs/git.py +++ b/powerline/lib/vcs/git.py @@ -41,18 +41,24 @@ def do_status(directory, path, func): # for some reason I cannot be bothered to figure out. return get_file_status( directory, os.path.join(gitd, 'index'), - path, '.gitignore', func, extra_ignore_files=tuple(os.path.join(gitd, x) for x in ('HEAD', 'info/exclude'))) + path, '.gitignore', func, extra_ignore_files=tuple(os.path.join(gitd, x) for x in ('logs/HEAD', 'info/exclude'))) return func(directory, path) +def ignore_event(path, name): + # Ignore changes to the index.lock file, since they happen frequently and + # dont indicate an actual change in the working tree status + return False + return path.endswith('.git') and name == 'index.lock' try: import pygit2 as git class Repository(object): - __slots__ = ('directory') + __slots__ = ('directory', 'ignore_event') def __init__(self, directory): self.directory = os.path.abspath(directory) + self.ignore_event = ignore_event def do_status(self, directory, path): if path: @@ -135,10 +141,11 @@ except ImportError: yield line[:-1].decode('utf-8') class Repository(object): - __slots__ = ('directory',) + __slots__ = ('directory', 'ignore_event') def __init__(self, directory): self.directory = os.path.abspath(directory) + self.ignore_event = ignore_event def _gitcmd(self, directory, *args): return readlines(('git',) + args, directory)