Various performance improvements for the vcs backend

1) Fix a bug where watching a directory would also report the directory
as changed if any files inside it were changed. This would causes
excessive polling by the branch segment

2) Fix a bug where the watch for the repo dir in the branch segment
was being continually recreated, again causing excessive polling. This
was particularly noticeable with the patch to fix fugitive file status
updates.

3) Improve the performance of branch coloring for git repos by ignoring
change events for the index.lock file. This file is created/deleted
every time git access the index and would cause the entire working tree
status to be re-queried every time.

powerline/lib/file_watcher.py

@@ -84,14 +84,18 @@ class INotifyWatch(INotify):
 					eno = ctypes.get_errno()
 					if eno != errno.ENOTDIR:
 						self.handle_error()
-				# Try watching path as a file
-				flags |= (self.MODIFY | self.ATTRIB)
-				wd = self._add_watch(self._inotify_fd, buf, flags)
-				if wd == -1:
-					self.handle_error()
+					# Try watching path as a file
+					flags |= (self.MODIFY | self.ATTRIB)
+					wd = self._add_watch(self._inotify_fd, buf, flags)
+					if wd == -1:
+						self.handle_error()
 				self.watches[path] = wd
 				self.modified[path] = False
 
+	def is_watched(self, path):
+		with self.lock:
+			return realpath(path) in self.watches
+
 	def __call__(self, path):
 		''' Return True if path has been modified since the last call. Can
 		raise OSError if the path does not exist. '''
@@ -141,6 +145,10 @@ class StatWatch(object):
 		with self.lock:
 			self.watches.pop(path, None)
 
+	def is_watched(self, path):
+		with self.lock:
+			return realpath(path) in self.watches
+
 	def __call__(self, path):
 		path = realpath(path)
 		with self.lock:

powerline/lib/tree_watcher.py

@@ -31,11 +31,12 @@ class INotifyTreeWatcher(INotify):
 
 	is_dummy = False
 
-	def __init__(self, basedir):
+	def __init__(self, basedir, ignore_event=None):
 		super(INotifyTreeWatcher, self).__init__()
 		self.basedir = realpath(basedir)
 		self.watch_tree()
 		self.modified = True
+		self.ignore_event = (lambda path, name: False) if ignore_event is None else ignore_event
 
 	def watch_tree(self):
 		self.watched_dirs = {}
@@ -93,7 +94,7 @@ class INotifyTreeWatcher(INotify):
 
 				self.MODIFY | self.CREATE | self.DELETE |
 				self.MOVE_SELF | self.MOVED_FROM | self.MOVED_TO |
-				self.ATTRIB | self.MOVE_SELF | self.DELETE_SELF)
+				self.ATTRIB | self.DELETE_SELF)
 		if wd == -1:
 			eno = ctypes.get_errno()
 			if eno == errno.ENOTDIR:
@@ -112,7 +113,7 @@ class INotifyTreeWatcher(INotify):
 			return
 		path = self.watched_rmap.get(wd, None)
 		if path is not None:
-			self.modified = True
+			self.modified = not self.ignore_event(path, name)
 			if mask & self.CREATE:
 				# A new sub-directory might have been created, monitor it.
 				try:
@@ -152,10 +153,10 @@ class TreeWatcher(object):
 		self.last_query_times = {}
 		self.expire_time = expire_time * 60
 
-	def watch(self, path, logger=None):
+	def watch(self, path, logger=None, ignore_event=None):
 		path = realpath(path)
 		try:
-			w = INotifyTreeWatcher(path)
+			w = INotifyTreeWatcher(path, ignore_event=ignore_event)
 		except (INotifyError, DirTooLarge) as e:
 			if logger is not None:
 				logger.warn('Failed to watch path: {0} with error: {1}'.format(path, e))
@@ -176,14 +177,14 @@ class TreeWatcher(object):
 		for path in pop:
 			del self.last_query_times[path]
 
-	def __call__(self, path, logger=None):
+	def __call__(self, path, logger=None, ignore_event=None):
 		path = realpath(path)
 		self.expire_old_queries()
 		self.last_query_times[path] = monotonic()
 		w = self.watches.get(path, None)
 		if w is None:
 			try:
-				self.watch(path)
+				self.watch(path, logger=logger, ignore_event=ignore_event)
 			except NoSuchDir:
 				pass
 			return True

powerline/lib/vcs/__init__.py

@@ -39,8 +39,15 @@ def get_branch_name(directory, config_file, get_func):
 	global branch_name_cache
 	with branch_lock:
 		# Check if the repo directory was moved/deleted
+		# We cannot use the file_watcher for this as the inotify based file
+		# watcher will mark a directory as changed if any files inside it have
+		# changed, this is a big performance hit in vim, which continuously
+		# changes files inside the repo dir (backup/swap files, for instance).
+		# Check if the repo directory was moved/deleted
+		fw = file_watcher()
+		is_watched = fw.is_watched(directory)
 		try:
-			changed = file_watcher()(directory)
+			changed = fw(directory)
 		except OSError as e:
 			if getattr(e, 'errno', None) != errno.ENOENT:
 				raise
@@ -48,12 +55,13 @@ def get_branch_name(directory, config_file, get_func):
 		if changed:
 			branch_name_cache.pop(config_file, None)
 			# Remove the watches for this repo
-			file_watcher().unwatch(directory)
-			file_watcher().unwatch(config_file)
+			if is_watched:
+				fw.unwatch(directory)
+				fw.unwatch(config_file)
 		else:
 			# Check if the config file has changed
 			try:
-				changed = file_watcher()(config_file)
+				changed = fw(config_file)
 			except OSError as e:
 				if getattr(e, 'errno', None) != errno.ENOENT:
 					raise
@@ -176,7 +184,7 @@ class TreeStatusCache(dict):
 	def __call__(self, repo, logger):
 		key = repo.directory
 		try:
-			if self.tw(key):
+			if self.tw(key, logger=logger, ignore_event=getattr(repo, 'ignore_event', None)):
 				self.pop(key, None)
 		except OSError as e:
 			logger.warn('Failed to check %s for changes, with error: %s'% key, e)
@@ -209,7 +217,7 @@ def debug():
 	''' To use run python -c "from powerline.lib.vcs import debug; debug()" some_file_to_watch '''
 	import sys
 	dest = sys.argv[-1]
-	repo = guess(dest)
+	repo = guess(os.path.abspath(dest))
 	if repo is None:
 		print ('%s is not a recognized vcs repo' % dest)
 		raise SystemExit(1)
@@ -224,3 +232,5 @@ def debug():
 			raw_input('Press Enter to check again: ')
 	except KeyboardInterrupt:
 		pass
+	except EOFError:
+		pass

powerline/lib/vcs/git.py

@@ -41,18 +41,24 @@ def do_status(directory, path, func):
 		# for some reason I cannot be bothered to figure out.
 		return get_file_status(
 			directory, os.path.join(gitd, 'index'),
-			path, '.gitignore', func, extra_ignore_files=tuple(os.path.join(gitd, x) for x in ('HEAD', 'info/exclude')))
+			path, '.gitignore', func, extra_ignore_files=tuple(os.path.join(gitd, x) for x in ('logs/HEAD', 'info/exclude')))
 	return func(directory, path)
 
+def ignore_event(path, name):
+	# Ignore changes to the index.lock file, since they happen frequently and
+	# dont indicate an actual change in the working tree status
+	return False
+	return path.endswith('.git') and name == 'index.lock'
 
 try:
 	import pygit2 as git
 
 	class Repository(object):
-		__slots__ = ('directory')
+		__slots__ = ('directory', 'ignore_event')
 
 		def __init__(self, directory):
 			self.directory = os.path.abspath(directory)
+			self.ignore_event = ignore_event
 
 		def do_status(self, directory, path):
 			if path:
@@ -135,10 +141,11 @@ except ImportError:
 				yield line[:-1].decode('utf-8')
 
 	class Repository(object):
-		__slots__ = ('directory',)
+		__slots__ = ('directory', 'ignore_event')
 
 		def __init__(self, directory):
 			self.directory = os.path.abspath(directory)
+			self.ignore_event = ignore_event
 
 		def _gitcmd(self, directory, *args):
 			return readlines(('git',) + args, directory)