Merge remote-tracking branch 'zyx-i/more-threaded-globals' into develop

.local.vimrc

@@ -1,2 +1,2 @@
 setlocal noexpandtab
-let g:syntastic_python_flake8_args = '--ignore=W191,E501,E121,E122,E123,E128'
+let g:syntastic_python_flake8_args = '--ignore=W191,E501,E121,E122,E123,E128,E225,W291'

docs/source/configuration.rst

@@ -166,6 +166,16 @@ Common configuration is a subdictionary that is a value of ``common`` key in
     String, determines format of the log messages. Defaults to 
     ``'%(asctime)s:%(level)s:%(message)s'``.
 
+``interval``
+    Number, determines time (in seconds) between checks for changed 
+    configuration. Checks are done in a seprate thread. Use ``null`` to check 
+    for configuration changes on ``.render()`` call in main thread.
+    Defaults to ``None``.
+
+``reload_config``
+    Boolean, determines whether configuration should be reloaded at all. 
+    Defaults to ``True``.
+
 Extension-specific configuration
 --------------------------------
 

docs/source/overview.rst

@@ -100,8 +100,14 @@ absolute path to your Powerline installation directory:
 
 If you're using Vundle or Pathogen and don't want Powerline functionality in 
 any other applications, simply add Powerline as a bundle and point the path 
-above to the Powerline bundle directory, e.g.  
+above to the Powerline bundle directory, e.g. 
-``~/.vim/bundle/powerline/powerline/bindings/vim``.
+``~/.vim/bundle/powerline/powerline/bindings/vim``. For vim-addon-manager it is 
+even easier since you don’t need to write this big path or install anything by 
+hand: ``powerline`` is installed and run just like any other plugin using
+
+.. code-block:: vim
+
+    call vam#ActivateAddons(['powerline'])
 
 Shell prompts
 -------------

powerline/__init__.py

@@ -1,64 +1,18 @@
 # vim:fileencoding=utf-8:noet
 
 from __future__ import absolute_import
-import json
 import os
 import sys
 import logging
 
 from powerline.colorscheme import Colorscheme
-from powerline.lib.file_watcher import create_file_watcher
+from powerline.lib.config import ConfigLoader
 
-from threading import Lock, Thread, Event
+from threading import Lock, Event
-from collections import defaultdict
 
 
 DEFAULT_SYSTEM_CONFIG_DIR = None
 
-watcher = None
-
-
-class MultiClientWatcher(object):
-	subscribers = set()
-	received_events = {}
-
-	def __init__(self):
-		global watcher
-		self.subscribers.add(self)
-		if not watcher:
-			watcher = create_file_watcher()
-
-	def watch(self, file):
-		watcher.watch(file)
-
-	def __call__(self, file):
-		if self not in self.subscribers:
-			return False
-
-		if file in self.received_events and self not in self.received_events[file]:
-			self.received_events[file].add(self)
-			if self.received_events[file] >= self.subscribers:
-				self.received_events.pop(file)
-			return True
-
-		if watcher(file):
-			self.received_events[file] = set([self])
-			return True
-
-		return False
-
-	def unsubscribe(self):
-		try:
-			self.subscribers.remove(self)
-		except KeyError:
-			pass
-
-	__del__ = unsubscribe
-
-
-def open_file(path):
-	return open(path, 'r')
-
 
 def find_config_file(search_paths, config_file):
 	config_file += '.json'
@@ -69,11 +23,6 @@ def find_config_file(search_paths, config_file):
 	raise IOError('Config file not found in search path: {0}'.format(config_file))
 
 
-def load_json_config(config_file_path, load=json.load, open_file=open_file):
-	with open_file(config_file_path) as config_file_fp:
-		return load(config_file_fp)
-
-
 class PowerlineState(object):
 	def __init__(self, use_daemon_threads, logger, ext):
 		self.logger = logger
@@ -85,7 +34,9 @@ class PowerlineState(object):
 	def _log(self, attr, msg, *args, **kwargs):
 		prefix = kwargs.get('prefix') or self.prefix
 		prefix = self.ext + ((':' + prefix) if prefix else '')
-		msg = prefix + ':' + msg.format(*args, **kwargs)
+		if args or kwargs:
+			msg = msg.format(*args, **kwargs)
+		msg = prefix + ':' + msg
 		key = attr + ':' + prefix
 		if msg != self.last_msgs.get(key):
 			getattr(self.logger, attr)(msg)
@@ -132,9 +83,12 @@ class Powerline(object):
 		during python session.
 	:param Logger logger:
 		If present, no new logger will be created and this logger will be used.
-	:param float interval:
+	:param bool use_daemon_threads:
-		When reloading configuration wait for this amount of seconds. Set it to 
+		Use daemon threads for.
-		None if you don’t want to reload configuration automatically.
+	:param Event shutdown_event:
+		Use this Event as shutdown_event.
+	:param ConfigLoader config_loader:
+		Class that manages (re)loading of configuration.
 	'''
 
 	def __init__(self,
@@ -143,41 +97,39 @@ class Powerline(object):
 				run_once=False,
 				logger=None,
 				use_daemon_threads=True,
-				interval=10,
+				shutdown_event=None,
-				watcher=None):
+				config_loader=None):
 		self.ext = ext
 		self.renderer_module = renderer_module or ext
 		self.run_once = run_once
 		self.logger = logger
 		self.use_daemon_threads = use_daemon_threads
-		self.interval = interval
 
 		if '.' not in self.renderer_module:
 			self.renderer_module = 'powerline.renderers.' + self.renderer_module
 		elif self.renderer_module[-1] == '.':
 			self.renderer_module = self.renderer_module[:-1]
 
-		self.config_paths = self.get_config_paths()
+		config_paths = self.get_config_paths()
+		self.find_config_file = lambda cfg_path: find_config_file(config_paths, cfg_path)
 
-		self.configs_lock = Lock()
 		self.cr_kwargs_lock = Lock()
-		self.create_renderer_kwargs = {}
+		self.create_renderer_kwargs = {
-		self.shutdown_event = Event()
+			'load_main': True,
-		self.configs = defaultdict(set)
+			'load_colors': True,
-		self.missing = defaultdict(set)
+			'load_colorscheme': True,
-
+			'load_theme': True,
-		self.thread = None
+		}
+		self.shutdown_event = shutdown_event or Event()
+		self.config_loader = config_loader or ConfigLoader(shutdown_event=self.shutdown_event)
+		self.run_loader_update = False
 
 		self.renderer_options = {}
 
-		self.watcher = watcher or MultiClientWatcher()
-
 		self.prev_common_config = None
 		self.prev_ext_config = None
 		self.pl = None
 
-		self.create_renderer(load_main=True, load_colors=True, load_colorscheme=True, load_theme=True)
-
 	def create_renderer(self, load_main=False, load_colors=False, load_colorscheme=False, load_theme=False):
 		'''(Re)create renderer object. Can be used after Powerline object was 
 		successfully initialized. If any of the below parameters except 
@@ -224,6 +176,8 @@ class Powerline(object):
 
 				if not self.pl:
 					self.pl = PowerlineState(self.use_daemon_threads, self.logger, self.ext)
+					if not self.config_loader.pl:
+						self.config_loader.pl = self.pl
 
 				self.renderer_options.update(
 					pl=self.pl,
@@ -235,9 +189,17 @@ class Powerline(object):
 						'ext': self.ext,
 						'common_config': self.common_config,
 						'run_once': self.run_once,
+						'shutdown_event': self.shutdown_event,
 					},
 				)
 
+				if not self.run_once and self.common_config.get('reload_config', True):
+					interval = self.common_config.get('interval', None)
+					self.config_loader.set_interval(interval)
+					self.run_loader_update = (interval is None)
+					if interval is not None and not self.config_loader.is_alive():
+						self.config_loader.start()
+
 			self.ext_config = config['ext'][self.ext]
 			if self.ext_config != self.prev_ext_config:
 				ext_config_differs = True
@@ -286,9 +248,6 @@ class Powerline(object):
 			else:
 				self.renderer = renderer
 
-		if not self.run_once and not self.is_alive() and self.interval is not None:
-			self.start()
-
 	def get_log_handler(self):
 		'''Get log handler.
 
@@ -324,24 +283,20 @@ class Powerline(object):
 		return config_paths
 
 	def _load_config(self, cfg_path, type):
-		'''Load configuration and setup watcher.'''
+		'''Load configuration and setup watches.'''
+		function = getattr(self, 'on_' + type + '_change')
 		try:
-			path = find_config_file(self.config_paths, cfg_path)
+			path = self.find_config_file(cfg_path)
 		except IOError:
-			with self.configs_lock:
+			self.config_loader.register_missing(self.find_config_file, function, cfg_path)
-				self.missing[type].add(cfg_path)
 			raise
-		with self.configs_lock:
+		self.config_loader.register(function, path)
-			self.configs[type].add(path)
+		return self.config_loader.load(path)
-			self.watcher.watch(path)
-		return load_json_config(path)
 
 	def _purge_configs(self, type):
-		try:
+		function = getattr(self, 'on_' + type + '_change')
-			with self.configs_lock:
+		self.config_loader.unregister_functions(set((function,)))
-				self.configs.pop(type)
+		self.config_loader.unregister_missing(set(((self.find_config_file, function),)))
-		except KeyError:
-			pass
 
 	def load_theme_config(self, name):
 		'''Get theme configuration.
@@ -393,63 +348,59 @@ class Powerline(object):
 		'''
 		return None
 
-	def render(self, *args, **kwargs):
+	def update_renderer(self):
-		'''Lock renderer from modifications and pass all arguments further to 
+		'''Updates/creates a renderer if needed.'''
-		``self.renderer.render()``.
+		if self.run_loader_update:
-		'''
+			self.config_loader.update()
+		create_renderer_kwargs = None
 		with self.cr_kwargs_lock:
 			if self.create_renderer_kwargs:
-				try:
+				create_renderer_kwargs = self.create_renderer_kwargs.copy()
-					self.create_renderer(**self.create_renderer_kwargs)
+		if create_renderer_kwargs:
-				except Exception as e:
+			try:
-					self.pl.exception('Failed to create renderer: {0}', str(e))
+				self.create_renderer(**create_renderer_kwargs)
-				finally:
+			except Exception as e:
-					self.create_renderer_kwargs.clear()
+				self.pl.exception('Failed to create renderer: {0}', str(e))
+			finally:
+				self.create_renderer_kwargs.clear()
+
+	def render(self, *args, **kwargs):
+		'''Update/create renderer if needed and pass all arguments further to 
+		``self.renderer.render()``.
+		'''
+		self.update_renderer()
 		return self.renderer.render(*args, **kwargs)
 
 	def shutdown(self):
-		'''Lock renderer from modifications and run its ``.shutdown()`` method.
+		'''Shut down all background threads. Must be run only prior to exiting 
+		current application.
 		'''
 		self.shutdown_event.set()
-		if self.use_daemon_threads and self.is_alive():
-			# Give the worker thread a chance to shutdown, but don't block for too long
-			self.thread.join(.01)
 		self.renderer.shutdown()
-		self.watcher.unsubscribe()
+		functions = (
+			self.on_main_change,
+			self.on_colors_change,
+			self.on_colorscheme_change,
+			self.on_theme_change,
+		)
+		self.config_loader.unregister_functions(set(functions))
+		self.config_loader.unregister_missing(set(((find_config_file, function) for function in functions)))
 
-	def is_alive(self):
+	def on_main_change(self, path):
-		return self.thread and self.thread.is_alive()
+		with self.cr_kwargs_lock:
+			self.create_renderer_kwargs['load_main'] = True
 
-	def start(self):
+	def on_colors_change(self, path):
-		self.thread = Thread(target=self.run)
+		with self.cr_kwargs_lock:
-		if self.use_daemon_threads:
+			self.create_renderer_kwargs['load_colors'] = True
-			self.thread.daemon = True
-		self.thread.start()
 
-	def run(self):
+	def on_colorscheme_change(self, path):
-		while not self.shutdown_event.is_set():
+		with self.cr_kwargs_lock:
-			kwargs = {}
+			self.create_renderer_kwargs['load_colorscheme'] = True
-			removes = []
+
-			with self.configs_lock:
+	def on_theme_change(self, path):
-				for type, paths in self.configs.items():
+		with self.cr_kwargs_lock:
-					for path in paths:
+			self.create_renderer_kwargs['load_theme'] = True
-						if self.watcher(path):
-							kwargs['load_' + type] = True
-				for type, cfg_paths in self.missing.items():
-					for cfg_path in cfg_paths:
-						try:
-							find_config_file(self.config_paths, cfg_path)
-						except IOError:
-							pass
-						else:
-							kwargs['load_' + type] = True
-							removes.append((type, cfg_path))
-				for type, cfg_path in removes:
-					self.missing[type].remove(cfg_path)
-			if kwargs:
-				with self.cr_kwargs_lock:
-					self.create_renderer_kwargs.update(kwargs)
-			self.shutdown_event.wait(self.interval)
 
 	def __enter__(self):
 		return self

powerline/bindings/zsh/powerline.zsh

@@ -14,14 +14,13 @@ _powerline_tmux_set_columns() {
 }
 
 _powerline_install_precmd() {
-	emulate -L zsh
+	emulate zsh
 	for f in "${precmd_functions[@]}"; do
 		if [[ "$f" = "_powerline_precmd" ]]; then
 			return
 		fi
 	done
 	chpwd_functions+=( _powerline_tmux_set_pwd )
-	setopt nolocaloptions
 	setopt promptpercent
 	setopt promptsubst
 	if zmodload zsh/zpython &>/dev/null ; then

powerline/lib/config.py

@@ -0,0 +1,156 @@
+# vim:fileencoding=utf-8:noet
+
+from powerline.lib.threaded import MultiRunnedThread
+from powerline.lib.file_watcher import create_file_watcher
+
+from threading import Event, Lock
+from collections import defaultdict
+
+import json
+
+
+def open_file(path):
+	return open(path, 'r')
+
+
+def load_json_config(config_file_path, load=json.load, open_file=open_file):
+	with open_file(config_file_path) as config_file_fp:
+		return load(config_file_fp)
+
+
+class ConfigLoader(MultiRunnedThread):
+	def __init__(self, shutdown_event=None, watcher=None, load=load_json_config):
+		super(ConfigLoader, self).__init__()
+		self.shutdown_event = shutdown_event or Event()
+		self.watcher = watcher or create_file_watcher()
+		self._load = load
+
+		self.pl = None
+		self.interval = None
+
+		self.lock = Lock()
+
+		self.watched = defaultdict(set)
+		self.missing = defaultdict(set)
+		self.loaded = {}
+
+	def set_pl(self, pl):
+		self.pl = pl
+
+	def set_interval(self, interval):
+		self.interval = interval
+
+	def register(self, function, path):
+		'''Register function that will be run when file changes.
+
+		:param function function:
+			Function that will be called when file at the given path changes.
+		:param str path:
+			Path that will be watched for.
+		'''
+		with self.lock:
+			self.watched[path].add(function)
+			self.watcher.watch(path)
+
+	def register_missing(self, condition_function, function, key):
+		'''Register any function that will be called with given key each 
+		interval seconds (interval is defined at __init__). Its result is then 
+		passed to ``function``, but only if the result is true.
+
+		:param function condition_function:
+			Function which will be called each ``interval`` seconds. All 
+			exceptions from it will be ignored.
+		:param function function:
+			Function which will be called if condition_function returns 
+			something that is true. Accepts result of condition_function as an 
+			argument.
+		:param str key:
+			Any value, it will be passed to condition_function on each call.
+
+		Note: registered functions will be automatically removed if 
+		condition_function results in something true.
+		'''
+		with self.lock:
+			self.missing[key].add((condition_function, function))
+
+	def unregister_functions(self, removed_functions):
+		'''Unregister files handled by these functions.
+
+		:param set removed_functions:
+			Set of functions previously passed to ``.register()`` method.
+		'''
+		with self.lock:
+			for path, functions in list(self.watched.items()):
+				functions -= removed_functions
+				if not functions:
+					self.watched.pop(path)
+					self.loaded.pop(path, None)
+
+	def unregister_missing(self, removed_functions):
+		'''Unregister files handled by these functions.
+
+		:param set removed_functions:
+			Set of pairs (2-tuples) representing ``(condition_function, 
+			function)`` function pairs previously passed as an arguments to 
+			``.register_missing()`` method.
+		'''
+		with self.lock:
+			for key, functions in list(self.missing.items()):
+				functions -= removed_functions
+				if not functions:
+					self.missing.pop(key)
+
+	def load(self, path):
+		try:
+			# No locks: GIL does what we need
+			return self.loaded[path]
+		except KeyError:
+			r = self._load(path)
+			self.loaded[path] = r
+			return r
+
+	def update(self):
+		toload = []
+		with self.lock:
+			for path, functions in self.watched.items():
+				for function in functions:
+					try:
+						modified = self.watcher(path)
+					except OSError as e:
+						modified = True
+						self.exception('Error while running watcher for path {0}: {1}', path, str(e))
+					else:
+						if modified:
+							toload.append(path)
+					if modified:
+						function(path)
+		with self.lock:
+			for key, functions in list(self.missing.items()):
+				for condition_function, function in list(functions):
+					try:
+						path = condition_function(key)
+					except Exception as e:
+						self.exception('Error while running condition function for key {0}: {1}', key, str(e))
+					else:
+						if path:
+							toload.append(path)
+							function(path)
+							functions.remove((condition_function, function))
+				if not functions:
+					self.missing.pop(key)
+		for path in toload:
+			try:
+				self.loaded[path] = self._load(path)
+			except Exception as e:
+				self.exception('Error while loading {0}: {1}', path, str(e))
+
+	def run(self):
+		while self.interval is not None and not self.shutdown_event.is_set():
+			self.update()
+			self.shutdown_event.wait(self.interval)
+
+	def exception(self, msg, *args, **kwargs):
+		if self.pl:
+			self.pl.exception(msg, prefix='config_loader', *args, **kwargs)
+		else:
+			raise

powerline/lib/file_watcher.py

@@ -1,4 +1,4 @@
-# vim:fileencoding=UTF-8:noet
+# vim:fileencoding=utf-8:noet
 from __future__ import unicode_literals, absolute_import
 
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
@@ -6,120 +6,23 @@ __docformat__ = 'restructuredtext en'
 
 import os
 import sys
-import errno
 from time import sleep
 from threading import RLock
 
 from powerline.lib.monotonic import monotonic
-
+from powerline.lib.inotify import INotify, INotifyError
-class INotifyError(Exception):
-	pass
 
 
-class INotifyWatch(object):
+class INotifyWatch(INotify):
-
 	is_stat_based = False
 
-	# See <sys/inotify.h> for the flags defined below
+	def __init__(self, expire_time=10):
-
+		super(INotifyWatch, self).__init__()
-	# Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH.
-	ACCESS = 0x00000001         # File was accessed.
-	MODIFY = 0x00000002         # File was modified.
-	ATTRIB = 0x00000004         # Metadata changed.
-	CLOSE_WRITE = 0x00000008    # Writtable file was closed.
-	CLOSE_NOWRITE = 0x00000010  # Unwrittable file closed.
-	OPEN = 0x00000020           # File was opened.
-	MOVED_FROM = 0x00000040     # File was moved from X.
-	MOVED_TO = 0x00000080       # File was moved to Y.
-	CREATE = 0x00000100         # Subfile was created.
-	DELETE = 0x00000200         # Subfile was deleted.
-	DELETE_SELF = 0x00000400    # Self was deleted.
-	MOVE_SELF = 0x00000800      # Self was moved.
-
-	# Events sent by the kernel.
-	UNMOUNT = 0x00002000     # Backing fs was unmounted.
-	Q_OVERFLOW = 0x00004000  # Event queued overflowed.
-	IGNORED = 0x00008000     # File was ignored.
-
-	# Helper events.
-	CLOSE = (CLOSE_WRITE | CLOSE_NOWRITE)  # Close.
-	MOVE = (MOVED_FROM | MOVED_TO)         # Moves.
-
-	# Special flags.
-	ONLYDIR = 0x01000000      # Only watch the path if it is a directory.
-	DONT_FOLLOW = 0x02000000  # Do not follow a sym link.
-	EXCL_UNLINK = 0x04000000  # Exclude events on unlinked objects.
-	MASK_ADD = 0x20000000     # Add to the mask of an already existing watch.
-	ISDIR = 0x40000000        # Event occurred against dir.
-	ONESHOT = 0x80000000      # Only send event once.
-
-	# All events which a program can wait on.
-	ALL_EVENTS = (ACCESS | MODIFY | ATTRIB | CLOSE_WRITE | CLOSE_NOWRITE |
-					OPEN | MOVED_FROM | MOVED_TO | CREATE | DELETE |
-					DELETE_SELF | MOVE_SELF)
-
-	# See <bits/inotify.h>
-	CLOEXEC = 0x80000
-	NONBLOCK = 0x800
-
-	def __init__(self, inotify_fd, add_watch, rm_watch, read, expire_time=10):
-		import ctypes
-		import struct
-		self._add_watch, self._rm_watch = add_watch, rm_watch
-		self._read = read
-		# We keep a reference to os to prevent it from being deleted
-		# during interpreter shutdown, which would lead to errors in the
-		# __del__ method
-		self.os = os
 		self.watches = {}
 		self.modified = {}
 		self.last_query = {}
-		self._buf = ctypes.create_string_buffer(5000)
-		self.fenc = sys.getfilesystemencoding() or 'utf-8'
-		self.hdr = struct.Struct(b'iIII')
-		if self.fenc == 'ascii':
-			self.fenc = 'utf-8'
 		self.lock = RLock()
 		self.expire_time = expire_time * 60
-		self._inotify_fd = inotify_fd
-
-	def handle_error(self):
-		import ctypes
-		eno = ctypes.get_errno()
-		raise OSError(eno, self.os.strerror(eno))
-
-	def __del__(self):
-		# This method can be called during interpreter shutdown, which means we
-		# must do the absolute minimum here. Note that there could be running
-		# daemon threads that are trying to call other methods on this object.
-		try:
-			self.os.close(self._inotify_fd)
-		except (AttributeError, TypeError):
-			pass
-
-	def read(self):
-		import ctypes
-		buf = []
-		while True:
-			num = self._read(self._inotify_fd, self._buf, len(self._buf))
-			if num == 0:
-				break
-			if num < 0:
-				en = ctypes.get_errno()
-				if en == errno.EAGAIN:
-					break  # No more data
-				if en == errno.EINTR:
-					continue  # Interrupted, try again
-				raise OSError(en, self.os.strerror(en))
-			buf.append(self._buf.raw[:num])
-		raw = b''.join(buf)
-		pos = 0
-		lraw = len(raw)
-		while lraw - pos >= self.hdr.size:
-			wd, mask, cookie, name_len = self.hdr.unpack_from(raw, pos)
-			# We dont care about names as we only watch files
-			pos += self.hdr.size + name_len
-			self.process_event(wd, mask, cookie)
 
 	def expire_watches(self):
 		now = monotonic()
@@ -127,7 +30,19 @@ class INotifyWatch(object):
 			if last_query - now > self.expire_time:
 				self.unwatch(path)
 
-	def process_event(self, wd, mask, cookie):
+	def process_event(self, wd, mask, cookie, name):
+		if wd == -1 and (mask & self.Q_OVERFLOW):
+			# We missed some INOTIFY events, so we dont
+			# know the state of any tracked files.
+			for path in tuple(self.modified):
+				if os.path.exists(path):
+					self.modified[path] = True
+				else:
+					self.watches.pop(path, None)
+					self.modified.pop(path, None)
+					self.last_query.pop(path, None)
+			return
+
 		for path, num in tuple(self.watches.items()):
 			if num == wd:
 				if mask & self.IGNORED:
@@ -176,7 +91,7 @@ class INotifyWatch(object):
 				# exist/you dont have permission
 				self.watch(path)
 				return True
-			self.read()
+			self.read(get_name=False)
 			if path not in self.modified:
 				# An ignored event was received which means the path has been
 				# automatically unwatched
@@ -193,50 +108,7 @@ class INotifyWatch(object):
 					self.unwatch(path)
 				except OSError:
 					pass
-			if hasattr(self, '_inotify_fd'):
+			super(INotifyWatch, self).close()
-				self.os.close(self._inotify_fd)
-				del self.os
-				del self._add_watch
-				del self._rm_watch
-				del self._inotify_fd
-
-
-def get_inotify(expire_time=10):
-	''' Initialize the inotify based file watcher '''
-	import ctypes
-	if not hasattr(ctypes, 'c_ssize_t'):
-		raise INotifyError('You need python >= 2.7 to use inotify')
-	from ctypes.util import find_library
-	name = find_library('c')
-	if not name:
-		raise INotifyError('Cannot find C library')
-	libc = ctypes.CDLL(name, use_errno=True)
-	for function in ("inotify_add_watch", "inotify_init1", "inotify_rm_watch"):
-		if not hasattr(libc, function):
-			raise INotifyError('libc is too old')
-	# inotify_init1()
-	prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, use_errno=True)
-	init1 = prototype(('inotify_init1', libc), ((1, "flags", 0),))
-
-	# inotify_add_watch()
-	prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32, use_errno=True)
-	add_watch = prototype(('inotify_add_watch', libc), (
-		(1, "fd"), (1, "pathname"), (1, "mask")), use_errno=True)
-
-	# inotify_rm_watch()
-	prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int, use_errno=True)
-	rm_watch = prototype(('inotify_rm_watch', libc), (
-		(1, "fd"), (1, "wd")), use_errno=True)
-
-	# read()
-	prototype = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, use_errno=True)
-	read = prototype(('read', libc), (
-		(1, "fd"), (1, "buf"), (1, "count")), use_errno=True)
-
-	inotify_fd = init1(INotifyWatch.CLOEXEC | INotifyWatch.NONBLOCK)
-	if inotify_fd == -1:
-		raise INotifyError(os.strerror(ctypes.get_errno()))
-	return INotifyWatch(inotify_fd, add_watch, rm_watch, read, expire_time=expire_time)
 
 
 class StatWatch(object):
@@ -289,7 +161,7 @@ def create_file_watcher(use_stat=False, expire_time=10):
 	if use_stat:
 		return StatWatch()
 	try:
-		return get_inotify(expire_time=expire_time)
+		return INotifyWatch(expire_time=expire_time)
 	except INotifyError:
 		pass
 	return StatWatch()

powerline/lib/inotify.py

@@ -0,0 +1,178 @@
+# vim:fileencoding=utf-8:noet
+from __future__ import unicode_literals, absolute_import
+
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys
+import os
+import errno
+
+
+class INotifyError(Exception):
+	pass
+
+
+_inotify = None
+
+
+def load_inotify():
+	''' Initialize the inotify library '''
+	global _inotify
+	if _inotify is None:
+		if hasattr(sys, 'getwindowsversion'):
+			# On windows abort before loading the C library. Windows has
+			# multiple, incompatible C runtimes, and we have no way of knowing
+			# if the one chosen by ctypes is compatible with the currently
+			# loaded one.
+			raise INotifyError('INotify not available on windows')
+		import ctypes
+		if not hasattr(ctypes, 'c_ssize_t'):
+			raise INotifyError('You need python >= 2.7 to use inotify')
+		from ctypes.util import find_library
+		name = find_library('c')
+		if not name:
+			raise INotifyError('Cannot find C library')
+		libc = ctypes.CDLL(name, use_errno=True)
+		for function in ("inotify_add_watch", "inotify_init1", "inotify_rm_watch"):
+			if not hasattr(libc, function):
+				raise INotifyError('libc is too old')
+		# inotify_init1()
+		prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, use_errno=True)
+		init1 = prototype(('inotify_init1', libc), ((1, "flags", 0),))
+
+		# inotify_add_watch()
+		prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32, use_errno=True)
+		add_watch = prototype(('inotify_add_watch', libc), (
+			(1, "fd"), (1, "pathname"), (1, "mask")), use_errno=True)
+
+		# inotify_rm_watch()
+		prototype = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int, use_errno=True)
+		rm_watch = prototype(('inotify_rm_watch', libc), (
+			(1, "fd"), (1, "wd")), use_errno=True)
+
+		# read()
+		prototype = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t, use_errno=True)
+		read = prototype(('read', libc), (
+			(1, "fd"), (1, "buf"), (1, "count")), use_errno=True)
+		_inotify = (init1, add_watch, rm_watch, read)
+	return _inotify
+
+
+class INotify(object):
+	# See <sys/inotify.h> for the flags defined below
+
+	# Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH.
+	ACCESS = 0x00000001         # File was accessed.
+	MODIFY = 0x00000002         # File was modified.
+	ATTRIB = 0x00000004         # Metadata changed.
+	CLOSE_WRITE = 0x00000008    # Writtable file was closed.
+	CLOSE_NOWRITE = 0x00000010  # Unwrittable file closed.
+	OPEN = 0x00000020           # File was opened.
+	MOVED_FROM = 0x00000040     # File was moved from X.
+	MOVED_TO = 0x00000080       # File was moved to Y.
+	CREATE = 0x00000100         # Subfile was created.
+	DELETE = 0x00000200         # Subfile was deleted.
+	DELETE_SELF = 0x00000400    # Self was deleted.
+	MOVE_SELF = 0x00000800      # Self was moved.
+
+	# Events sent by the kernel.
+	UNMOUNT = 0x00002000     # Backing fs was unmounted.
+	Q_OVERFLOW = 0x00004000  # Event queued overflowed.
+	IGNORED = 0x00008000     # File was ignored.
+
+	# Helper events.
+	CLOSE = (CLOSE_WRITE | CLOSE_NOWRITE)  # Close.
+	MOVE = (MOVED_FROM | MOVED_TO)         # Moves.
+
+	# Special flags.
+	ONLYDIR = 0x01000000      # Only watch the path if it is a directory.
+	DONT_FOLLOW = 0x02000000  # Do not follow a sym link.
+	EXCL_UNLINK = 0x04000000  # Exclude events on unlinked objects.
+	MASK_ADD = 0x20000000     # Add to the mask of an already existing watch.
+	ISDIR = 0x40000000        # Event occurred against dir.
+	ONESHOT = 0x80000000      # Only send event once.
+
+	# All events which a program can wait on.
+	ALL_EVENTS = (ACCESS | MODIFY | ATTRIB | CLOSE_WRITE | CLOSE_NOWRITE |
+					OPEN | MOVED_FROM | MOVED_TO | CREATE | DELETE |
+					DELETE_SELF | MOVE_SELF)
+
+	# See <bits/inotify.h>
+	CLOEXEC = 0x80000
+	NONBLOCK = 0x800
+
+	def __init__(self, cloexec=True, nonblock=True):
+		import ctypes
+		import struct
+		self._init1, self._add_watch, self._rm_watch, self._read = load_inotify()
+		flags = 0
+		if cloexec:
+			flags |= self.CLOEXEC
+		if nonblock:
+			flags |= self.NONBLOCK
+		self._inotify_fd = self._init1(flags)
+		if self._inotify_fd == -1:
+			raise INotifyError(os.strerror(ctypes.get_errno()))
+
+		self._buf = ctypes.create_string_buffer(5000)
+		self.fenc = sys.getfilesystemencoding() or 'utf-8'
+		self.hdr = struct.Struct(b'iIII')
+		if self.fenc == 'ascii':
+			self.fenc = 'utf-8'
+		# We keep a reference to os to prevent it from being deleted
+		# during interpreter shutdown, which would lead to errors in the
+		# __del__ method
+		self.os = os
+
+	def handle_error(self):
+		import ctypes
+		eno = ctypes.get_errno()
+		raise OSError(eno, self.os.strerror(eno))
+
+	def __del__(self):
+		# This method can be called during interpreter shutdown, which means we
+		# must do the absolute minimum here. Note that there could be running
+		# daemon threads that are trying to call other methods on this object.
+		try:
+			self.os.close(self._inotify_fd)
+		except (AttributeError, TypeError):
+			pass
+
+	def close(self):
+		if hasattr(self, '_inotify_fd'):
+			self.os.close(self._inotify_fd)
+			del self.os
+			del self._add_watch
+			del self._rm_watch
+			del self._inotify_fd
+
+	def read(self, get_name=True):
+		import ctypes
+		buf = []
+		while True:
+			num = self._read(self._inotify_fd, self._buf, len(self._buf))
+			if num == 0:
+				break
+			if num < 0:
+				en = ctypes.get_errno()
+				if en == errno.EAGAIN:
+					break  # No more data
+				if en == errno.EINTR:
+					continue  # Interrupted, try again
+				raise OSError(en, self.os.strerror(en))
+			buf.append(self._buf.raw[:num])
+		raw = b''.join(buf)
+		pos = 0
+		lraw = len(raw)
+		while lraw - pos >= self.hdr.size:
+			wd, mask, cookie, name_len = self.hdr.unpack_from(raw, pos)
+			pos += self.hdr.size
+			name = None
+			if get_name:
+				name = raw[pos:pos + name_len].rstrip(b'\0').decode(self.fenc)
+			pos += name_len
+			self.process_event(wd, mask, cookie, name)
+
+	def process_event(self, *args):
+		raise NotImplementedError()

powerline/lib/threaded.py

@@ -7,7 +7,28 @@ from powerline.lib.monotonic import monotonic
 from threading import Thread, Lock, Event
 
 
-class ThreadedSegment(object):
+class MultiRunnedThread(object):
+	daemon = True
+
+	def __init__(self):
+		self.thread = None
+
+	def is_alive(self):
+		return self.thread and self.thread.is_alive()
+
+	def start(self):
+		self.shutdown_event.clear()
+		self.thread = Thread(target=self.run)
+		self.thread.daemon = self.daemon
+		self.thread.start()
+
+	def join(self, *args, **kwargs):
+		if self.thread:
+			return self.thread.join(*args, **kwargs)
+		return None
+
+
+class ThreadedSegment(MultiRunnedThread):
 	min_sleep_time = 0.1
 	update_first = True
 	interval = 1
@@ -15,12 +36,11 @@ class ThreadedSegment(object):
 
 	def __init__(self):
 		super(ThreadedSegment, self).__init__()
-		self.shutdown_event = Event()
 		self.run_once = True
-		self.thread = None
 		self.skip = False
 		self.crashed_value = None
 		self.update_value = None
+		self.updated = False
 
 	def __call__(self, pl, update_first=True, **kwargs):
 		if self.run_once:
@@ -37,6 +57,7 @@ class ThreadedSegment(object):
 			self.start()
 		elif not self.updated:
 			update_value = self.get_update_value(True)
+			self.updated = True
 		else:
 			update_value = self.update_value
 
@@ -50,15 +71,6 @@ class ThreadedSegment(object):
 			self.update_value = self.update(self.update_value)
 		return self.update_value
 
-	def is_alive(self):
-		return self.thread and self.thread.is_alive()
-
-	def start(self):
-		self.shutdown_event.clear()
-		self.thread = Thread(target=self.run)
-		self.thread.daemon = self.daemon
-		self.thread.start()
-
 	def run(self):
 		while not self.shutdown_event.is_set():
 			start_time = monotonic()
@@ -77,8 +89,9 @@ class ThreadedSegment(object):
 	def shutdown(self):
 		self.shutdown_event.set()
 		if self.daemon and self.is_alive():
-			# Give the worker thread a chance to shutdown, but don't block for too long
+			# Give the worker thread a chance to shutdown, but don't block for 
-			self.thread.join(.01)
+			# too long
+			self.join(0.01)
 
 	def set_interval(self, interval=None):
 		# Allowing “interval” keyword in configuration.
@@ -88,9 +101,10 @@ class ThreadedSegment(object):
 		interval = interval or getattr(self, 'interval')
 		self.interval = interval
 
-	def set_state(self, interval=None, update_first=True, **kwargs):
+	def set_state(self, interval=None, update_first=True, shutdown_event=None, **kwargs):
 		self.set_interval(interval)
-		self.updated = not (update_first and self.update_first)
+		self.shutdown_event = shutdown_event or Event()
+		self.updated = self.updated or (not (update_first and self.update_first))
 
 	def startup(self, pl, **kwargs):
 		self.run_once = False
@@ -102,6 +116,15 @@ class ThreadedSegment(object):
 		if not self.is_alive():
 			self.start()
 
+	def critical(self, *args, **kwargs):
+		self.pl.critical(prefix=self.__class__.__name__, *args, **kwargs)
+
+	def exception(self, *args, **kwargs):
+		self.pl.exception(prefix=self.__class__.__name__, *args, **kwargs)
+
+	def info(self, *args, **kwargs):
+		self.pl.info(prefix=self.__class__.__name__, *args, **kwargs)
+
 	def error(self, *args, **kwargs):
 		self.pl.error(prefix=self.__class__.__name__, *args, **kwargs)
 
@@ -112,13 +135,6 @@ class ThreadedSegment(object):
 		self.pl.debug(prefix=self.__class__.__name__, *args, **kwargs)
 
 
-def printed(func):
-	def f(*args, **kwargs):
-		print(func.__name__)
-		return func(*args, **kwargs)
-	return f
-
-
 class KwThreadedSegment(ThreadedSegment):
 	drop_interval = 10 * 60
 	update_first = True
@@ -174,8 +190,9 @@ class KwThreadedSegment(ThreadedSegment):
 
 		return update_value
 
-	def set_state(self, interval=None, **kwargs):
+	def set_state(self, interval=None, shutdown_event=None, **kwargs):
 		self.set_interval(interval)
+		self.shutdown_event = shutdown_event or Event()
 
 	@staticmethod
 	def render_one(update_state, **kwargs):

powerline/lib/tree_watcher.py

@@ -0,0 +1,199 @@
+# vim:fileencoding=utf-8:noet
+from __future__ import (unicode_literals, absolute_import, print_function)
+
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys
+import os
+import errno
+from time import sleep
+from powerline.lib.monotonic import monotonic
+
+from powerline.lib.inotify import INotify, INotifyError
+
+
+class NoSuchDir(ValueError):
+	pass
+
+
+class DirTooLarge(ValueError):
+	def __init__(self, bdir):
+		ValueError.__init__(self, 'The directory {0} is too large to monitor. Try increasing the value in /proc/sys/fs/inotify/max_user_watches'.format(bdir))
+
+
+class INotifyTreeWatcher(INotify):
+	is_dummy = False
+
+	def __init__(self, basedir):
+		super(INotifyTreeWatcher, self).__init__()
+		self.basedir = os.path.abspath(basedir)
+		self.watch_tree()
+		self.modified = True
+
+	def watch_tree(self):
+		self.watched_dirs = {}
+		self.watched_rmap = {}
+		try:
+			self.add_watches(self.basedir)
+		except OSError as e:
+			if e.errno == errno.ENOSPC:
+				raise DirTooLarge(self.basedir)
+
+	def add_watches(self, base, top_level=True):
+		''' Add watches for this directory and all its descendant directories,
+		recursively. '''
+		base = os.path.abspath(base)
+		try:
+			is_dir = self.add_watch(base)
+		except OSError as e:
+			if e.errno == errno.ENOENT:
+				# The entry could have been deleted between listdir() and
+				# add_watch().
+				if top_level:
+					raise NoSuchDir('The dir {0} does not exist'.format(base))
+				return
+			if e.errno == errno.EACCES:
+				# We silently ignore entries for which we dont have permission,
+				# unless they are the top level dir
+				if top_level:
+					raise NoSuchDir('You do not have permission to monitor {0}'.format(base))
+				return
+			raise
+		else:
+			if is_dir:
+				try:
+					files = os.listdir(base)
+				except OSError as e:
+					if e.errno in (errno.ENOTDIR, errno.ENOENT):
+						# The dir was deleted/replaced between the add_watch()
+						# and listdir()
+						if top_level:
+							raise NoSuchDir('The dir {0} does not exist'.format(base))
+						return
+					raise
+				for x in files:
+					self.add_watches(os.path.join(base, x), top_level=False)
+			elif top_level:
+				# The top level dir is a file, not good.
+				raise NoSuchDir('The dir {0} does not exist'.format(base))
+
+	def add_watch(self, path):
+		import ctypes
+		bpath = path if isinstance(path, bytes) else path.encode(self.fenc)
+		wd = self._add_watch(self._inotify_fd, ctypes.c_char_p(bpath),
+				# Ignore symlinks and watch only directories
+				self.DONT_FOLLOW | self.ONLYDIR |
+
+				self.MODIFY | self.CREATE | self.DELETE |
+				self.MOVE_SELF | self.MOVED_FROM | self.MOVED_TO |
+				self.ATTRIB | self.MOVE_SELF | self.DELETE_SELF)
+		if wd == -1:
+			eno = ctypes.get_errno()
+			if eno == errno.ENOTDIR:
+				return False
+			raise OSError(eno, 'Failed to add watch for: {0}: {1}'.format(path, self.os.strerror(eno)))
+		self.watched_dirs[path] = wd
+		self.watched_rmap[wd] = path
+		return True
+
+	def process_event(self, wd, mask, cookie, name):
+		if wd == -1 and (mask & self.Q_OVERFLOW):
+			# We missed some INOTIFY events, so we dont
+			# know the state of any tracked dirs.
+			self.watch_tree()
+			self.modified = True
+			return
+		path = self.watched_rmap.get(wd, None)
+		if path is not None:
+			self.modified = True
+			if mask & self.CREATE:
+				# A new sub-directory might have been created, monitor it.
+				try:
+					self.add_watch(os.path.join(path, name))
+				except OSError as e:
+					if e.errno == errno.ENOENT:
+						# Deleted before add_watch()
+						pass
+					elif e.errno == errno.ENOSPC:
+						raise DirTooLarge(self.basedir)
+					else:
+						raise
+
+	def __call__(self):
+		self.read()
+		ret = self.modified
+		self.modified = False
+		return ret
+
+
+class DummyTreeWatcher(object):
+	is_dummy = True
+
+	def __init__(self, basedir):
+		self.basedir = os.path.abspath(basedir)
+
+	def __call__(self):
+		return False
+
+
+class TreeWatcher(object):
+	def __init__(self, expire_time=10):
+		self.watches = {}
+		self.last_query_times = {}
+		self.expire_time = expire_time * 60
+
+	def watch(self, path, logger=None):
+		path = os.path.abspath(path)
+		try:
+			w = INotifyTreeWatcher(path)
+		except (INotifyError, DirTooLarge) as e:
+			if logger is not None:
+				logger.warn('Failed to watch path: {0} with error: {1}'.format(path, e))
+			w = DummyTreeWatcher(path)
+		self.watches[path] = w
+		return w
+
+	def is_actually_watched(self, path):
+		w = self.watches.get(path, None)
+		return not getattr(w, 'is_dummy', True)
+
+	def expire_old_queries(self):
+		pop = []
+		now = monotonic()
+		for path, lt in self.last_query_times.items():
+			if now - lt > self.expire_time:
+				pop.append(path)
+		for path in pop:
+			del self.last_query_times[path]
+
+	def __call__(self, path, logger=None):
+		path = os.path.abspath(path)
+		self.expire_old_queries()
+		self.last_query_times[path] = monotonic()
+		w = self.watches.get(path, None)
+		if w is None:
+			try:
+				self.watch(path)
+			except NoSuchDir:
+				pass
+			return True
+		try:
+			return w()
+		except DirTooLarge as e:
+			if logger is not None:
+				logger.warn(str(e))
+			self.watches[path] = DummyTreeWatcher(path)
+			return False
+
+if __name__ == '__main__':
+	w = INotifyTreeWatcher(sys.argv[-1])
+	w()
+	print ('Monitoring', sys.argv[-1], 'press Ctrl-C to stop')
+	try:
+		while True:
+			if w():
+				print (sys.argv[-1], 'changed')
+			sleep(1)
+	except KeyboardInterrupt:
+		raise SystemExit(0)

powerline/lint/__init__.py

@@ -1,5 +1,6 @@
 from powerline.lint.markedjson import load
-from powerline import load_json_config, find_config_file, Powerline
+from powerline import find_config_file, Powerline
+from powerline.lib.config import load_json_config
 from powerline.lint.markedjson.error import echoerr, MarkedError
 from powerline.segments.vim import vim_modes
 import itertools
@@ -73,11 +74,15 @@ class Spec(object):
 				spec.context_message(msg)
 		return self
 
-	def check_type(self, value, context_mark, data, context, echoerr, t):
+	def check_type(self, value, context_mark, data, context, echoerr, types):
-		if type(value.value) is not t:
+		if type(value.value) not in types:
 			echoerr(context=self.cmsg.format(key=context_key(context)),
 					context_mark=context_mark,
-					problem='{0!r} must be a {1} instance, not {2}'.format(value, t.__name__, type(value.value).__name__),
+					problem='{0!r} must be a {1} instance, not {2}'.format(
+						value,
+						', '.join((t.__name__ for t in types)),
+						type(value.value).__name__
+					),
 					problem_mark=value.mark)
 			return False, True
 		return True, False
@@ -141,8 +146,8 @@ class Spec(object):
 				return False, hadproblem
 		return True, hadproblem
 
-	def type(self, t):
+	def type(self, *args):
-		self.checks.append(('check_type', t))
+		self.checks.append(('check_type', args))
 		return self
 
 	cmp_funcs = {
@@ -172,12 +177,14 @@ class Spec(object):
 	def cmp(self, comparison, cint, msg_func=None):
 		if type(cint) is str:
 			self.type(unicode)
+		elif type(cint) is float:
+			self.type(int, float)
 		else:
 			self.type(type(cint))
 		cmp_func = self.cmp_funcs[comparison]
 		msg_func = msg_func or (lambda value: '{0} is not {1} {2}'.format(value, self.cmp_msgs[comparison], cint))
 		self.checks.append(('check_func',
-					(lambda value, *args: (True, True, not cmp_func(value, cint))),
+					(lambda value, *args: (True, True, not cmp_func(value.value, cint))),
 					msg_func))
 		return self
 
@@ -242,6 +249,11 @@ class Spec(object):
 						msg_func))
 		return self
 
+	def error(self, msg):
+		self.checks.append(('check_func', lambda *args: (True, True, True),
+							lambda value: msg.format(value)))
+		return self
+
 	def either(self, *specs):
 		start = len(self.specs)
 		self.specs.extend(specs)
@@ -406,6 +418,8 @@ main_spec = (Spec(
 		log_level=Spec().re('^[A-Z]+$').func(lambda value, *args: (True, True, not hasattr(logging, value)),
 										lambda value: 'unknown debugging level {0}'.format(value)).optional(),
 		log_format=Spec().type(str).optional(),
+		interval=Spec().either(Spec().cmp('gt', 0.0), Spec().type(type(None))).optional(),
+		reload_config=Spec().type(bool).optional(),
 	).context_message('Error while loading common configuration (key {key})'),
 	ext=Spec(
 		vim=Spec(
@@ -786,8 +800,11 @@ def check_segment_data_key(key, data, context, echoerr):
 
 # FIXME More checks, limit existing to ThreadedSegment instances only
 args_spec = Spec(
-	interval=Spec().either(Spec().type(float), Spec().type(int)).optional(),
+	interval=Spec().cmp('gt', 0.0).optional(),
 	update_first=Spec().type(bool).optional(),
+	shutdown_event=Spec().error('Shutdown event must be set by powerline').optional(),
+	pl=Spec().error('pl object must be set by powerline').optional(),
+	segment_info=Spec().error('Segment info dictionary must be set by powerline').optional(),
 ).unknown_spec(Spec(), Spec()).optional().copy
 highlight_group_spec = Spec().type(unicode).copy
 segment_module_spec = Spec().type(unicode).func(check_segment_module).optional().copy
@@ -801,7 +818,7 @@ segments_spec = Spec().optional().list(
 		draw_soft_divider=Spec().type(bool).optional(),
 		draw_inner_divider=Spec().type(bool).optional(),
 		module=segment_module_spec(),
-		priority=Spec().cmp('ge', -1).optional(),
+		priority=Spec().either(Spec().cmp('eq', -1), Spec().cmp('ge', 0.0)).optional(),
 		after=Spec().type(unicode).optional(),
 		before=Spec().type(unicode).optional(),
 		width=Spec().either(Spec().unsigned(), Spec().cmp('eq', 'auto')).optional(),

powerline/segments/common.py

@@ -18,8 +18,10 @@ from powerline.lib.humanize_bytes import humanize_bytes
 from powerline.theme import requires_segment_info
 from collections import namedtuple
 
+
 cpu_count = None
 
+
 @requires_segment_info
 def hostname(pl, segment_info, only_if_ssh=False, exclude_domain=False):
 	'''Return the current hostname.
@@ -840,7 +842,7 @@ class EmailIMAPSegment(KwThreadedSegment):
 			return [{
 				'contents': str(unread_count),
 				'highlight_group': ['email_alert_gradient', 'email_alert'],
-				'gradient_level': unread_count * 100.0 / max_msgs,
+				'gradient_level': min(unread_count * 100.0 / max_msgs, 100),
 			}]
 
 

powerline/theme.py

@@ -22,7 +22,14 @@ def requires_segment_info(func):
 
 
 class Theme(object):
-	def __init__(self, ext, theme_config, common_config, pl, top_theme_config=None, run_once=False):
+	def __init__(self,
+				ext,
+				theme_config,
+				common_config,
+				pl,
+				top_theme_config=None,
+				run_once=False,
+				shutdown_event=None):
 		self.dividers = theme_config.get('dividers', common_config['dividers'])
 		self.spaces = theme_config.get('spaces', common_config['spaces'])
 		self.segments = {
@@ -44,7 +51,7 @@ class Theme(object):
 				if not run_once:
 					if segment['startup']:
 						try:
-							segment['startup'](pl=pl, **segment['args'])
+							segment['startup'](pl=pl, shutdown_event=shutdown_event, **segment['args'])
 						except Exception as e:
 							pl.error('Exception during {0} startup: {1}', segment['name'], str(e))
 							continue

powerline/vim.py

@@ -46,6 +46,7 @@ class VimPowerline(Powerline):
 			``True`` if theme was added successfully and ``False`` if theme with 
 			the same matcher already exists.
 		'''
+		self.update_renderer()
 		key = self.get_matcher(key)
 		try:
 			self.renderer.add_local_theme(key, {'config': config})

tests/__init__.py

@@ -2,5 +2,7 @@
 import sys
 if sys.version_info < (2, 7):
 	from unittest2 import TestCase, main  # NOQA
+	from unittest2.case import SkipTest  # NOQA
 else:
 	from unittest import TestCase, main  # NOQA
+	from unittest.case import SkipTest  # NOQA

tests/lib/config_mock.py

@@ -1,6 +1,7 @@
 # vim:fileencoding=utf-8:noet
 from threading import Lock
 from powerline.renderer import Renderer
+from powerline.lib.config import ConfigLoader
 from powerline import Powerline
 from copy import deepcopy
 
@@ -9,12 +10,12 @@ access_log = []
 access_lock = Lock()
 
 
-def load_json_config(config, config_file_path, *args, **kwargs):
+def load_json_config(config_file_path, *args, **kwargs):
 	global access_log
 	with access_lock:
 		access_log.append(config_file_path)
 	try:
-		return deepcopy(config[config_file_path])
+		return deepcopy(config_container['config'][config_file_path])
 	except KeyError:
 		raise IOError(config_file_path)
 
@@ -41,10 +42,10 @@ class Watcher(object):
 		pass
 
 	def __call__(self, file):
-		if file in self.events:
+		with self.lock:
-			with self.lock:
+			if file in self.events:
 				self.events.remove(file)
-			return True
+				return True
 		return False
 
 	def _reset(self, files):
@@ -98,18 +99,20 @@ def get_powerline(**kwargs):
 	return TestPowerline(
 		ext='test',
 		renderer_module='tests.lib.config_mock',
-		interval=0,
 		logger=Logger(),
-		watcher=Watcher(),
+		config_loader=ConfigLoader(load=load_json_config, watcher=Watcher()),
 		**kwargs
 	)
 
 
-def swap_attributes(config_container, powerline_module, replaces):
+config_container = None
+
+
+def swap_attributes(cfg_container, powerline_module, replaces):
+	global config_container
+	config_container = cfg_container
 	if not replaces:
 		replaces = {
-			'watcher': Watcher(),
-			'load_json_config': lambda *args: load_json_config(config_container['config'], *args),
 			'find_config_file': lambda *args: find_config_file(config_container['config'], *args),
 		}
 	for attr, val in replaces.items():

tests/test_config_reload.py

@@ -6,7 +6,6 @@ from tests import TestCase
 from tests.lib import replace_item
 from tests.lib.config_mock import swap_attributes, get_powerline, pop_events
 from copy import deepcopy
-from threading import Lock
 
 
 config = {
@@ -23,6 +22,7 @@ config = {
 				},
 			},
 			'spaces': 0,
+			'interval': 0,
 		},
 		'ext': {
 			'test': {
@@ -97,7 +97,7 @@ def sleep(interval):
 
 
 def add_watcher_events(p, *args, **kwargs):
-	p.watcher._reset(args)
+	p.config_loader.watcher._reset(args)
 	while not p._will_create_renderer():
 		sleep(kwargs.get('interval', 0.000001))
 		if not kwargs.get('wait', True):
@@ -127,8 +127,8 @@ class TestConfigReload(TestCase):
 	def test_reload_main(self):
 		with get_powerline(run_once=False) as p:
 			with replace_item(globals(), 'config', deepcopy(config)):
-				self.assertAccessEvents('config', 'colors', 'colorschemes/test/default', 'themes/test/default')
 				self.assertEqual(p.render(), '<1 2 1> s<2 4 False>>><3 4 4>g<4 False False>>><None None None>')
+				self.assertAccessEvents('config', 'colors', 'colorschemes/test/default', 'themes/test/default')
 
 				config['config']['common']['spaces'] = 1
 				add_watcher_events(p, 'config')
@@ -187,7 +187,7 @@ class TestConfigReload(TestCase):
 				add_watcher_events(p, 'config')
 				self.assertEqual(p.render(), '<1 2 1> s<2 4 False>>><3 4 4>g<4 False False>>><None None None>')
 				self.assertAccessEvents('config')
-				self.assertEqual(p.logger._pop_msgs(), ['exception:test:Failed to create renderer: fcf:colorschemes/test/nonexistentraise'])
+				self.assertIn('exception:test:Failed to create renderer: fcf:colorschemes/test/nonexistentraise', p.logger._pop_msgs())
 
 				config['colorschemes/test/nonexistentraise'] = {
 					'groups': {
@@ -241,6 +241,20 @@ class TestConfigReload(TestCase):
 				self.assertEqual(p.logger._pop_msgs(), [])
 		pop_events()
 
+	def test_reload_theme_main(self):
+		with replace_item(globals(), 'config', deepcopy(config)):
+			config['config']['common']['interval'] = None
+			with get_powerline(run_once=False) as p:
+				self.assertEqual(p.render(), '<1 2 1> s<2 4 False>>><3 4 4>g<4 False False>>><None None None>')
+				self.assertAccessEvents('config', 'colors', 'colorschemes/test/default', 'themes/test/default')
+
+				config['themes/test/default']['segments']['left'][0]['contents'] = 'col3'
+				add_watcher_events(p, 'themes/test/default', wait=False)
+				self.assertEqual(p.render(), '<1 2 1> col3<2 4 False>>><3 4 4>g<4 False False>>><None None None>')
+				self.assertAccessEvents('themes/test/default')
+				self.assertEqual(p.logger._pop_msgs(), [])
+		pop_events()
+
 
 replaces = {}
 

tests/test_lib.py

@@ -5,7 +5,8 @@ from powerline.lib.vcs import guess
 from subprocess import call, PIPE
 import os
 import sys
-from tests import TestCase
+from functools import partial
+from tests import TestCase, SkipTest
 
 
 class TestLib(TestCase):
@@ -37,6 +38,88 @@ class TestLib(TestCase):
 		self.assertEqual(humanize_bytes(1000000000, si_prefix=False), '953.7 MiB')
 
 
+class TestFilesystemWatchers(TestCase):
+	def do_test_for_change(self, watcher, path):
+		import time
+		st = time.time()
+		while time.time() - st < 1:
+			if watcher(path):
+				return
+			time.sleep(0.1)
+		self.fail('The change to {0} was not detected'.format(path))
+
+	def test_file_watcher(self):
+		from powerline.lib.file_watcher import create_file_watcher
+		w = create_file_watcher(use_stat=False)
+		if w.is_stat_based:
+			raise SkipTest('This test is not suitable for a stat based file watcher')
+		f1, f2 = os.path.join(INOTIFY_DIR, 'file1'), os.path.join(INOTIFY_DIR, 'file2')
+		with open(f1, 'wb'):
+			with open(f2, 'wb'):
+				pass
+		ne = os.path.join(INOTIFY_DIR, 'notexists')
+		self.assertRaises(OSError, w, ne)
+		self.assertTrue(w(f1))
+		self.assertTrue(w(f2))
+		os.utime(f1, None), os.utime(f2, None)
+		self.do_test_for_change(w, f1)
+		self.do_test_for_change(w, f2)
+		# Repeat once
+		os.utime(f1, None), os.utime(f2, None)
+		self.do_test_for_change(w, f1)
+		self.do_test_for_change(w, f2)
+		# Check that no false changes are reported
+		self.assertFalse(w(f1), 'Spurious change detected')
+		self.assertFalse(w(f2), 'Spurious change detected')
+		# Check that open the file with 'w' triggers a change
+		with open(f1, 'wb'):
+			with open(f2, 'wb'):
+				pass
+		self.do_test_for_change(w, f1)
+		self.do_test_for_change(w, f2)
+		# Check that writing to a file with 'a' triggers a change
+		with open(f1, 'ab') as f:
+			f.write(b'1')
+		self.do_test_for_change(w, f1)
+		# Check that deleting a file registers as a change
+		os.unlink(f1)
+		self.do_test_for_change(w, f1)
+
+	def test_tree_watcher(self):
+		from powerline.lib.tree_watcher import TreeWatcher
+		tw = TreeWatcher()
+		subdir = os.path.join(INOTIFY_DIR, 'subdir')
+		os.mkdir(subdir)
+		if tw.watch(INOTIFY_DIR).is_dummy:
+			raise SkipTest('No tree watcher available')
+		import shutil
+		self.assertTrue(tw(INOTIFY_DIR))
+		self.assertFalse(tw(INOTIFY_DIR))
+		changed = partial(self.do_test_for_change, tw, INOTIFY_DIR)
+		open(os.path.join(INOTIFY_DIR, 'tree1'), 'w').close()
+		changed()
+		open(os.path.join(subdir, 'tree1'), 'w').close()
+		changed()
+		os.unlink(os.path.join(subdir, 'tree1'))
+		changed()
+		os.rmdir(subdir)
+		changed()
+		os.mkdir(subdir)
+		changed()
+		os.rename(subdir, subdir + '1')
+		changed()
+		shutil.rmtree(subdir + '1')
+		changed()
+		os.mkdir(subdir)
+		f = os.path.join(subdir, 'f')
+		open(f, 'w').close()
+		changed()
+		with open(f, 'a') as s:
+			s.write(' ')
+		changed()
+		os.rename(f, f + '1')
+		changed()
+
 use_mercurial = use_bzr = sys.version_info < (3, 0)
 
 
@@ -106,6 +189,7 @@ old_cwd = None
 GIT_REPO = 'git_repo' + os.environ.get('PYTHON', '')
 HG_REPO = 'hg_repo' + os.environ.get('PYTHON', '')
 BZR_REPO = 'bzr_repo' + os.environ.get('PYTHON', '')
+INOTIFY_DIR = 'inotify' + os.environ.get('PYTHON', '')
 
 
 def setUpModule():
@@ -130,12 +214,13 @@ def setUpModule():
 		call(['bzr', 'config', 'email=Foo <bar@example.org>'], cwd=BZR_REPO)
 		call(['bzr', 'config', 'nickname=test_powerline'], cwd=BZR_REPO)
 		call(['bzr', 'config', 'create_signatures=0'], cwd=BZR_REPO)
+	os.mkdir(INOTIFY_DIR)
 
 
 def tearDownModule():
 	global old_cwd
 	global old_HGRCPATH
-	for repo_dir in [GIT_REPO] + ([HG_REPO] if use_mercurial else []) + ([BZR_REPO] if use_bzr else []):
+	for repo_dir in [INOTIFY_DIR, GIT_REPO] + ([HG_REPO] if use_mercurial else []) + ([BZR_REPO] if use_bzr else []):
 		for root, dirs, files in list(os.walk(repo_dir, topdown=False)):
 			for file in files:
 				os.remove(os.path.join(root, file))

tests/vim.py

@@ -11,26 +11,114 @@ _options = {
 _last_bufnr = 0
 _highlights = {}
 
-buffers = {}
 
-windows = []
+_thread_id = None
 
 
-def _buffer():
+def _set_thread_id():
-	return windows[_window - 1].buffer.number
+	global _thread_id
+	from threading import current_thread
+	_thread_id = current_thread().ident
 
 
-def _logged(func):
+# Assuming import is done from the main thread
+_set_thread_id()
+
+
+def _vim(func):
 	from functools import wraps
+	from threading import current_thread
 
 	@wraps(func)
 	def f(*args, **kwargs):
+		global _thread_id
+		if _thread_id != current_thread().ident:
+			raise RuntimeError('Accessing vim from separate threads is not allowed')
 		_log.append((func.__name__, args))
 		return func(*args, **kwargs)
 
 	return f
 
 
+class _Buffers(object):
+	@_vim
+	def __init__(self):
+		self.d = {}
+
+	@_vim
+	def __getitem__(self, item):
+		return self.d[item]
+
+	@_vim
+	def __setitem__(self, item, value):
+		self.d[item] = value
+
+	@_vim
+	def __contains__(self, item):
+		return item in self.d
+
+	@_vim
+	def __nonzero__(self):
+		return not not self.d
+
+	@_vim
+	def keys(self):
+		return self.d.keys()
+
+	@_vim
+	def pop(self, *args, **kwargs):
+		return self.d.pop(*args, **kwargs)
+
+
+buffers = _Buffers()
+
+
+class _Windows(object):
+	@_vim
+	def __init__(self):
+		self.l = []
+
+	@_vim
+	def __getitem__(self, item):
+		return self.l[item]
+
+	@_vim
+	def __setitem__(self, item, value):
+		self.l[item] = value
+
+	@_vim
+	def __len__(self):
+		return len(self.l)
+
+	@_vim
+	def __iter__(self):
+		return iter(self.l)
+
+	@_vim
+	def __nonzero__(self):
+		return not not self.l
+
+	@_vim
+	def pop(self, *args, **kwargs):
+		return self.l.pop(*args, **kwargs)
+
+	@_vim
+	def append(self, *args, **kwargs):
+		return self.l.append(*args, **kwargs)
+
+	@_vim
+	def index(self, *args, **kwargs):
+		return self.l.index(*args, **kwargs)
+
+
+windows = _Windows()
+
+
+@_vim
+def _buffer():
+	return windows[_window - 1].buffer.number
+
+
 def _construct_result(r):
 	import sys
 	if sys.version_info < (3,):
@@ -58,7 +146,7 @@ def _log_print():
 		sys.stdout.write(repr(entry) + '\n')
 
 
-@_logged
+@_vim
 def command(cmd):
 	if cmd.startswith('let g:'):
 		import re
@@ -71,7 +159,7 @@ def command(cmd):
 		raise NotImplementedError
 
 
-@_logged
+@_vim
 def eval(expr):
 	if expr.startswith('g:'):
 		return _g[expr[2:]]
@@ -83,7 +171,7 @@ def eval(expr):
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 def bindeval(expr):
 	if expr == 'g:':
 		return _g
@@ -95,7 +183,7 @@ def bindeval(expr):
 		raise NotImplementedError
 
 
-@_logged
+@_vim
 @_str_func
 def _emul_mode(*args):
 	if args and args[0]:
@@ -104,11 +192,11 @@ def _emul_mode(*args):
 		return _mode[0]
 
 
-@_logged
+@_vim
 @_str_func
 def _emul_getbufvar(bufnr, varname):
 	if varname[0] == '&':
-		if bufnr not in _buf_options:
+		if bufnr not in buffers:
 			return ''
 		try:
 			return _buf_options[bufnr][varname[1:]]
@@ -120,25 +208,25 @@ def _emul_getbufvar(bufnr, varname):
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 @_str_func
 def _emul_getwinvar(winnr, varname):
 	return _win_scopes[winnr][varname]
 
 
-@_logged
+@_vim
 def _emul_setwinvar(winnr, varname, value):
 	_win_scopes[winnr][varname] = value
 
 
-@_logged
+@_vim
 def _emul_virtcol(expr):
 	if expr == '.':
 		return windows[_window - 1].cursor[1] + 1
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 @_str_func
 def _emul_fnamemodify(path, modstring):
 	import os
@@ -154,7 +242,7 @@ def _emul_fnamemodify(path, modstring):
 	return path
 
 
-@_logged
+@_vim
 @_str_func
 def _emul_expand(expr):
 	if expr == '<abuf>':
@@ -162,21 +250,21 @@ def _emul_expand(expr):
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 def _emul_bufnr(expr):
 	if expr == '$':
 		return _last_bufnr
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 def _emul_exists(varname):
 	if varname.startswith('g:'):
 		return varname[2:] in _g
 	raise NotImplementedError
 
 
-@_logged
+@_vim
 def _emul_line2byte(line):
 	buflines = _buf_lines[_buffer()]
 	if line == len(buflines) + 1:
@@ -287,7 +375,7 @@ current = _Current()
 _dict = None
 
 
-@_logged
+@_vim
 def _init():
 	global _dict
 
@@ -302,7 +390,7 @@ def _init():
 	return _dict
 
 
-@_logged
+@_vim
 def _get_segment_info():
 	mode_translations = {
 			chr(ord('V') - 0x40): '^V',
@@ -319,12 +407,12 @@ def _get_segment_info():
 	}
 
 
-@_logged
+@_vim
 def _launch_event(event):
 	pass
 
 
-@_logged
+@_vim
 def _start_mode(mode):
 	global _mode
 	if mode == 'i':
@@ -334,7 +422,7 @@ def _start_mode(mode):
 	_mode = mode
 
 
-@_logged
+@_vim
 def _undo():
 	if len(_undostate[_buffer()]) == 1:
 		return
@@ -344,7 +432,7 @@ def _undo():
 		_buf_options[_buffer()]['modified'] = 0
 
 
-@_logged
+@_vim
 def _edit(name=None):
 	global _last_bufnr
 	if _buffer() and buffers[_buffer()].name is None:
@@ -355,14 +443,14 @@ def _edit(name=None):
 		windows[_window - 1].buffer = buf
 
 
-@_logged
+@_vim
 def _new(name=None):
 	global _window
 	_Window(buffer={'name': name})
 	_window = len(windows)
 
 
-@_logged
+@_vim
 def _del_window(winnr):
 	win = windows.pop(winnr - 1)
 	_win_scopes.pop(winnr)
@@ -371,7 +459,7 @@ def _del_window(winnr):
 	return win
 
 
-@_logged
+@_vim
 def _close(winnr, wipe=True):
 	global _window
 	win = _del_window(winnr)
@@ -387,7 +475,7 @@ def _close(winnr, wipe=True):
 		_Window()
 
 
-@_logged
+@_vim
 def _bw(bufnr=None):
 	bufnr = bufnr or _buffer()
 	winnr = 1
@@ -401,12 +489,12 @@ def _bw(bufnr=None):
 	_b(max(buffers.keys()))
 
 
-@_logged
+@_vim
 def _b(bufnr):
 	windows[_window - 1].buffer = buffers[bufnr]
 
 
-@_logged
+@_vim
 def _set_cursor(line, col):
 	windows[_window - 1].cursor = (line, col)
 	if _mode == 'n':
@@ -415,12 +503,12 @@ def _set_cursor(line, col):
 		_launch_event('CursorMovedI')
 
 
-@_logged
+@_vim
 def _get_buffer():
 	return buffers[_buffer()]
 
 
-@_logged
+@_vim
 def _set_bufoption(option, value, bufnr=None):
 	_buf_options[bufnr or _buffer()][option] = value
 	if option == 'filetype':
@@ -440,7 +528,7 @@ class _WithNewBuffer(object):
 		_bw(self.bufnr)
 
 
-@_logged
+@_vim
 def _set_dict(d, new, setfunc=None):
 	if not setfunc:
 		def setfunc(k, v):
@@ -496,7 +584,7 @@ class _WithDict(object):
 			self.d.pop(k)
 
 
-@_logged
+@_vim
 def _with(key, *args, **kwargs):
 	if key == 'buffer':
 		return _WithNewBuffer(_edit, *args, **kwargs)