Make powerline work with non-utf filenames

Fixes #281

powerline/bindings/vim/__init__.py

@@ -1,6 +1,7 @@
 # vim:fileencoding=utf-8:noet
 
 import sys
+import codecs
 
 try:
 	import vim
@@ -113,4 +114,34 @@ class VimEnviron(object):
 					+ value.replace('"', '\\"').replace('\\', '\\\\').replace('\n', '\\n').replace('\0', '')
 					+ '"')
 
+
+if sys.version_info < (3,):
+	def buffer_name(buf):
+		return buf.name
+else:
+	vim_bufname = vim_get_func('bufname')
+	def buffer_name(buf):  # NOQA
+		try:
+			name = buf.name
+		except UnicodeDecodeError:
+			return vim_bufname(buf.number)
+		else:
+			return name.encode('utf-8') if name else None
+
+
+vim_strtrans = vim_get_func('strtrans')
+
+
+def powerline_vim_strtrans_error(e):
+	if not isinstance(e, UnicodeDecodeError):
+		raise NotImplementedError
+	# Assuming &encoding is utf-8 strtrans should not return anything but ASCII 
+	# under current circumstances
+	text = vim_strtrans(e.object[e.start:e.end]).decode()
+	return (text, e.end)
+
+
+codecs.register_error('powerline_vim_strtrans_error', powerline_vim_strtrans_error)
+
+
 environ = VimEnviron()

powerline/segments/common.py

@@ -1,6 +1,6 @@
 # vim:fileencoding=utf-8:noet
 
-from __future__ import absolute_import
+from __future__ import unicode_literals, absolute_import
 
 import os
 import sys

powerline/segments/vim.py

@@ -1,6 +1,6 @@
 # vim:fileencoding=utf-8:noet
 
-from __future__ import absolute_import, division
+from __future__ import unicode_literals, absolute_import, division
 
 import os
 try:
@@ -8,7 +8,8 @@ try:
 except ImportError:
 	vim = {}  # NOQA
 
-from powerline.bindings.vim import vim_get_func, getbufvar, vim_getbufoption
+from powerline.bindings.vim import (vim_get_func, getbufvar, vim_getbufoption,
+									buffer_name)
 from powerline.theme import requires_segment_info
 from powerline.lib import add_divider_highlight_group
 from powerline.lib.vcs import guess, tree_status
@@ -19,8 +20,8 @@ from collections import defaultdict
 vim_funcs = {
 	'virtcol': vim_get_func('virtcol', rettype=int),
 	'getpos': vim_get_func('getpos'),
-	'fnamemodify': vim_get_func('fnamemodify', rettype=str),
-	'expand': vim_get_func('expand', rettype=str),
+	'fnamemodify': vim_get_func('fnamemodify'),
+	'expand': vim_get_func('expand'),
 	'bufnr': vim_get_func('bufnr', rettype=int),
 	'line2byte': vim_get_func('line2byte', rettype=int),
 	'line': vim_get_func('line', rettype=int),
@@ -157,14 +158,18 @@ def file_directory(pl, segment_info, shorten_user=True, shorten_cwd=True, shorte
 	:param bool shorten_home:
 		shorten all directories in :file:`/home/` to :file:`~user/` instead of :file:`/home/user/`.
 	'''
-	name = segment_info['buffer'].name
+	name = buffer_name(segment_info['buffer'])
 	if not name:
 		return None
+	import sys
 	file_directory = vim_funcs['fnamemodify'](name, (':~' if shorten_user else '')
 												+ (':.' if shorten_cwd else '') + ':h')
+	if not file_directory:
+		return None
 	if shorten_home and file_directory.startswith('/home/'):
-		file_directory = '~' + file_directory[6:]
-	return file_directory + os.sep if file_directory else None
+		file_directory = b'~' + file_directory[6:]
+	file_directory = file_directory.decode('utf-8', 'powerline_vim_strtrans_error')
+	return file_directory + os.sep
 
 
 @requires_segment_info
@@ -178,7 +183,7 @@ def file_name(pl, segment_info, display_no_file=False, no_file_text='[No file]')
 
 	Highlight groups used: ``file_name_no_file`` or ``file_name``, ``file_name``.
 	'''
-	name = segment_info['buffer'].name
+	name = buffer_name(segment_info['buffer'])
 	if not name:
 		if display_no_file:
 			return [{
@@ -187,8 +192,7 @@ def file_name(pl, segment_info, display_no_file=False, no_file_text='[No file]')
 			}]
 		else:
 			return None
-	file_name = vim_funcs['fnamemodify'](name, ':~:.:t')
-	return file_name
+	return os.path.basename(name).decode('utf-8', 'powerline_vim_strtrans_error')
 
 
 @window_cached

tests/test_segments.py

@@ -1,5 +1,7 @@
 # vim:fileencoding=utf-8:noet
 
+from __future__ import unicode_literals
+
 from powerline.segments import shell, common
 import tests.vim as vim_module
 import sys
@@ -467,6 +469,10 @@ class TestVim(TestCase):
 		segment_info = vim_module._get_segment_info()
 		self.assertEqual(vim.file_directory(pl=pl, segment_info=segment_info), None)
 		with replace_env('HOME', '/home/foo', os.environ):
+			with vim_module._with('buffer', '/tmp/’’/abc') as segment_info:
+				self.assertEqual(vim.file_directory(pl=pl, segment_info=segment_info), '/tmp/’’/')
+			with vim_module._with('buffer', b'/tmp/\xFF\xFF/abc') as segment_info:
+				self.assertEqual(vim.file_directory(pl=pl, segment_info=segment_info), '/tmp/<ff><ff>/')
 			with vim_module._with('buffer', '/tmp/abc') as segment_info:
 				self.assertEqual(vim.file_directory(pl=pl, segment_info=segment_info), '/tmp/')
 				os.environ['HOME'] = '/tmp'
@@ -484,6 +490,8 @@ class TestVim(TestCase):
 			self.assertEqual(vim.file_name(pl=pl, segment_info=segment_info), 'abc')
 		with vim_module._with('buffer', '/tmp/’’') as segment_info:
 			self.assertEqual(vim.file_name(pl=pl, segment_info=segment_info), '’’')
+		with vim_module._with('buffer', b'/tmp/\xFF\xFF') as segment_info:
+			self.assertEqual(vim.file_name(pl=pl, segment_info=segment_info), '<ff><ff>')
 
 	def test_file_size(self):
 		pl = Pl()

tests/vim.py

@@ -60,7 +60,7 @@ class _Buffers(object):
 
 	@_vim
 	def __nonzero__(self):
-		return not not self.d
+		return bool(self.d)
 
 	@_vim
 	def keys(self):
@@ -261,8 +261,8 @@ def _emul_getpos(expr):
 def _emul_fnamemodify(path, modstring):
 	import os
 	_modifiers = {
-		'~': lambda path: path.replace(os.environ['HOME'], '~') if path.startswith(os.environ['HOME']) else path,
-		'.': lambda path: (lambda tpath: path if tpath[:3] == '..' + os.sep else tpath)(os.path.relpath(path)),
+		'~': lambda path: path.replace(os.environ['HOME'].encode('utf-8'), b'~') if path.startswith(os.environ['HOME'].encode('utf-8')) else path,
+		'.': lambda path: (lambda tpath: path if tpath[:3] == b'..' + os.sep.encode() else tpath)(os.path.relpath(path)),
 		't': lambda path: os.path.basename(path),
 		'h': lambda path: os.path.dirname(path),
 	}
@@ -313,6 +313,22 @@ def _emul_line(expr):
 	raise NotImplementedError
 
 
+@_vim
+@_str_func
+def _emul_strtrans(s):
+	# FIXME Do more replaces
+	return s.replace(b'\xFF', b'<ff>')
+
+
+@_vim
+@_str_func
+def _emul_bufname(bufnr):
+	try:
+		return buffers[bufnr]._name or b''
+	except KeyError:
+		return b''
+
+
 _window_ids = [None]
 _window_id = 0
 
@@ -348,11 +364,11 @@ _undo_written = {}
 class _Buffer(object):
 	def __init__(self, name=None):
 		global _last_bufnr
-		import os
 		_last_bufnr += 1
 		bufnr = _last_bufnr
 		self.number = bufnr
-		self.name = os.path.abspath(name) if name else None
+		# FIXME Use unicode() for python-3
+		self.name = name
 		self.vars = {}
 		self.options = {
 			'modified': 0,
@@ -369,6 +385,25 @@ class _Buffer(object):
 		_undo_written[bufnr] = len(_undostate[bufnr])
 		buffers[bufnr] = self
 
+	@property
+	def name(self):
+		import sys
+		if sys.version_info < (3,):
+			return self._name
+		else:
+			return str(self._name, 'utf-8') if self._name else None
+
+	@name.setter
+	def name(self, name):
+		if name is None:
+			self._name = None
+		else:
+			import os
+			if type(name) is not bytes:
+				name = name.encode('utf-8')
+			import sys
+			self._name = os.path.abspath(name)
+
 	def __getitem__(self, line):
 		return _buf_lines[self.number][line]
 
@@ -676,3 +711,7 @@ def _with(key, *args, **kwargs):
 		return _WithDict(vars, **kwargs)
 	elif key == 'split':
 		return _WithSplit()
+
+
+class error(Exception):
+	pass