Use proper encoding to convert values in various places

Fixes #1072

docs/source/develop/segments.rst

@@ -371,6 +371,10 @@ Vim ``segment_info`` argument is a dictionary with the following keys:
 ``mode``
     Current mode.
 
+``encoding``
+    Value of ``&encoding`` from the time when powerline was initialized. It 
+    should be used to convert return values.
+
 .. note::
     Your segment generally should not assume that it is run for the current 
     window, current buffer or current tabpage. “Current window” and “current 

powerline/bindings/vim/__init__.py

@@ -13,13 +13,20 @@ if not hasattr(vim, 'bindeval'):
 	import json
 
 
+try:
+	vim_encoding = vim.eval('&encoding')
+except AttributeError:
+	vim_encoding = 'utf-8'
+
+
 if hasattr(vim, 'bindeval'):
 	def vim_get_func(f, rettype=None):
 		'''Return a vim function binding.'''
 		try:
 			func = vim.bindeval('function("' + f + '")')
 			if sys.version_info >= (3,) and rettype is str:
-				return (lambda *args, **kwargs: func(*args, **kwargs).decode('utf-8', errors='replace'))
+				return (lambda *args, **kwargs: func(*args, **kwargs).decode(
+					vim_encoding, errors='powerline_vim_strtrans_error'))
 			return func
 		except vim.error:
 			return None
@@ -102,7 +109,7 @@ else:
 if sys.version_info < (3,):
 	getbufvar = _getbufvar
 else:
-	_vim_to_python_types[bytes] = lambda value: value.decode('utf-8')
+	_vim_to_python_types[bytes] = lambda value: value.decode(vim_encoding)
 
 	def getbufvar(*args):
 		return _vim_to_python(_getbufvar(*args))
@@ -256,18 +263,18 @@ class VimEnviron(object):
 
 
 if sys.version_info < (3,):
-	def buffer_name(buf):
-		return buf.name
+	def buffer_name(segment_info):
+		return segment_info['buffer'].name
 else:
 	vim_bufname = vim_get_func('bufname')
 
-	def buffer_name(buf):
+	def buffer_name(segment_info):
 		try:
-			name = buf.name
+			name = segment_info['buffer'].name
 		except UnicodeDecodeError:
-			return vim_bufname(buf.number)
+			return vim_bufname(segment_info['bufnr'])
 		else:
-			return name.encode('utf-8') if name else None
+			return name.encode(segment_info['encoding']) if name else None
 
 
 vim_strtrans = vim_get_func('strtrans')

powerline/bindings/zsh/__init__.py

@@ -4,6 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_funct
 import atexit
 
 from weakref import WeakValueDictionary, ref
+from locale import getpreferredencoding
 
 import zsh
 
@@ -65,7 +66,7 @@ class Args(object):
 
 def string(s):
 	if type(s) is bytes:
-		return s.decode('utf-8', 'replace')
+		return s.decode(getpreferredencoding(), 'replace')
 	else:
 		return str(s)
 
@@ -154,9 +155,9 @@ class Prompt(object):
 		)
 		if type(r) is not str:
 			if type(r) is bytes:
-				return r.decode('utf-8')
+				return r.decode(getpreferredencoding(), 'replace')
 			else:
-				return r.encode('utf-8')
+				return r.encode(getpreferredencoding(), 'replace')
 		return r
 
 	def __del__(self):

powerline/lib/inotify.py

@@ -123,8 +123,6 @@ class INotify(object):
 		self._buf = ctypes.create_string_buffer(5000)
 		self.fenc = sys.getfilesystemencoding() or 'utf-8'
 		self.hdr = struct.Struct(b'iIII')
-		if self.fenc == 'ascii':
-			self.fenc = 'utf-8'
 		# We keep a reference to os to prevent it from being deleted
 		# during interpreter shutdown, which would lead to errors in the
 		# __del__ method

powerline/lib/vcs/bzr.py

@@ -6,6 +6,7 @@ import os
 import re
 
 from io import StringIO
+from locale import getpreferredencoding
 
 from bzrlib import (workingtree, status, library_state, trace, ui)
 
@@ -15,7 +16,7 @@ from powerline.lib.vcs import get_branch_name, get_file_status
 class CoerceIO(StringIO):
 	def write(self, arg):
 		if isinstance(arg, bytes):
-			arg = arg.decode('utf-8', 'replace')
+			arg = arg.decode(getpreferredencoding(), 'replace')
 		return super(CoerceIO, self).write(arg)
 
 
@@ -29,7 +30,7 @@ def branch_name_from_config_file(directory, config_file):
 			for line in f:
 				m = nick_pat.match(line)
 				if m is not None:
-					ans = m.group(1).strip().decode('utf-8', 'replace')
+					ans = m.group(1).strip().decode(getpreferredencoding(), 'replace')
 					break
 	except Exception:
 		pass

powerline/lib/vcs/git.py

@@ -5,6 +5,8 @@ import os
 import sys
 import re
 
+from locale import getpreferredencoding
+
 from powerline.lib.vcs import get_branch_name, get_file_status
 from powerline.lib.shell import readlines
 
@@ -20,7 +22,7 @@ def branch_name_from_config_file(directory, config_file):
 		return os.path.basename(directory)
 	m = _ref_pat.match(raw)
 	if m is not None:
-		return m.group(1).decode('utf-8', 'replace')
+		return m.group(1).decode(getpreferredencoding(), 'replace')
 	return raw[:7]
 
 

powerline/lib/vcs/mercurial.py

@@ -3,6 +3,8 @@ from __future__ import (unicode_literals, division, absolute_import, print_funct
 
 import os
 
+from locale import getpreferredencoding
+
 from mercurial import hg, ui, match
 
 from powerline.lib.vcs import get_branch_name, get_file_status
@@ -12,7 +14,7 @@ def branch_name_from_config_file(directory, config_file):
 	try:
 		with open(config_file, 'rb') as f:
 			raw = f.read()
-		return raw.decode('utf-8', 'replace').strip()
+		return raw.decode(getpreferredencoding(), 'replace').strip()
 	except Exception:
 		return 'default'
 

powerline/renderers/vim.py

@@ -42,6 +42,7 @@ class VimRenderer(Renderer):
 		self.hl_groups = {}
 		self.prev_highlight = None
 		self.strwidth_error_name = register_strwidth_error(self.strwidth)
+		self.encoding = vim.eval('&encoding')
 
 	def shutdown(self):
 		self.theme.shutdown()
@@ -72,11 +73,10 @@ class VimRenderer(Renderer):
 
 	if hasattr(vim, 'strwidth'):
 		if sys.version_info < (3,):
-			@staticmethod
-			def strwidth(string):
+			def strwidth(self, string):
 				# Does not work with tabs, but neither is strwidth from default 
 				# renderer
-				return vim.strwidth(string.encode('utf-8'))
+				return vim.strwidth(string.encode(self.encoding, 'replace'))
 		else:
 			@staticmethod
 			def strwidth(string):
@@ -102,6 +102,7 @@ class VimRenderer(Renderer):
 			winnr=winnr,
 			buffer=window.buffer,
 			tabpage=current_tabpage(),
+			encoding=self.encoding,
 		)
 		segment_info['tabnr'] = segment_info['tabpage'].number
 		segment_info['bufnr'] = segment_info['buffer'].number
@@ -116,7 +117,7 @@ class VimRenderer(Renderer):
 			segment_info=segment_info,
 			matcher_info=(None if is_tabline else segment_info),
 		)
-		statusline = statusline.encode(vim.eval('&encoding'), self.strwidth_error_name)
+		statusline = statusline.encode(self.encoding, self.strwidth_error_name)
 		return statusline
 
 	def reset_highlight(self):

powerline/segments/vim/__init__.py

@@ -225,7 +225,7 @@ def file_scheme(pl, segment_info):
 		name will look like :file:`zipfile:/path/to/archive.zip::file.txt`. 
 		``file_scheme`` segment will catch ``zipfile`` part here.
 	'''
-	name = buffer_name(segment_info['buffer'])
+	name = buffer_name(segment_info)
 	if not name:
 		return None
 	match = SCHEME_RE.match(name)
@@ -254,7 +254,7 @@ def file_directory(pl, segment_info, remove_scheme=True, shorten_user=True, shor
 		Shorten all directories in :file:`/home/` to :file:`~user/` instead of 
 		:file:`/home/user/`. Does not work for files with scheme present.
 	'''
-	name = buffer_name(segment_info['buffer'])
+	name = buffer_name(segment_info)
 	if not name:
 		return None
 	match = SCHEME_RE.match(name)
@@ -271,7 +271,7 @@ def file_directory(pl, segment_info, remove_scheme=True, shorten_user=True, shor
 			return None
 		if shorten_home and file_directory.startswith('/home/'):
 			file_directory = b'~' + file_directory[6:]
-	file_directory = file_directory.decode('utf-8', 'powerline_vim_strtrans_error')
+	file_directory = file_directory.decode(segment_info['encoding'], 'powerline_vim_strtrans_error')
 	return file_directory + os.sep
 
 
@@ -286,7 +286,7 @@ def file_name(pl, segment_info, display_no_file=False, no_file_text='[No file]')
 
 	Highlight groups used: ``file_name_no_file`` or ``file_name``, ``file_name``.
 	'''
-	name = buffer_name(segment_info['buffer'])
+	name = buffer_name(segment_info)
 	if not name:
 		if display_no_file:
 			return [{
@@ -295,7 +295,7 @@ def file_name(pl, segment_info, display_no_file=False, no_file_text='[No file]')
 			}]
 		else:
 			return None
-	return os.path.basename(name).decode('utf-8', 'powerline_vim_strtrans_error')
+	return os.path.basename(name).decode(segment_info['encoding'], 'powerline_vim_strtrans_error')
 
 
 @window_cached

tests/vim.py

@@ -651,6 +651,7 @@ def _get_segment_info():
 		'tabnr': tabpage.number,
 		'window_id': window._window_id,
 		'mode': mode,
+		'encoding': options['encoding'],
 	}