Merge pull request #1198 from ZyX-I/fix-shell-unicode-error

Fix shell unicode error when current directory name is not ASCII
This commit is contained in:
Nikolai Aleksandrovich Pavlov 2014-11-28 23:16:55 +03:00
commit ddac11a64a
25 changed files with 107 additions and 33 deletions

View File

@ -3,8 +3,20 @@
from __future__ import (division, absolute_import, print_function)
import argparse
import sys
from powerline.lib import mergedicts, parsedotval
from powerline.lib.encoding import get_preferred_arguments_encoding
if sys.version_info < (3,):
encoding = get_preferred_arguments_encoding()
def arg_to_unicode(s):
return unicode(s, encoding, 'replace') if not isinstance(s, unicode) else s
else:
def arg_to_unicode(s):
return s
def mergeargs(argvalue):
@ -39,9 +51,9 @@ def get_argparser(ArgumentParser=argparse.ArgumentParser):
parser.add_argument('--last_exit_code', metavar='INT', type=int, help='Last exit code.')
parser.add_argument('--last_pipe_status', metavar='LIST', default='', type=lambda s: [int(status) for status in s.split()], help='Like above, but is supposed to contain space-separated array of statuses, representing exit statuses of commands in one pipe.')
parser.add_argument('--jobnum', metavar='INT', type=int, help='Number of jobs.')
parser.add_argument('-c', '--config', metavar='KEY.KEY=VALUE', action='append', help='Configuration overrides for `config.json\'. Is translated to a dictionary and merged with the dictionary obtained from actual JSON configuration: KEY.KEY=VALUE is translated to `{"KEY": {"KEY": VALUE}}\' and then merged recursively. VALUE may be any JSON value, values that are not `null\', `true\', `false\', start with digit, `{\', `[\' are treated like strings. If VALUE is omitted then corresponding key is removed.')
parser.add_argument('-t', '--theme_option', metavar='THEME.KEY.KEY=VALUE', action='append', help='Like above, but theme-specific. THEME should point to an existing and used theme to have any effect, but it is fine to use any theme here.')
parser.add_argument('-R', '--renderer_arg', metavar='KEY=VAL', action='append', help='Like above, but provides argument for renderer. Is supposed to be used only by shell bindings to provide various data like last_exit_code or last_pipe_status (they are not using `--renderer_arg\' for historical resons: `--renderer_arg\' was added later).')
parser.add_argument('-c', '--config', metavar='KEY.KEY=VALUE', type=arg_to_unicode, action='append', help='Configuration overrides for `config.json\'. Is translated to a dictionary and merged with the dictionary obtained from actual JSON configuration: KEY.KEY=VALUE is translated to `{"KEY": {"KEY": VALUE}}\' and then merged recursively. VALUE may be any JSON value, values that are not `null\', `true\', `false\', start with digit, `{\', `[\' are treated like strings. If VALUE is omitted then corresponding key is removed.')
parser.add_argument('-t', '--theme_option', metavar='THEME.KEY.KEY=VALUE', type=arg_to_unicode, action='append', help='Like above, but theme-specific. THEME should point to an existing and used theme to have any effect, but it is fine to use any theme here.')
parser.add_argument('-R', '--renderer_arg', metavar='KEY=VAL', type=arg_to_unicode, action='append', help='Like above, but provides argument for renderer. Is supposed to be used only by shell bindings to provide various data like last_exit_code or last_pipe_status (they are not using `--renderer_arg\' for historical resons: `--renderer_arg\' was added later).')
parser.add_argument('-p', '--config_path', action='append', metavar='PATH', help='Path to configuration directory. If it is present then configuration files will only be seeked in the provided path. May be provided multiple times to search in a list of directories.')
parser.add_argument('--socket', metavar='ADDRESS', type=str, help='Socket address to use in daemon clients. Is always UNIX domain socket on linux and file socket on Mac OS X. Not used here, present only for compatibility with other powerline clients. This argument must always be the first one and be in a form `--socket ADDRESS\': no `=\' or short form allowed (in other powerline clients, not here).')
return parser

View File

@ -64,6 +64,21 @@ def get_preferred_input_encoding():
)
def get_preferred_arguments_encoding():
'''Get encoding that should be used for command-line arguments
.. warning::
Falls back to latin1 so that function is less likely to throw as
non-ASCII command-line arguments most likely contain non-ASCII
filenames and screwing them up due to unidentified locale is not much of
a problem.
'''
return (
locale.getdefaultlocale()[1]
or 'latin1'
)
def get_preferred_environment_encoding():
'''Get encoding that should be used for decoding environment variables
'''

View File

@ -75,11 +75,20 @@ class Renderer(object):
See documentation of ``unicode.translate`` for details.
'''
np_character_translations = dict(((i, '^' + unichr(i + 0x40)) for i in range(0x20)))
np_character_translations = dict(chain(
# Control characters: ^@ … ^Y
((i1, '^' + unichr(i1 + 0x40)) for i1 in range(0x20)),
# Invalid unicode characters obtained using 'surrogateescape' error
# handler.
((i2, '<{0:02x}>'.format(i2 - 0xDC00)) for i2 in range(0xDC80, 0xDD00)),
))
'''Non-printable character translations
These are used to transform characters in range 0x000x1F into ``^@``,
``^A`` and so on. Unilke with ``.escape()`` method (and
``^A`` and so on and characters in range 0xDC800xDCFF into ``<80>``,
``<81>`` and so on (latter are invalid characters obtained using
``surrogateescape`` error handling method used automatically in a number of
places in Python3). Unilke with ``.escape()`` method (and
``character_translations``) result is passed to ``.strwidth()`` method.
Note: transforms tab into ``^I``.

View File

@ -10,6 +10,7 @@ class TcshPromptRenderer(ZshPromptRenderer):
character_translations[ord('%')] = '%%'
character_translations[ord('\\')] = '\\\\'
character_translations[ord('^')] = '\\^'
character_translations[ord('!')] = '\\!'
renderer = TcshPromptRenderer

View File

@ -25,5 +25,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -25,5 +25,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -24,5 +24,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -24,5 +24,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -23,5 +23,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -23,5 +23,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -14,22 +14,23 @@
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  (echo)  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  $(echo)  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  
 USER  ⋯  shell  3rd  `echo`  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  «Unicode!»  
 USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
   BRANCH 
   BRANCH 
 INSERT  USER  ⋯  shell  3rd  `echo`  
 INSERT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
 DEFAULT  USER  ⋯  shell  3rd  `echo`  
 DEFAULT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
 INSERT  USER  ⋯  shell  3rd  `echo`  
 INSERT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
 DEFAULT  USER  ⋯  shell  3rd  `echo`  
 DEFAULT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
 INSERT  USER  ⋯  shell  3rd  `echo`  
 INSERT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
   BRANCH 
 INSERT  USER  ⋯  shell  3rd  `echo`  
 INSERT  USER  ⋯  shell  3rd  «Unicode!»  
   BRANCH 
   BRANCH 
   BRANCH 

View File

@ -24,6 +24,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
false
true is the last line

View File

@ -24,6 +24,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
false
true is the last line

View File

@ -24,6 +24,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
false
true is the last line

View File

@ -22,6 +22,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
set POWERLINE_COMMAND "$POWERLINE_COMMAND -c ext.shell.theme=default"
set -g fish_key_bindings fish_vi_key_bindings
ii

View File

@ -25,6 +25,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
false
true is the last line

View File

@ -17,6 +17,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode\!»'
false
true is the last line
exit

View File

@ -26,6 +26,7 @@ cd ../'#[bold]'
cd ../'(echo)'
cd ../'$(echo)'
cd ../'`echo`'
cd ../'«Unicode!»'
cd ..
POWERLINE_COMMAND="${POWERLINE_COMMAND//_leftonly}" ; bindkey -v


View File

@ -27,5 +27,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -27,5 +27,6 @@ def
  BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  BRANCH  ⋯  shell  3rd  `echo`  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  `echo` $ABCfalse
  BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  BRANCH  ⋯  shell  3rd  «Unicode!»  POWERLINE_COMMAND="$POWERLINE_COMMAND -t default_leftonly.dividers.left.hard=\$ABC"
  BRANCH $ABC⋯  shell  3rd  «Unicode!» $ABCfalse

View File

@ -1,3 +1,7 @@
width 1024
height 1
logfile "tests/shell/${SH}.${TEST_TYPE}.${TEST_CLIENT}.full.log"
# Having utf8 setting on causes screen to recode file passed to readreg. Adding
# `-e utf8` just after `readreg` causes screen to fail with `-X: copyreg:
# character, ^x, or (octal) \032 expected.` in place of fixing the issue.
defutf8 off

View File

@ -14,3 +14,4 @@
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  (echo)  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  $(echo)  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  «Unicode!»  

View File

@ -1,5 +1,6 @@
#!/bin/sh
: ${PYTHON:=python}
FAIL_SUMMARY=""
FAILED=0
if test "x$1" = "x--fast" ; then
FAST=1
@ -78,6 +79,10 @@ run() {
"$@"
}
# HACK: get newline for use in strings given that "\n" and $'' do not work.
NL="$(printf '\nE')"
NL="${NL%E}"
run_test() {
TEST_TYPE="$1"
shift
@ -105,7 +110,7 @@ run_test() {
# …
# prompt1> prompt2> …
while read -r line ; do
screen -S "$SESNAME" -p 0 -X stuff "$line"$(printf '\r')
screen -S "$SESNAME" -p 0 -X stuff "$line$NL"
sleep 1
done < tests/test_shells/input.$SH
else
@ -173,6 +178,7 @@ mkdir tests/shell/3rd/'#[bold]'
mkdir tests/shell/3rd/'(echo)'
mkdir tests/shell/3rd/'$(echo)'
mkdir tests/shell/3rd/'`echo`'
mkdir tests/shell/3rd/'«Unicode!»'
mkdir tests/shell/fish_home
mkdir tests/shell/fish_home/fish
@ -316,6 +322,7 @@ if test -z "${ONLY_SHELL}" || test "x${ONLY_SHELL%sh}" != "x${ONLY_SHELL}" || te
echo ">>> $(which $SH)"
if ! run_test $TEST_TYPE $TEST_CLIENT $TEST_COMMAND ; then
FAILED=1
FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T ${TEST_TYPE} ${TEST_CLIENT} ${TEST_COMMAND}"
fi
done
done
@ -328,6 +335,7 @@ if test -z "${ONLY_SHELL}" || test "x${ONLY_SHELL%sh}" != "x${ONLY_SHELL}" || te
echo '============================================================'
cat tests/shell/daemon_log
FAILED=1
FAIL_SUMMARY="${FAIL_SUMMARY}${NL}L ${TEST_TYPE} ${TEST_CLIENT} ${TEST_COMMAND}"
fi
fi
done
@ -339,15 +347,16 @@ if $PYTHON scripts/powerline-daemon -s$ADDRESS > tests/shell/daemon_log_2 2>&1 ;
else
echo "Daemon exited with status $?"
FAILED=1
FAIL_SUMMARY="${FAIL_SUMMARY}${NL}D"
fi
if ! test -z "$(cat tests/shell/daemon_log_2)" ; then
FAILED=1
echo '____________________________________________________________'
echo "Daemon log (2nd):"
echo '============================================================'
cat tests/shell/daemon_log_2
FAILED=1
FAIL_SUMMARY="${FAIL_SUMMARY}${NL}L"
fi
if test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xipython" ; then
@ -355,9 +364,14 @@ if test "x${ONLY_SHELL}" = "x" || test "x${ONLY_SHELL}" = "xipython" ; then
echo "> $(which ipython)"
if ! run_test ipython ipython ipython ; then
FAILED=1
FAIL_SUMMARY="${FAIL_SUMMARY}${NL}T ipython"
fi
fi
fi
test $FAILED -eq 0 && rm -r tests/shell
if test $FAILED -eq 0 ; then
rm -r tests/shell
else
echo "${FAIL_SUMMARY}"
fi
exit $FAILED

View File

@ -16,7 +16,8 @@
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  cd ..
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  «Unicode!»  cd ..
  HOSTNAME  USER   BRANCH  ⋯  tests  shell  3rd  POWERLINE_COMMAND="${POWERLINE_COMMAND//_leftonly}" ; bindkey -v
 INSERT   HOSTNAME  USER  ⋯  tests  shell  3rd   COMMND   HOSTNAME  USER  ⋯  tests  shell  3rd  
 INSERT   HOSTNAME  USER  ⋯  tests  shell  3rd  

View File

@ -16,7 +16,8 @@
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  #[bold]  cd ../'(echo)'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  (echo)  cd ../'$(echo)'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  $(echo)  cd ../'`echo`'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  cd ..
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  `echo`  cd ../'«Unicode!»'
  HOSTNAME  USER   BRANCH  ⋯  shell  3rd  «Unicode!»  cd ..
  HOSTNAME  USER   BRANCH  ⋯  tests  shell  3rd  POWERLINE_COMMAND="${POWERLINE_COMMAND//_leftonly}" ; bindkey -v
 INSERT   HOSTNAME  USER  ⋯  tests  shell  3rd   COMMND   HOSTNAME  USER  ⋯  tests  shell  3rd  
 INSERT   HOSTNAME  USER  ⋯  tests  shell  3rd 