From ee78221af46e6235571b18869600ea753331e095 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 7 Jan 2015 19:55:00 +0300 Subject: [PATCH 1/3] Save information about old values of values --- powerline/lint/markedjson/error.py | 46 +++++++++++++++++++----- powerline/lint/markedjson/markedvalue.py | 15 ++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/powerline/lint/markedjson/error.py b/powerline/lint/markedjson/error.py index 7ee65217..1d8efdcd 100644 --- a/powerline/lint/markedjson/error.py +++ b/powerline/lint/markedjson/error.py @@ -42,15 +42,16 @@ def strtrans(s): class Mark: - def __init__(self, name, line, column, buffer, pointer): + def __init__(self, name, line, column, buffer, pointer, old_mark=None): self.name = name self.line = line self.column = column self.buffer = buffer self.pointer = pointer + self.old_mark = old_mark def copy(self): - return Mark(self.name, self.line, self.column, self.buffer, self.pointer) + return Mark(self.name, self.line, self.column, self.buffer, self.pointer, self.old_mark) def get_snippet(self, indent=4, max_length=75): if self.buffer is None: @@ -85,17 +86,46 @@ class Mark: ret.pointer += diff return ret - def __str__(self): - snippet = self.get_snippet() - where = (' in "%s", line %d, column %d' % ( - self.name, self.line + 1, self.column + 1)) - if snippet is not None: - where += ':\n' + snippet + def set_old_mark(self, old_mark): + if self is old_mark: + return + checked_marks = set([id(self)]) + older_mark = old_mark + while True: + if id(older_mark) in checked_marks: + raise ValueError('Trying to set recursive marks') + checked_marks.add(id(older_mark)) + older_mark = older_mark.old_mark + if not older_mark: + break + self.old_mark = old_mark + + def to_string(self, indent=0): + mark = self + where = '' + processed_marks = set() + while mark: + indentstr = ' ' * indent + snippet = mark.get_snippet(indent=(indent + 4)) + where += (indentstr + ' in "%s", line %d, column %d' % ( + mark.name, mark.line + 1, mark.column + 1)) + if snippet: + where += ':\n' + snippet + processed_marks.add(id(mark)) + if mark.old_mark: + where += '\n' + indentstr + ' which replaced value\n' + indent += 4 + mark = mark.old_mark + if id(mark) in processed_marks: + raise ValueError('Trying to dump recursive mark') if type(where) is str: return where else: return where.encode('utf-8') + def __str__(self): + return self.to_string() + def echoerr(*args, **kwargs): stream = kwargs.pop('stream', sys.stderr) diff --git a/powerline/lint/markedjson/markedvalue.py b/powerline/lint/markedjson/markedvalue.py index c17a8e35..7e3a742a 100644 --- a/powerline/lint/markedjson/markedvalue.py +++ b/powerline/lint/markedjson/markedvalue.py @@ -66,6 +66,21 @@ class MarkedDict(dict): return r def __setitem__(self, key, value): + try: + old_value = self[key] + except KeyError: + pass + else: + try: + key.mark.set_old_mark(self.keydict[key].mark) + except AttributeError: + pass + except KeyError: + pass + try: + value.mark.set_old_mark(old_value.mark) + except AttributeError: + pass dict.__setitem__(self, key, value) self.keydict[key] = key From 7d6b9c5c5138b62129d9b0759ac979f5c86ebf22 Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 7 Jan 2015 20:02:25 +0300 Subject: [PATCH 2/3] Also save what was merged in --- powerline/lib/dict.py | 7 +++++ powerline/lint/markedjson/error.py | 36 ++++++++++++++++-------- powerline/lint/markedjson/markedvalue.py | 6 ++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/powerline/lib/dict.py b/powerline/lib/dict.py index 76728aba..d8d2088b 100644 --- a/powerline/lib/dict.py +++ b/powerline/lib/dict.py @@ -35,6 +35,7 @@ def mergedicts(d1, d2, remove=True): First dictionary is modified in-place. ''' + _setmerged(d1, d2) for k in d2: if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): mergedicts(d1[k], d2[k], remove) @@ -58,6 +59,11 @@ def mergedefaults(d1, d2): d1.setdefault(k, d2[k]) +def _setmerged(d1, d2): + if hasattr(d1, 'setmerged'): + d1.setmerged(d2) + + def mergedicts_copy(d1, d2): '''Recursively merge two dictionaries. @@ -65,6 +71,7 @@ def mergedicts_copy(d1, d2): that first dictionary supports .copy() method. ''' ret = d1.copy() + _setmerged(ret, d2) for k in d2: if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict): ret[k] = mergedicts_copy(d1[k], d2[k]) diff --git a/powerline/lint/markedjson/error.py b/powerline/lint/markedjson/error.py index 1d8efdcd..d63c638a 100644 --- a/powerline/lint/markedjson/error.py +++ b/powerline/lint/markedjson/error.py @@ -42,16 +42,17 @@ def strtrans(s): class Mark: - def __init__(self, name, line, column, buffer, pointer, old_mark=None): + def __init__(self, name, line, column, buffer, pointer, old_mark=None, merged_marks=None): self.name = name self.line = line self.column = column self.buffer = buffer self.pointer = pointer self.old_mark = old_mark + self.merged_marks = merged_marks or [] def copy(self): - return Mark(self.name, self.line, self.column, self.buffer, self.pointer, self.old_mark) + return Mark(self.name, self.line, self.column, self.buffer, self.pointer, self.old_mark, self.merged_marks[:]) def get_snippet(self, indent=4, max_length=75): if self.buffer is None: @@ -100,21 +101,32 @@ class Mark: break self.old_mark = old_mark - def to_string(self, indent=0): + def set_merged_mark(self, merged_mark): + self.merged_marks.append(merged_mark) + + def to_string(self, indent=0, head_text='in ', add_snippet=True): mark = self where = '' processed_marks = set() while mark: indentstr = ' ' * indent - snippet = mark.get_snippet(indent=(indent + 4)) - where += (indentstr + ' in "%s", line %d, column %d' % ( - mark.name, mark.line + 1, mark.column + 1)) - if snippet: - where += ':\n' + snippet - processed_marks.add(id(mark)) - if mark.old_mark: - where += '\n' + indentstr + ' which replaced value\n' - indent += 4 + where += ('%s %s"%s", line %d, column %d' % ( + indentstr, head_text, mark.name, mark.line + 1, mark.column + 1)) + if add_snippet: + snippet = mark.get_snippet(indent=(indent + 4)) + if snippet: + where += ':\n' + snippet + if mark.merged_marks: + where += '\n' + indentstr + ' with additionally merged\n' + where += mark.merged_marks[0].to_string(indent + 4, head_text='', add_snippet=False) + for mmark in mark.merged_marks[1:]: + where += '\n' + indentstr + ' and\n' + where += mmark.to_string(indent + 4, head_text='', add_snippet=False) + if add_snippet: + processed_marks.add(id(mark)) + if mark.old_mark: + where += '\n' + indentstr + ' which replaced value\n' + indent += 4 mark = mark.old_mark if id(mark) in processed_marks: raise ValueError('Trying to dump recursive mark') diff --git a/powerline/lint/markedjson/markedvalue.py b/powerline/lint/markedjson/markedvalue.py index 7e3a742a..3b8db3e8 100644 --- a/powerline/lint/markedjson/markedvalue.py +++ b/powerline/lint/markedjson/markedvalue.py @@ -65,6 +65,12 @@ class MarkedDict(dict): r.keydict = dict(((key, key) for key in r)) return r + def setmerged(self, d): + try: + self.mark.set_merged_mark(d.mark) + except AttributeError: + pass + def __setitem__(self, key, value): try: old_value = self[key] From 7eed352a50875bbaac7a0805bc400bce768c424f Mon Sep 17 00:00:00 2001 From: ZyX Date: Wed, 7 Jan 2015 21:25:46 +0300 Subject: [PATCH 3/3] Rewrite file loading code Apparently old variant did not bother to check user configuration and in any case was unnecessary complex. --- powerline/lint/__init__.py | 221 ++++++++++++++++++++----------------- 1 file changed, 120 insertions(+), 101 deletions(-) diff --git a/powerline/lint/__init__.py b/powerline/lint/__init__.py index 91531849..d6ba0bfe 100644 --- a/powerline/lint/__init__.py +++ b/powerline/lint/__init__.py @@ -5,6 +5,7 @@ import os import logging from collections import defaultdict +from itertools import chain from functools import partial from powerline import generate_config_finder, get_config_paths, load_config @@ -12,6 +13,7 @@ from powerline.segments.vim import vim_modes from powerline.lib.dict import mergedicts_copy from powerline.lib.config import ConfigLoader from powerline.lib.unicode import unicode +from powerline.lib.path import join from powerline.lint.markedjson import load from powerline.lint.markedjson.error import echoerr, EchoErr, MarkedError from powerline.lint.checks import (check_matcher_func, check_ext, check_config, check_top_theme, @@ -294,6 +296,74 @@ def register_common_names(): register_common_name('player', 'powerline.segments.common.players', '_player') +def load_json_file(path): + with open_file(path) as F: + try: + config, hadproblem = load(F) + except MarkedError as e: + return True, None, str(e) + else: + return hadproblem, config, None + + +def updated_with_config(d): + hadproblem, config, error = load_json_file(d['path']) + d.update( + hadproblem=hadproblem, + config=config, + error=error, + ) + return d + + +def find_all_ext_config_files(search_paths, subdir): + for config_root in search_paths: + top_config_subpath = join(config_root, subdir) + if not os.path.isdir(top_config_subpath): + if os.path.exists(top_config_subpath): + yield { + 'error': 'Path {0} is not a directory'.format(top_config_subpath), + 'path': top_config_subpath, + } + continue + for ext_name in os.listdir(top_config_subpath): + ext_path = os.path.join(top_config_subpath, ext_name) + if not os.path.isdir(ext_path): + if ext_name.endswith('.json') and os.path.isfile(ext_path): + yield updated_with_config({ + 'error': False, + 'path': ext_path, + 'name': ext_name[:-5], + 'ext': None, + 'type': 'top_' + subdir, + }) + else: + yield { + 'error': 'Path {0} is not a directory or configuration file'.format(ext_path), + 'path': ext_path, + } + continue + for config_file_name in os.listdir(ext_path): + config_file_path = os.path.join(ext_path, config_file_name) + if config_file_name.endswith('.json') and os.path.isfile(config_file_path): + yield updated_with_config({ + 'error': False, + 'path': config_file_path, + 'name': config_file_name[:-5], + 'ext': ext_name, + 'type': subdir, + }) + else: + yield { + 'error': 'Path {0} is not a configuration file'.format(config_file_path), + 'path': config_file_path, + } + + +def dict2(d): + return defaultdict(dict, ((k, dict(v)) for k, v in d.items())) + + def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): '''Check configuration sanity @@ -313,6 +383,8 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): ``False`` if user configuration seems to be completely sane and ``True`` if some problems were found. ''' + hadproblem = False + register_common_names() search_paths = paths or get_config_paths() find_config_files = generate_config_finder(lambda: search_paths) @@ -337,65 +409,60 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): config_loader = ConfigLoader(run_once=True, load=load_json_config) - paths = { - 'themes': defaultdict(lambda: []), - 'colorschemes': defaultdict(lambda: []), - 'top_colorschemes': [], - 'top_themes': [], - } lists = { 'colorschemes': set(), 'themes': set(), 'exts': set(), } - for path in reversed(search_paths): - for typ in ('themes', 'colorschemes'): - d = os.path.join(path, typ) - if os.path.isdir(d): - for subp in os.listdir(d): - extpath = os.path.join(d, subp) - if os.path.isdir(extpath): - lists['exts'].add(subp) - paths[typ][subp].append(extpath) - elif extpath.endswith('.json'): - name = subp[:-5] - if name != '__main__': - lists[typ].add(name) - paths['top_' + typ].append(extpath) - else: + found_dir = { + 'themes': False, + 'colorschemes': False, + } + config_paths = defaultdict(lambda: defaultdict(dict)) + loaded_configs = defaultdict(lambda: defaultdict(dict)) + for d in chain( + find_all_ext_config_files(search_paths, 'colorschemes'), + find_all_ext_config_files(search_paths, 'themes'), + ): + if d['error']: + hadproblem = True + ee(problem=d['error']) + continue + if d['hadproblem']: + hadproblem = True + if d['ext']: + found_dir[d['type']] = True + lists['exts'].add(d['ext']) + if d['name'] == '__main__': + pass + elif d['name'].startswith('__') or d['name'].endswith('__'): hadproblem = True - ee(problem='Path {0} is supposed to be a directory, but it is not'.format(d)) + ee(problem='File name is not supposed to start or end with “__”: {0}'.format( + d['path'])) + else: + lists[d['type']].add(d['name']) + config_paths[d['type']][d['ext']][d['name']] = d['path'] + loaded_configs[d['type']][d['ext']][d['name']] = d['config'] + else: + config_paths[d['type']][d['name']] = d['path'] + loaded_configs[d['type']][d['name']] = d['config'] - hadproblem = False - - configs = defaultdict(lambda: defaultdict(lambda: {})) for typ in ('themes', 'colorschemes'): - for ext in paths[typ]: - for d in paths[typ][ext]: - for subp in os.listdir(d): - if subp.endswith('.json'): - name = subp[:-5] - if name != '__main__': - lists[typ].add(name) - if name.startswith('__') or name.endswith('__'): - hadproblem = True - ee(problem='File name is not supposed to start or end with “__”: {0}'.format( - os.path.join(d, subp) - )) - configs[typ][ext][name] = os.path.join(d, subp) - for path in paths['top_' + typ]: - name = os.path.basename(path)[:-5] - configs['top_' + typ][name] = path + if not found_dir[typ]: + hadproblem = True + ee(problem='Subdirectory {0} was not found in paths {1}'.format(typ, ', '.join(search_paths))) - diff = set(configs['colorschemes']) - set(configs['themes']) + diff = set(config_paths['colorschemes']) - set(config_paths['themes']) if diff: hadproblem = True for ext in diff: - typ = 'colorschemes' if ext in configs['themes'] else 'themes' - if not configs['top_' + typ] or typ == 'themes': + typ = 'colorschemes' if ext in config_paths['themes'] else 'themes' + if not config_paths['top_' + typ] or typ == 'themes': ee(problem='{0} extension {1} not present in {2}'.format( ext, - 'configuration' if (ext in paths['themes'] and ext in paths['colorschemes']) else 'directory', + 'configuration' if ( + ext in loaded_configs['themes'] and ext in loaded_configs['colorschemes'] + ) else 'directory', typ, )) @@ -412,7 +479,7 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): else: if used_main_spec.match( main_config, - data={'configs': configs, 'lists': lists}, + data={'configs': config_paths, 'lists': lists}, context=Context(main_config), echoerr=ee )[1]: @@ -437,42 +504,19 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): if lhadproblem[0]: hadproblem = True - top_colorscheme_configs = {} + top_colorscheme_configs = dict(loaded_configs['top_colorschemes']) data = { 'ext': None, 'top_colorscheme_configs': top_colorscheme_configs, 'ext_colorscheme_configs': {}, 'colors_config': colors_config } - for colorscheme, cfile in configs['top_colorschemes'].items(): - with open_file(cfile) as config_file_fp: - try: - config, lhadproblem = load(config_file_fp) - except MarkedError as e: - ee(problem=str(e)) - hadproblem = True - continue - if lhadproblem: - hadproblem = True - top_colorscheme_configs[colorscheme] = config + for colorscheme, config in loaded_configs['top_colorschemes'].items(): data['colorscheme'] = colorscheme if top_colorscheme_spec.match(config, context=Context(config), data=data, echoerr=ee)[1]: hadproblem = True - ext_colorscheme_configs = defaultdict(lambda: {}) - for ext in configs['colorschemes']: - for colorscheme, cfile in configs['colorschemes'][ext].items(): - with open_file(cfile) as config_file_fp: - try: - config, lhadproblem = load(config_file_fp) - except MarkedError as e: - ee(problem=str(e)) - hadproblem = True - continue - if lhadproblem: - hadproblem = True - ext_colorscheme_configs[ext][colorscheme] = config - + ext_colorscheme_configs = dict2(loaded_configs['colorschemes']) for ext, econfigs in ext_colorscheme_configs.items(): data = { 'ext': ext, @@ -512,33 +556,8 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): config = mconfig colorscheme_configs[colorscheme] = config - theme_configs = defaultdict(lambda: {}) - for ext in configs['themes']: - for theme, sfile in configs['themes'][ext].items(): - with open_file(sfile) as config_file_fp: - try: - config, lhadproblem = load(config_file_fp) - except MarkedError as e: - ee(problem=str(e)) - hadproblem = True - continue - if lhadproblem: - hadproblem = True - theme_configs[ext][theme] = config - - top_theme_configs = {} - for top_theme, top_theme_file in configs['top_themes'].items(): - with open_file(top_theme_file) as config_file_fp: - try: - config, lhadproblem = load(config_file_fp) - except MarkedError as e: - ee(problem=str(e)) - hadproblem = True - continue - if lhadproblem: - hadproblem = True - top_theme_configs[top_theme] = config - + theme_configs = dict2(loaded_configs['themes']) + top_theme_configs = dict(loaded_configs['top_themes']) for ext, configs in theme_configs.items(): data = { 'ext': ext, @@ -562,12 +581,12 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None): for top_theme, config in top_theme_configs.items(): data = { - 'ext': ext, + 'ext': None, 'colorscheme_configs': colorscheme_configs, 'import_paths': import_paths, 'main_config': main_config, 'theme_configs': theme_configs, - 'ext_theme_configs': configs, + 'ext_theme_configs': None, 'colors_config': colors_config } data['theme_type'] = 'top'