Merge pull request #1253 from ZyX-I/fix-lint

Fix powerline-lint

powerline/lib/dict.py

@@ -35,6 +35,7 @@ def mergedicts(d1, d2, remove=True):
 
 	First dictionary is modified in-place.
 	'''
+	_setmerged(d1, d2)
 	for k in d2:
 		if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict):
 			mergedicts(d1[k], d2[k], remove)
@@ -58,6 +59,11 @@ def mergedefaults(d1, d2):
 			d1.setdefault(k, d2[k])
 
 
+def _setmerged(d1, d2):
+	if hasattr(d1, 'setmerged'):
+		d1.setmerged(d2)
+
+
 def mergedicts_copy(d1, d2):
 	'''Recursively merge two dictionaries.
 
@@ -65,6 +71,7 @@ def mergedicts_copy(d1, d2):
 	that first dictionary supports .copy() method.
 	'''
 	ret = d1.copy()
+	_setmerged(ret, d2)
 	for k in d2:
 		if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict):
 			ret[k] = mergedicts_copy(d1[k], d2[k])

powerline/lint/__init__.py

@@ -5,6 +5,7 @@ import os
 import logging
 
 from collections import defaultdict
+from itertools import chain
 from functools import partial
 
 from powerline import generate_config_finder, get_config_paths, load_config
@@ -12,6 +13,7 @@ from powerline.segments.vim import vim_modes
 from powerline.lib.dict import mergedicts_copy
 from powerline.lib.config import ConfigLoader
 from powerline.lib.unicode import unicode
+from powerline.lib.path import join
 from powerline.lint.markedjson import load
 from powerline.lint.markedjson.error import echoerr, EchoErr, MarkedError
 from powerline.lint.checks import (check_matcher_func, check_ext, check_config, check_top_theme,
@@ -294,6 +296,74 @@ def register_common_names():
 	register_common_name('player', 'powerline.segments.common.players', '_player')
 
 
+def load_json_file(path):
+	with open_file(path) as F:
+		try:
+			config, hadproblem = load(F)
+		except MarkedError as e:
+			return True, None, str(e)
+		else:
+			return hadproblem, config, None
+
+
+def updated_with_config(d):
+	hadproblem, config, error = load_json_file(d['path'])
+	d.update(
+		hadproblem=hadproblem,
+		config=config,
+		error=error,
+	)
+	return d
+
+
+def find_all_ext_config_files(search_paths, subdir):
+	for config_root in search_paths:
+		top_config_subpath = join(config_root, subdir)
+		if not os.path.isdir(top_config_subpath):
+			if os.path.exists(top_config_subpath):
+				yield {
+					'error': 'Path {0} is not a directory'.format(top_config_subpath),
+					'path': top_config_subpath,
+				}
+			continue
+		for ext_name in os.listdir(top_config_subpath):
+			ext_path = os.path.join(top_config_subpath, ext_name)
+			if not os.path.isdir(ext_path):
+				if ext_name.endswith('.json') and os.path.isfile(ext_path):
+					yield updated_with_config({
+						'error': False,
+						'path': ext_path,
+						'name': ext_name[:-5],
+						'ext': None,
+						'type': 'top_' + subdir,
+					})
+				else:
+					yield {
+						'error': 'Path {0} is not a directory or configuration file'.format(ext_path),
+						'path': ext_path,
+					}
+				continue
+			for config_file_name in os.listdir(ext_path):
+				config_file_path = os.path.join(ext_path, config_file_name)
+				if config_file_name.endswith('.json') and os.path.isfile(config_file_path):
+					yield updated_with_config({
+						'error': False,
+						'path': config_file_path,
+						'name': config_file_name[:-5],
+						'ext': ext_name,
+						'type': subdir,
+					})
+				else:
+					yield {
+						'error': 'Path {0} is not a configuration file'.format(config_file_path),
+						'path': config_file_path,
+					}
+
+
+def dict2(d):
+	return defaultdict(dict, ((k, dict(v)) for k, v in d.items()))
+
+
 def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 	'''Check configuration sanity
 
@@ -313,6 +383,8 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 		``False`` if user configuration seems to be completely sane and ``True`` 
 		if some problems were found.
 	'''
+	hadproblem = False
+
 	register_common_names()
 	search_paths = paths or get_config_paths()
 	find_config_files = generate_config_finder(lambda: search_paths)
@@ -337,65 +409,60 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 
 	config_loader = ConfigLoader(run_once=True, load=load_json_config)
 
-	paths = {
-		'themes': defaultdict(lambda: []),
-		'colorschemes': defaultdict(lambda: []),
-		'top_colorschemes': [],
-		'top_themes': [],
-	}
 	lists = {
 		'colorschemes': set(),
 		'themes': set(),
 		'exts': set(),
 	}
-	for path in reversed(search_paths):
+	found_dir = {
-		for typ in ('themes', 'colorschemes'):
+		'themes': False,
-			d = os.path.join(path, typ)
+		'colorschemes': False,
-			if os.path.isdir(d):
+	}
-				for subp in os.listdir(d):
+	config_paths = defaultdict(lambda: defaultdict(dict))
-					extpath = os.path.join(d, subp)
+	loaded_configs = defaultdict(lambda: defaultdict(dict))
-					if os.path.isdir(extpath):
+	for d in chain(
-						lists['exts'].add(subp)
+		find_all_ext_config_files(search_paths, 'colorschemes'),
-						paths[typ][subp].append(extpath)
+		find_all_ext_config_files(search_paths, 'themes'),
-					elif extpath.endswith('.json'):
+	):
-						name = subp[:-5]
+		if d['error']:
-						if name != '__main__':
+			hadproblem = True
-							lists[typ].add(name)
+			ee(problem=d['error'])
-						paths['top_' + typ].append(extpath)
+			continue
-			else:
+		if d['hadproblem']:
+			hadproblem = True
+		if d['ext']:
+			found_dir[d['type']] = True
+			lists['exts'].add(d['ext'])
+			if d['name'] == '__main__':
+				pass
+			elif d['name'].startswith('__') or d['name'].endswith('__'):
 				hadproblem = True
-				ee(problem='Path {0} is supposed to be a directory, but it is not'.format(d))
+				ee(problem='File name is not supposed to start or end with “__”: {0}'.format(
+					d['path']))
+			else:
+				lists[d['type']].add(d['name'])
+			config_paths[d['type']][d['ext']][d['name']] = d['path']
+			loaded_configs[d['type']][d['ext']][d['name']] = d['config']
+		else:
+			config_paths[d['type']][d['name']] = d['path']
+			loaded_configs[d['type']][d['name']] = d['config']
 
-	hadproblem = False
-
-	configs = defaultdict(lambda: defaultdict(lambda: {}))
 	for typ in ('themes', 'colorschemes'):
-		for ext in paths[typ]:
+		if not found_dir[typ]:
-			for d in paths[typ][ext]:
+			hadproblem = True
-				for subp in os.listdir(d):
+			ee(problem='Subdirectory {0} was not found in paths {1}'.format(typ, ', '.join(search_paths)))
-					if subp.endswith('.json'):
-						name = subp[:-5]
-						if name != '__main__':
-							lists[typ].add(name)
-							if name.startswith('__') or name.endswith('__'):
-								hadproblem = True
-								ee(problem='File name is not supposed to start or end with “__”: {0}'.format(
-									os.path.join(d, subp)
-								))
-						configs[typ][ext][name] = os.path.join(d, subp)
-		for path in paths['top_' + typ]:
-			name = os.path.basename(path)[:-5]
-			configs['top_' + typ][name] = path
 
-	diff = set(configs['colorschemes']) - set(configs['themes'])
+	diff = set(config_paths['colorschemes']) - set(config_paths['themes'])
 	if diff:
 		hadproblem = True
 		for ext in diff:
-			typ = 'colorschemes' if ext in configs['themes'] else 'themes'
+			typ = 'colorschemes' if ext in config_paths['themes'] else 'themes'
-			if not configs['top_' + typ] or typ == 'themes':
+			if not config_paths['top_' + typ] or typ == 'themes':
 				ee(problem='{0} extension {1} not present in {2}'.format(
 					ext,
-					'configuration' if (ext in paths['themes'] and ext in paths['colorschemes']) else 'directory',
+					'configuration' if (
+						ext in loaded_configs['themes'] and ext in loaded_configs['colorschemes']
+					) else 'directory',
 					typ,
 				))
 
@@ -412,7 +479,7 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 	else:
 		if used_main_spec.match(
 			main_config,
-			data={'configs': configs, 'lists': lists},
+			data={'configs': config_paths, 'lists': lists},
 			context=Context(main_config),
 			echoerr=ee
 		)[1]:
@@ -437,42 +504,19 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 	if lhadproblem[0]:
 		hadproblem = True
 
-	top_colorscheme_configs = {}
+	top_colorscheme_configs = dict(loaded_configs['top_colorschemes'])
 	data = {
 		'ext': None,
 		'top_colorscheme_configs': top_colorscheme_configs,
 		'ext_colorscheme_configs': {},
 		'colors_config': colors_config
 	}
-	for colorscheme, cfile in configs['top_colorschemes'].items():
+	for colorscheme, config in loaded_configs['top_colorschemes'].items():
-		with open_file(cfile) as config_file_fp:
-			try:
-				config, lhadproblem = load(config_file_fp)
-			except MarkedError as e:
-				ee(problem=str(e))
-				hadproblem = True
-				continue
-		if lhadproblem:
-			hadproblem = True
-		top_colorscheme_configs[colorscheme] = config
 		data['colorscheme'] = colorscheme
 		if top_colorscheme_spec.match(config, context=Context(config), data=data, echoerr=ee)[1]:
 			hadproblem = True
 
-	ext_colorscheme_configs = defaultdict(lambda: {})
+	ext_colorscheme_configs = dict2(loaded_configs['colorschemes'])
-	for ext in configs['colorschemes']:
-		for colorscheme, cfile in configs['colorschemes'][ext].items():
-			with open_file(cfile) as config_file_fp:
-				try:
-					config, lhadproblem = load(config_file_fp)
-				except MarkedError as e:
-					ee(problem=str(e))
-					hadproblem = True
-					continue
-			if lhadproblem:
-				hadproblem = True
-			ext_colorscheme_configs[ext][colorscheme] = config
-
 	for ext, econfigs in ext_colorscheme_configs.items():
 		data = {
 			'ext': ext,
@@ -512,33 +556,8 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 					config = mconfig
 			colorscheme_configs[colorscheme] = config
 
-	theme_configs = defaultdict(lambda: {})
+	theme_configs = dict2(loaded_configs['themes'])
-	for ext in configs['themes']:
+	top_theme_configs = dict(loaded_configs['top_themes'])
-		for theme, sfile in configs['themes'][ext].items():
-			with open_file(sfile) as config_file_fp:
-				try:
-					config, lhadproblem = load(config_file_fp)
-				except MarkedError as e:
-					ee(problem=str(e))
-					hadproblem = True
-					continue
-			if lhadproblem:
-				hadproblem = True
-			theme_configs[ext][theme] = config
-
-	top_theme_configs = {}
-	for top_theme, top_theme_file in configs['top_themes'].items():
-		with open_file(top_theme_file) as config_file_fp:
-			try:
-				config, lhadproblem = load(config_file_fp)
-			except MarkedError as e:
-				ee(problem=str(e))
-				hadproblem = True
-				continue
-			if lhadproblem:
-				hadproblem = True
-			top_theme_configs[top_theme] = config
-
 	for ext, configs in theme_configs.items():
 		data = {
 			'ext': ext,
@@ -562,12 +581,12 @@ def check(paths=None, debug=False, echoerr=echoerr, require_ext=None):
 
 	for top_theme, config in top_theme_configs.items():
 		data = {
-			'ext': ext,
+			'ext': None,
 			'colorscheme_configs': colorscheme_configs,
 			'import_paths': import_paths,
 			'main_config': main_config,
 			'theme_configs': theme_configs,
-			'ext_theme_configs': configs,
+			'ext_theme_configs': None,
 			'colors_config': colors_config
 		}
 		data['theme_type'] = 'top'

powerline/lint/markedjson/error.py

@@ -42,15 +42,17 @@ def strtrans(s):
 
 
 class Mark:
-	def __init__(self, name, line, column, buffer, pointer):
+	def __init__(self, name, line, column, buffer, pointer, old_mark=None, merged_marks=None):
 		self.name = name
 		self.line = line
 		self.column = column
 		self.buffer = buffer
 		self.pointer = pointer
+		self.old_mark = old_mark
+		self.merged_marks = merged_marks or []
 
 	def copy(self):
-		return Mark(self.name, self.line, self.column, self.buffer, self.pointer)
+		return Mark(self.name, self.line, self.column, self.buffer, self.pointer, self.old_mark, self.merged_marks[:])
 
 	def get_snippet(self, indent=4, max_length=75):
 		if self.buffer is None:
@@ -85,17 +87,57 @@ class Mark:
 		ret.pointer += diff
 		return ret
 
-	def __str__(self):
+	def set_old_mark(self, old_mark):
-		snippet = self.get_snippet()
+		if self is old_mark:
-		where = ('  in "%s", line %d, column %d' % (
+			return
-			self.name, self.line + 1, self.column + 1))
+		checked_marks = set([id(self)])
-		if snippet is not None:
+		older_mark = old_mark
-			where += ':\n' + snippet
+		while True:
+			if id(older_mark) in checked_marks:
+				raise ValueError('Trying to set recursive marks')
+			checked_marks.add(id(older_mark))
+			older_mark = older_mark.old_mark
+			if not older_mark:
+				break
+		self.old_mark = old_mark
+
+	def set_merged_mark(self, merged_mark):
+		self.merged_marks.append(merged_mark)
+
+	def to_string(self, indent=0, head_text='in ', add_snippet=True):
+		mark = self
+		where = ''
+		processed_marks = set()
+		while mark:
+			indentstr = ' ' * indent
+			where += ('%s  %s"%s", line %d, column %d' % (
+				indentstr, head_text, mark.name, mark.line + 1, mark.column + 1))
+			if add_snippet:
+				snippet = mark.get_snippet(indent=(indent + 4))
+				if snippet:
+					where += ':\n' + snippet
+			if mark.merged_marks:
+				where += '\n' + indentstr + '  with additionally merged\n'
+				where += mark.merged_marks[0].to_string(indent + 4, head_text='', add_snippet=False)
+				for mmark in mark.merged_marks[1:]:
+					where += '\n' + indentstr + '  and\n'
+					where += mmark.to_string(indent + 4, head_text='', add_snippet=False)
+			if add_snippet:
+				processed_marks.add(id(mark))
+				if mark.old_mark:
+					where += '\n' + indentstr + '  which replaced value\n'
+					indent += 4
+			mark = mark.old_mark
+			if id(mark) in processed_marks:
+				raise ValueError('Trying to dump recursive mark')
 		if type(where) is str:
 			return where
 		else:
 			return where.encode('utf-8')
 
+	def __str__(self):
+		return self.to_string()
+
 
 def echoerr(*args, **kwargs):
 	stream = kwargs.pop('stream', sys.stderr)

powerline/lint/markedjson/markedvalue.py

@@ -65,7 +65,28 @@ class MarkedDict(dict):
 		r.keydict = dict(((key, key) for key in r))
 		return r
 
+	def setmerged(self, d):
+		try:
+			self.mark.set_merged_mark(d.mark)
+		except AttributeError:
+			pass
+
 	def __setitem__(self, key, value):
+		try:
+			old_value = self[key]
+		except KeyError:
+			pass
+		else:
+			try:
+				key.mark.set_old_mark(self.keydict[key].mark)
+			except AttributeError:
+				pass
+			except KeyError:
+				pass
+			try:
+				value.mark.set_old_mark(old_value.mark)
+			except AttributeError:
+				pass
 		dict.__setitem__(self, key, value)
 		self.keydict[key] = key