Make lint checker work

powerline/lint/__init__.py

@@ -7,6 +7,7 @@ from powerline.lint.markedjson.error import echoerr, MarkedError
 from powerline.segments.vim import vim_modes
 from powerline.lint.inspect import getconfigargspec
 from powerline.lib.threaded import ThreadedSegment
+from powerline.lib import mergedicts_copy
 import itertools
 import sys
 import os
@@ -414,14 +415,20 @@ def check_matcher_func(ext, match_name, data, context, echoerr):
 def check_ext(ext, data, context, echoerr):
 	hadsomedirs = False
 	hadproblem = False
-	for subdir in ('themes', 'colorschemes'):
+	if ext not in data['lists']['exts']:
-		if ext not in data['configs'][subdir]:
+		hadproblem = True
-			hadproblem = True
+		echoerr(context='Error while loading {0} extension configuration'.format(ext),
-			echoerr(context='Error while loading {0} extension configuration'.format(ext),
+				context_mark=ext.mark,
-					context_mark=ext.mark,
+				problem='extension configuration does not exist')
-					problem='{0} configuration does not exist'.format(subdir))
+	else:
-		else:
+		for typ in ('themes', 'colorschemes'):
-			hadsomedirs = True
+			if ext not in data['configs'][typ] and not data['configs']['top_' + typ]:
+				hadproblem = True
+				echoerr(context='Error while loading {0} extension configuration'.format(ext),
+						context_mark=ext.mark,
+						problem='{0} configuration does not exist'.format(typ))
+			else:
+				hadsomedirs = True
 	return hadsomedirs, hadproblem
 
 
@@ -431,7 +438,13 @@ def check_config(d, theme, data, context, echoerr):
 	else:
 		# local_themes
 		ext = context[-3][0]
-	if ext not in data['configs'][d] or theme not in data['configs'][d][ext]:
+	if ext not in data['lists']['exts']:
+		echoerr(context='Error while loading {0} extension configuration'.format(ext),
+				context_mark=ext.mark,
+				problem='extension configuration does not exist')
+		return True, False, True
+	if ((ext not in data['configs'][d] or theme not in data['configs'][d][ext])
+			and theme not in data['configs']['top_' + d]):
 		echoerr(context='Error while loading {0} from {1} extension configuration'.format(d[:-1], ext),
 				problem='failed to find configuration file {0}/{1}/{2}.json'.format(d, ext, theme),
 				problem_mark=theme.mark)
@@ -531,22 +544,84 @@ def check_color(color, data, context, echoerr):
 
 
 def check_translated_group_name(group, data, context, echoerr):
-	if group not in context[0][1].get('groups', {}):
+	return check_group(group, data, context, echoerr)
-		echoerr(context='Error while checking translated group in colorscheme (key {key})'.format(key=context_key(context)),
+
-				problem='translated group {0} is not in main groups dictionary'.format(group),
+
-				problem_mark=group.mark)
+def check_group(group, data, context, echoerr):
-		return True, False, True
+	if not isinstance(group, unicode):
-	return True, False, False
+		return True, False, False
+	colorscheme = data['colorscheme']
+	ext = data['ext']
+	configs = []
+	if ext:
+		if colorscheme == '__main__':
+			configs.append([config for config in data['ext_colorscheme_configs'][ext].items()])
+			configs.append([config for config in data['top_colorscheme_configs'].items()])
+		else:
+			try:
+				configs.append([data['ext_colorscheme_configs'][ext][colorscheme]])
+			except KeyError:
+				pass
+			try:
+				configs.append([data['ext_colorscheme_configs'][ext]['__main__']])
+			except KeyError:
+				pass
+			try:
+				configs.append([data['top_colorscheme_configs'][colorscheme]])
+			except KeyError:
+				pass
+	else:
+		try:
+			configs.append([data['top_colorscheme_configs'][colorscheme]])
+		except KeyError:
+			pass
+	new_echoerr = DelayedEchoErr(echoerr)
+	hadproblem = False
+	for config_lst in configs:
+		tofind = len(config_lst)
+		not_found = []
+		for config in config_lst:
+			if isinstance(config, tuple):
+				new_colorscheme, config = config
+				new_data = data.copy()
+				new_data['colorscheme'] = new_colorscheme
+			else:
+				new_data = data
+			try:
+				group_data = config['groups'][group]
+			except KeyError:
+				not_found.append(config.mark.name)
+			else:
+				proceed, echo, chadproblem = check_group(
+					group_data,
+					new_data,
+					context,
+					echoerr,
+				)
+				if chadproblem:
+					hadproblem = True
+				else:
+					tofind -= 1
+					if not tofind:
+						return proceed, echo, hadproblem
+				if not proceed:
+					break
+		if not_found:
+			new_echoerr(context='Error while checking group definition in colorscheme (key {key})'.format(key=context_key(context)),
+						problem='name {0} is not present in {1} {2} colorschemes: {3}'.format(group, tofind, ext, ', '.join(not_found)),
+						problem_mark=group.mark)
+	new_echoerr.echo_all()
+	return True, False, hadproblem
 
 
 color_spec = Spec().type(unicode).func(check_color).copy
 name_spec = Spec().type(unicode).len('gt', 0).optional().copy
-group_spec = Spec(
+group_name_spec = Spec().re('^\w+(?::\w+)?$').copy
+group_spec = Spec().either(Spec(
 	fg=color_spec(),
 	bg=color_spec(),
-	attr=Spec().list(Spec().type(unicode).oneof(set(('bold', 'italic', 'underline')))).optional(),
+	attr=Spec().list(Spec().type(unicode).oneof(set(('bold', 'italic', 'underline')))),
-).copy
+), group_name_spec().func(check_group)).copy
-group_name_spec = Spec().re('^\w+(?::\w+)?$').copy
 groups_spec = Spec().unknown_spec(
 	group_name_spec(),
 	group_spec(),
@@ -555,23 +630,32 @@ colorscheme_spec = (Spec(
 	name=name_spec(),
 	groups=groups_spec(),
 ).context_message('Error while loading coloscheme'))
+mode_translations_value_spec = Spec(
+	colors=Spec().unknown_spec(
+		color_spec(),
+		color_spec(),
+	).optional(),
+	groups=Spec().unknown_spec(
+		group_name_spec().func(check_translated_group_name),
+		group_spec(),
+	).optional(),
+).copy
+top_colorscheme_spec = (Spec(
+	name=name_spec(),
+	groups=groups_spec(),
+	mode_translations=Spec().unknown_spec(
+		Spec().type(unicode),
+		mode_translations_value_spec(),
+	).optional().context_message('Error while loading mode translations (key {key})').optional(),
+).context_message('Error while loading top-level coloscheme'))
 vim_mode_spec = Spec().oneof(set(list(vim_modes) + ['nc'])).copy
 vim_colorscheme_spec = (Spec(
 	name=name_spec(),
 	groups=groups_spec(),
 	mode_translations=Spec().unknown_spec(
 		vim_mode_spec(),
-		Spec(
+		mode_translations_value_spec(),
-			colors=Spec().unknown_spec(
+	).optional().context_message('Error while loading mode translations (key {key})'),
-				color_spec(),
-				color_spec(),
-			).optional(),
-			groups=Spec().unknown_spec(
-				group_name_spec().func(check_translated_group_name),
-				group_spec(),
-			).optional(),
-		),
-	).context_message('Error while loading mode translations (key {key})'),
 ).context_message('Error while loading vim colorscheme'))
 shell_mode_spec = Spec().re('^(?:[\w\-]+|\.safe)$').copy
 shell_colorscheme_spec = (Spec(
@@ -579,17 +663,8 @@ shell_colorscheme_spec = (Spec(
 	groups=groups_spec(),
 	mode_translations=Spec().unknown_spec(
 		shell_mode_spec(),
-		Spec(
+		mode_translations_value_spec(),
-			colors=Spec().unknown_spec(
+	).optional().context_message('Error while loading mode translations (key {key})'),
-				color_spec(),
-				color_spec(),
-			).optional(),
-			groups=Spec().unknown_spec(
-				group_name_spec().func(check_translated_group_name),
-				group_spec(),
-			).optional(),
-		),
-	).context_message('Error while loading mode translations (key {key})'),
 ).context_message('Error while loading shell colorscheme'))
 
 
@@ -1044,44 +1119,60 @@ def check(path=None, debug=False):
 
 	ee = EchoErr(echoerr, logger)
 
-	dirs = {
+	paths = {
 		'themes': defaultdict(lambda: []),
-		'colorschemes': defaultdict(lambda: [])
+		'colorschemes': defaultdict(lambda: []),
+		'top_colorschemes': [],
+		'top_themes': [],
+	}
+	lists = {
+		'colorschemes': set(),
+		'themes': set(),
+		'exts': set(),
 	}
 	for path in reversed(search_paths):
-		for subdir in ('themes', 'colorschemes'):
+		for typ in ('themes', 'colorschemes'):
-			d = os.path.join(path, subdir)
+			d = os.path.join(path, typ)
 			if os.path.isdir(d):
-				for ext in os.listdir(d):
+				for subp in os.listdir(d):
-					extd = os.path.join(d, ext)
+					extpath = os.path.join(d, subp)
-					if os.path.isdir(extd):
+					if os.path.isdir(extpath):
-						dirs[subdir][ext].append(extd)
+						lists['exts'].add(subp)
-			elif os.path.exists(d):
+						paths[typ][subp].append(extpath)
+					elif extpath.endswith('.json'):
+						name = subp[:-5]
+						if name != '__main__':
+							lists[typ].add(name)
+						paths['top_' + typ].append(extpath)
+			else:
 				hadproblem = True
 				sys.stderr.write('Path {0} is supposed to be a directory, but it is not\n'.format(d))
 
-	configs = {
+	configs = defaultdict(lambda: defaultdict(lambda: {}))
-		'themes': defaultdict(lambda: {}),
+	for typ in ('themes', 'colorschemes'):
-		'colorschemes': defaultdict(lambda: {})
+		for ext in paths[typ]:
-	}
+			for d in paths[typ][ext]:
-	for subdir in ('themes', 'colorschemes'):
+				for subp in os.listdir(d):
-		for ext in dirs[subdir]:
+					if subp.endswith('.json'):
-			for d in dirs[subdir][ext]:
+						name = subp[:-5]
-				for config in os.listdir(d):
+						if name != '__main__':
-					if os.path.isdir(os.path.join(d, config)):
+							lists[typ].add(name)
-						dirs[subdir][ext].append(os.path.join(d, config))
+						configs[typ][ext][name] = os.path.join(d, subp)
-					elif config.endswith('.json'):
+		for path in paths['top_' + typ]:
-						configs[subdir][ext][config[:-5]] = os.path.join(d, config)
+			name = os.path.basename(path)[:-5]
+			configs['top_' + typ][name] = path
 
 	diff = set(configs['themes']) ^ set(configs['colorschemes'])
 	if diff:
 		hadproblem = True
 		for ext in diff:
-			sys.stderr.write('{0} extension {1} present only in {2}\n'.format(
+			typ = 'colorschemes' if ext in configs['themes'] else 'themes'
-				ext,
+			if not configs['top_' + typ] or typ == 'themes':
-				'configuration' if (ext in dirs['themes'] and ext in dirs['colorschemes']) else 'directory',
+				sys.stderr.write('{0} extension {1} not present in {2}\n'.format(
-				'themes' if ext in configs['themes'] else 'colorschemes',
+					ext,
-			))
+					'configuration' if (ext in paths['themes'] and ext in paths['colorschemes']) else 'directory',
+					typ,
+				))
 
 	lhadproblem = [False]
 
@@ -1103,7 +1194,7 @@ def check(path=None, debug=False):
 		sys.stderr.write(str(e) + '\n')
 		hadproblem = True
 	else:
-		if main_spec.match(main_config, data={'configs': configs}, context=(('', main_config),), echoerr=ee)[1]:
+		if main_spec.match(main_config, data={'configs': configs, 'lists': lists}, context=(('', main_config),), echoerr=ee)[1]:
 			hadproblem = True
 
 	import_paths = [os.path.expanduser(path) for path in main_config.get('common', {}).get('paths', [])]
@@ -1125,9 +1216,30 @@ def check(path=None, debug=False):
 	if lhadproblem[0]:
 		hadproblem = True
 
-	colorscheme_configs = defaultdict(lambda: {})
+	top_colorscheme_configs = {}
+	data = {
+		'ext': None,
+		'top_colorscheme_configs': top_colorscheme_configs,
+		'ext_colorscheme_configs': {},
+		'colors_config': colors_config
+	}
+	for colorscheme, cfile in configs['top_colorschemes'].items():
+		with open_file(cfile) as config_file_fp:
+			try:
+				config, lhadproblem = load(config_file_fp)
+			except MarkedError as e:
+				sys.stderr.write(str(e) + '\n')
+				hadproblem = True
+				continue
+		if lhadproblem:
+			hadproblem = True
+		top_colorscheme_configs[colorscheme] = config
+		data['colorscheme'] = colorscheme
+		if top_colorscheme_spec.match(config, context=(('', config),), data=data, echoerr=ee)[1]:
+			hadproblem = True
+
+	ext_colorscheme_configs = defaultdict(lambda: {})
 	for ext in configs['colorschemes']:
-		data = {'ext': ext, 'colors_config': colors_config}
 		for colorscheme, cfile in configs['colorschemes'][ext].items():
 			with open_file(cfile) as config_file_fp:
 				try:
@@ -1138,7 +1250,17 @@ def check(path=None, debug=False):
 					continue
 			if lhadproblem:
 				hadproblem = True
-			colorscheme_configs[ext][colorscheme] = config
+			ext_colorscheme_configs[ext][colorscheme] = config
+
+	for ext, econfigs in ext_colorscheme_configs.items():
+		data = {
+			'ext': ext,
+			'top_colorscheme_configs': top_colorscheme_configs,
+			'ext_colorscheme_configs': ext_colorscheme_configs,
+			'colors_config': colors_config,
+		}
+		for colorscheme, config in econfigs.items():
+			data['colorscheme'] = colorscheme
 			if ext == 'vim':
 				spec = vim_colorscheme_spec
 			elif ext == 'shell':
@@ -1148,6 +1270,27 @@ def check(path=None, debug=False):
 			if spec.match(config, context=(('', config),), data=data, echoerr=ee)[1]:
 				hadproblem = True
 
+	colorscheme_configs = {}
+	for ext in lists['exts']:
+		colorscheme_configs[ext] = {}
+		for colorscheme in lists['colorschemes']:
+			econfigs = ext_colorscheme_configs[ext]
+			ecconfigs = econfigs.get(colorscheme)
+			mconfigs = (
+				top_colorscheme_configs.get(colorscheme),
+				econfigs.get('__main__'),
+				ecconfigs,
+			)
+			config = None
+			for mconfig in mconfigs:
+				if not mconfig:
+					continue
+				if config:
+					config = mergedicts_copy(config, mconfig)
+				else:
+					config = mconfig
+			colorscheme_configs[colorscheme] = config
+
 	theme_configs = defaultdict(lambda: {})
 	for ext in configs['themes']:
 		for theme, sfile in configs['themes'][ext].items():
@@ -1161,6 +1304,7 @@ def check(path=None, debug=False):
 			if lhadproblem:
 				hadproblem = True
 			theme_configs[ext][theme] = config
+
 	for ext, configs in theme_configs.items():
 		data = {'ext': ext, 'colorscheme_configs': colorscheme_configs, 'import_paths': import_paths,
 				'main_config': main_config, 'ext_theme_configs': configs, 'colors_config': colors_config}