DSL: add function*(){} and yield x

lib/config/CMakeLists.txt

@@ -28,6 +28,7 @@ set(config_SOURCES
   configitem.cpp configitem.hpp
   configitembuilder.cpp configitembuilder.hpp
   expression.cpp expression.hpp
+  generator-function.cpp generator-function.hpp
   objectrule.cpp objectrule.hpp
   vmops.hpp
   ${FLEX_config_lexer_OUTPUTS} ${BISON_config_parser_OUTPUTS}

lib/config/config_lexer.ll

@@ -178,6 +178,7 @@ function			return T_FUNCTION;
 return				return T_RETURN;
 break				return T_BREAK;
 continue			return T_CONTINUE;
+yield			return T_YIELD;
 for				return T_FOR;
 if				return T_IF;
 else				return T_ELSE;

lib/config/config_parser.yy

@@ -151,6 +151,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig
 %token T_RETURN "return (T_RETURN)"
 %token T_BREAK "break (T_BREAK)"
 %token T_CONTINUE "continue (T_CONTINUE)"
+%token T_YIELD "yield (T_YIELD)"
 %token T_FOR "for (T_FOR)"
 %token T_IF "if (T_IF)"
 %token T_ELSE "else (T_ELSE)"
@@ -191,6 +192,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig
 %type <cvlist> use_specifier
 %type <cvlist> use_specifier_items
 %type <cvitem> use_specifier_item
+%type <boolean> generator_specifier
 %type <num> object_declaration
 
 %right T_FOLLOWS
@@ -200,7 +202,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig
 %right '?' ':'
 %left T_LOGICAL_OR
 %left T_LOGICAL_AND
-%left T_RETURN T_BREAK T_CONTINUE
+%left T_RETURN T_BREAK T_CONTINUE T_YIELD
 %left T_IDENTIFIER
 %left T_BINARY_OR
 %left T_XOR
@@ -357,7 +359,7 @@ object:
 	}
 	object_declaration rterm optional_rterm use_specifier default_specifier ignore_specifier
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope_require_side_effect
 	{
@@ -493,7 +495,7 @@ lterm: T_LIBRARY rterm
 	}
 	| T_ASSIGN T_WHERE
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope %dprec 2
 	{
@@ -529,7 +531,7 @@ lterm: T_LIBRARY rterm
 	}
 	| T_IGNORE T_WHERE
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope %dprec 2
 	{
@@ -563,11 +565,21 @@ lterm: T_LIBRARY rterm
 
 		$$ = MakeLiteralRaw();
 	}
-	| T_RETURN optional_rterm
+	| T_RETURN
 	{
-		UseFlowControl(context, FlowControlReturn, @$);
+		UseFlowControl(context, FlowControlReturnVoid, @$);
+		$$ = new ReturnExpression(std::unique_ptr<Expression>(MakeLiteralRaw()), @$);
+	}
+	| T_RETURN rterm
+	{
+		UseFlowControl(context, FlowControlReturnValue, @$);
 		$$ = new ReturnExpression(std::unique_ptr<Expression>($2), @$);
 	}
+	| T_YIELD rterm
+	{
+		UseFlowControl(context, FlowControlYield, @$);
+		$$ = new YieldExpression(std::unique_ptr<Expression>($2), @$);
+	}
 	| T_BREAK
 	{
 		UseFlowControl(context, FlowControlBreak, @$);
@@ -584,7 +596,7 @@ lterm: T_LIBRARY rterm
 	}
 	| T_NAMESPACE rterm
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope_require_side_effect
 	{
@@ -625,20 +637,20 @@ lterm: T_LIBRARY rterm
 		$$ = new ForExpression(std::move(*$4), "", std::unique_ptr<Expression>($6), std::unique_ptr<Expression>($9), @$);
 		delete $4;
 	}
-	| T_FUNCTION identifier '(' identifier_items ')' use_specifier
+	| T_FUNCTION generator_specifier identifier '(' identifier_items ')' use_specifier
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | ($2 ? FlowControlYield : FlowControlReturnValue), false);
 	}
 	rterm_scope
 	{
 		EndFlowControlBlock(context);
 
-		std::unique_ptr<FunctionExpression> fexpr{new FunctionExpression(*$2, std::move(*$4), std::move(*$6), std::unique_ptr<Expression>($8), @$)};
+		std::unique_ptr<FunctionExpression> fexpr{new FunctionExpression(*$3, std::move(*$5), std::move(*$7), std::unique_ptr<Expression>($9), $2, @$)};
-		delete $4;
+		delete $5;
-		delete $6;
+		delete $7;
 
-		$$ = new SetExpression(MakeIndexer(ScopeThis, std::move(*$2)), OpSetLiteral, std::move(fexpr), @$);
+		$$ = new SetExpression(MakeIndexer(ScopeThis, std::move(*$3)), OpSetLiteral, std::move(fexpr), @$);
-		delete $2;
+		delete $3;
 	}
 	| T_CONST T_IDENTIFIER T_SET rterm
 	{
@@ -915,7 +927,7 @@ rterm_no_side_effect_no_dict: T_STRING
 	}
 	| identifier T_FOLLOWS
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope %dprec 2
 	{
@@ -925,7 +937,7 @@ rterm_no_side_effect_no_dict: T_STRING
 		args.emplace_back(std::move(*$1));
 		delete $1;
 
-		$$ = new FunctionExpression("<anonymous>", std::move(args), {}, std::unique_ptr<Expression>($4), @$);
+		$$ = new FunctionExpression("<anonymous>", std::move(args), {}, std::unique_ptr<Expression>($4), false, @$);
 	}
 	| identifier T_FOLLOWS rterm %dprec 1
 	{
@@ -935,17 +947,17 @@ rterm_no_side_effect_no_dict: T_STRING
 		args.emplace_back(std::move(*$1));
 		delete $1;
 
-		$$ = new FunctionExpression("<anonymous>", std::move(args), {}, std::unique_ptr<Expression>($3), @$);
+		$$ = new FunctionExpression("<anonymous>", std::move(args), {}, std::unique_ptr<Expression>($3), false, @$);
 	}
 	| '(' identifier_items ')' use_specifier T_FOLLOWS
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope %dprec 2
 	{
 		EndFlowControlBlock(context);
 
-		$$ = new FunctionExpression("<anonymous>", std::move(*$2), std::move(*$4), std::unique_ptr<Expression>($7), @$);
+		$$ = new FunctionExpression("<anonymous>", std::move(*$2), std::move(*$4), std::unique_ptr<Expression>($7), false, @$);
 		delete $2;
 		delete $4;
 	}
@@ -953,7 +965,7 @@ rterm_no_side_effect_no_dict: T_STRING
 	{
 		ASSERT(!dynamic_cast<DictExpression *>($6));
 
-		$$ = new FunctionExpression("<anonymous>", std::move(*$2), std::move(*$4), std::unique_ptr<Expression>($6), @$);
+		$$ = new FunctionExpression("<anonymous>", std::move(*$2), std::move(*$4), std::unique_ptr<Expression>($6), false, @$);
 		delete $2;
 		delete $4;
 	}
@@ -987,21 +999,21 @@ rterm_no_side_effect_no_dict: T_STRING
 	| rterm T_DIVIDE_OP rterm { MakeRBinaryOp<DivideExpression>(&$$, $1, $3, @1, @3); }
 	| rterm T_MODULO rterm { MakeRBinaryOp<ModuloExpression>(&$$, $1, $3, @1, @3); }
 	| rterm T_XOR rterm { MakeRBinaryOp<XorExpression>(&$$, $1, $3, @1, @3); }
-	| T_FUNCTION '(' identifier_items ')' use_specifier
+	| T_FUNCTION generator_specifier '(' identifier_items ')' use_specifier
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | ($2 ? FlowControlYield : FlowControlReturnValue), false);
 	}
 	rterm_scope
 	{
 		EndFlowControlBlock(context);
 
-		$$ = new FunctionExpression("<anonymous>", std::move(*$3), std::move(*$5), std::unique_ptr<Expression>($7), @$);
+		$$ = new FunctionExpression("<anonymous>", std::move(*$4), std::move(*$6), std::unique_ptr<Expression>($8), $2, @$);
-		delete $3;
+		delete $4;
-		delete $5;
+		delete $6;
 	}
 	| T_NULLARY_LAMBDA_BEGIN
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	statements T_NULLARY_LAMBDA_END
 	{
@@ -1019,7 +1031,7 @@ rterm_no_side_effect_no_dict: T_STRING
 		std::unique_ptr<DictExpression> aexpr{new DictExpression(std::move(dlist), @$)};
 		aexpr->MakeInline();
 
-		$$ = new FunctionExpression("<anonymous>", {}, {}, std::move(aexpr), @$);
+		$$ = new FunctionExpression("<anonymous>", {}, {}, std::move(aexpr), false, @$);
 	}
 	;
 
@@ -1105,6 +1117,16 @@ use_specifier_item: identifier
 	}
 	;
 
+generator_specifier: /* empty */
+	{
+		$$ = false;
+	}
+	| T_MULTIPLY
+	{
+		$$ = true;
+	}
+	;
+
 apply_for_specifier: /* empty */
 	| T_FOR '(' optional_var identifier T_FOLLOWS optional_var identifier T_IN rterm ')'
 	{
@@ -1147,7 +1169,7 @@ apply:
 	}
 	T_APPLY identifier optional_rterm apply_for_specifier target_type_specifier use_specifier ignore_specifier
 	{
-		BeginFlowControlBlock(context, FlowControlReturn, false);
+		BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false);
 	}
 	rterm_scope_require_side_effect
 	{

lib/config/configcompiler.hpp

@@ -50,9 +50,11 @@ struct EItemInfo
 
 enum FlowControlType
 {
-	FlowControlReturn = 1,
+	FlowControlReturnVoid = 1,
 	FlowControlContinue = 2,
-	FlowControlBreak = 4
+	FlowControlBreak = 4,
+	FlowControlReturnValue = 8,
+	FlowControlYield = 16
 };
 
 struct ZoneFragment

lib/config/expression.cpp

@@ -3,6 +3,7 @@
 #include "config/expression.hpp"
 #include "config/configitem.hpp"
 #include "config/configcompiler.hpp"
+#include "config/generator-function.hpp"
 #include "config/vmops.hpp"
 #include "base/array.hpp"
 #include "base/json.hpp"
@@ -734,6 +735,15 @@ ExpressionResult ContinueExpression::DoEvaluate(ScriptFrame& frame, DebugHint *d
 	return ExpressionResult(Empty, ResultContinue);
 }
 
+ExpressionResult YieldExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const
+{
+	ExpressionResult operand = m_Operand->Evaluate(frame);
+	CHECK_RESULT(operand);
+
+	l_GeneratorFunction->YieldItem(operand.GetValue());
+	return Empty;
+}
+
 ExpressionResult IndexerExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const
 {
 	ExpressionResult operand1 = m_Operand1->Evaluate(frame, dhint);
@@ -904,7 +914,7 @@ ExpressionResult ImportDefaultTemplatesExpression::DoEvaluate(ScriptFrame& frame
 
 ExpressionResult FunctionExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const
 {
-	return VMOps::NewFunction(frame, m_Name, m_Args, m_ClosedVars, m_Expression);
+	return VMOps::NewFunction(frame, m_Name, m_Args, m_ClosedVars, m_Expression, m_Generator);
 }
 
 ExpressionResult ApplyExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const

lib/config/expression.hpp

@@ -734,6 +734,17 @@ protected:
 	ExpressionResult DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const override;
 };
 
+class YieldExpression final : public UnaryExpression
+{
+public:
+	YieldExpression(std::unique_ptr<Expression> expression, const DebugInfo& debugInfo = DebugInfo())
+		: UnaryExpression(std::move(expression), debugInfo)
+	{ }
+
+protected:
+	ExpressionResult DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const override;
+};
+
 class GetScopeExpression final : public Expression
 {
 public:
@@ -812,8 +823,9 @@ class FunctionExpression final : public DebuggableExpression
 {
 public:
 	FunctionExpression(String name, std::vector<String> args,
-		std::map<String, std::unique_ptr<Expression> >&& closedVars, std::unique_ptr<Expression> expression, const DebugInfo& debugInfo = DebugInfo())
+		std::map<String, std::unique_ptr<Expression> >&& closedVars, std::unique_ptr<Expression> expression, bool generator, const DebugInfo& debugInfo = DebugInfo())
-		: DebuggableExpression(debugInfo), m_Name(std::move(name)), m_Args(std::move(args)), m_ClosedVars(std::move(closedVars)), m_Expression(expression.release())
+		: DebuggableExpression(debugInfo), m_Name(std::move(name)), m_Args(std::move(args)),
+		m_ClosedVars(std::move(closedVars)), m_Expression(expression.release()), m_Generator(generator)
 	{ }
 
 protected:
@@ -824,6 +836,7 @@ private:
 	std::vector<String> m_Args;
 	std::map<String, std::unique_ptr<Expression> > m_ClosedVars;
 	Expression::Ptr m_Expression;
+	bool m_Generator;
 };
 
 class ApplyExpression final : public DebuggableExpression

lib/config/generator-function.cpp

@@ -0,0 +1,138 @@
+/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */
+
+#include "base/debug.hpp"
+#include "base/exception.hpp"
+#include "base/logger.hpp"
+#include "base/objectlock.hpp"
+#include "base/scriptframe.hpp"
+#include "base/shared.hpp"
+#include "base/value.hpp"
+#include "config/generator-function.hpp"
+#include <exception>
+#include <future>
+#include <mutex>
+#include <thread>
+#include <utility>
+
+using namespace icinga;
+
+GeneratorFunction::GeneratorFunction(const Expression::Ptr& expression)
+{
+	auto originFrame (ScriptFrame::GetCurrentFrame());
+
+	std::promise<void> promise;
+	auto future (promise.get_future());
+
+	m_Thread = std::thread([this, expression, originFrame, &promise]() {
+		ScriptFrame frame (false);
+		frame = *originFrame;
+
+		promise.set_value();
+
+		frame.Depth = 0;
+
+		for (std::unique_lock<std::mutex> lock (m_ITC.Lock);;) {
+			if (!m_ITC.NextQueue.empty()) {
+				break;
+			}
+
+			if (m_ITC.Shutdown) {
+				return;
+			}
+
+			m_ITC.CV.wait(lock);
+		}
+
+		l_GeneratorFunction = this;
+
+		try {
+			expression->Evaluate(frame);
+		} catch (ForcedUnwind) {
+		} catch (...) {
+			std::unique_lock<std::mutex> lock (m_ITC.Lock);
+			ASSERT(!m_ITC.NextQueue.empty());
+
+			auto next (std::move(m_ITC.NextQueue.front()));
+			m_ITC.NextQueue.pop();
+			lock.unlock();
+
+			next.set_exception(std::current_exception());
+		}
+
+		std::unique_lock<std::mutex> lock (m_ITC.Lock);
+		m_ITC.Shutdown = true;
+
+		while (!m_ITC.NextQueue.empty()) {
+			auto next (std::move(m_ITC.NextQueue.front()));
+			m_ITC.NextQueue.pop();
+
+			next.set_value({Empty, false});
+		}
+	});
+
+	future.wait();
+}
+
+GeneratorFunction::~GeneratorFunction()
+{
+	{
+		std::unique_lock<std::mutex> lock (m_ITC.Lock);
+		m_ITC.Shutdown = true;
+		m_ITC.CV.notify_all();
+	}
+
+	m_Thread.join();
+}
+
+bool GeneratorFunction::GetNext(Value& out)
+{
+	std::unique_lock<std::mutex> lock (m_ITC.Lock);
+
+	if (m_ITC.Shutdown) {
+		return false;
+	}
+
+	std::promise<std::pair<Value, bool>> promise;
+	auto future (promise.get_future());
+
+	m_ITC.NextQueue.emplace(std::move(promise));
+	m_ITC.CV.notify_all();
+
+	lock.unlock();
+
+	future.wait();
+
+	auto res (future.get());
+
+	if (res.second) {
+		out = std::move(res.first);
+	}
+
+	return res.second;
+}
+
+void GeneratorFunction::YieldItem(Value item)
+{
+	std::unique_lock<std::mutex> lock (m_ITC.Lock);
+	ASSERT(!m_ITC.NextQueue.empty());
+
+	auto next (std::move(m_ITC.NextQueue.front()));
+	m_ITC.NextQueue.pop();
+	lock.unlock();
+
+	next.set_value({std::move(item), true});
+
+	for (lock.lock();;) {
+		if (!m_ITC.NextQueue.empty()) {
+			break;
+		}
+
+		if (m_ITC.Shutdown) {
+			throw ForcedUnwind();
+		}
+
+		m_ITC.CV.wait(lock);
+	}
+}
+
+thread_local GeneratorFunction* icinga::l_GeneratorFunction = nullptr;

lib/config/generator-function.hpp

@@ -0,0 +1,54 @@
+/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */
+
+#ifndef GENERATOR_FUNCTION_H
+#define GENERATOR_FUNCTION_H
+
+#include "base/generator.hpp"
+#include "base/value.hpp"
+#include "config/expression.hpp"
+#include <condition_variable>
+#include <exception>
+#include <future>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <utility>
+
+namespace icinga
+{
+
+/**
+ * Supplier a generator function's items.
+ *
+ * @ingroup config
+ */
+class GeneratorFunction final : public Generator
+{
+public:
+	class ForcedUnwind
+	{
+	};
+
+	GeneratorFunction(const Expression::Ptr& expression);
+	~GeneratorFunction() override;
+
+	bool GetNext(Value& out) override;
+	void YieldItem(Value item);
+
+private:
+	std::thread m_Thread;
+
+	struct {
+		std::queue<std::promise<std::pair<Value, bool>>> NextQueue;
+		bool Shutdown = false;
+		std::mutex Lock;
+		std::condition_variable CV;
+		std::exception_ptr Exception;
+	} m_ITC;
+};
+
+extern thread_local GeneratorFunction* l_GeneratorFunction;
+
+}
+
+#endif /* GENERATOR_FUNCTION_H */

lib/config/vmops.hpp

@@ -7,6 +7,7 @@
 #include "config/expression.hpp"
 #include "config/configitembuilder.hpp"
 #include "config/applyrule.hpp"
+#include "config/generator-function.hpp"
 #include "config/objectrule.hpp"
 #include "base/debuginfo.hpp"
 #include "base/array.hpp"
@@ -93,11 +94,11 @@ public:
 	}
 
 	static inline Value NewFunction(ScriptFrame& frame, const String& name, const std::vector<String>& argNames,
-		const std::map<String, std::unique_ptr<Expression> >& closedVars, const Expression::Ptr& expression)
+		const std::map<String, std::unique_ptr<Expression> >& closedVars, const Expression::Ptr& expression, bool generator)
 	{
 		auto evaluatedClosedVars = EvaluateClosedVars(frame, closedVars);
 
-		auto wrapper = [argNames, evaluatedClosedVars, expression](const std::vector<Value>& arguments) -> Value {
+		auto wrapper = [argNames, evaluatedClosedVars, expression, generator](const std::vector<Value>& arguments) -> Value {
 			if (arguments.size() < argNames.size())
 				BOOST_THROW_EXCEPTION(std::invalid_argument("Too few arguments for function"));
 
@@ -111,7 +112,11 @@ public:
 			for (std::vector<Value>::size_type i = 0; i < std::min(arguments.size(), argNames.size()); i++)
 				frame->Locals->Set(argNames[i], arguments[i]);
 
+			if (generator) {
+				return new GeneratorFunction(expression);
+			} else {
 				return expression->Evaluate(*frame);
+			}
 		};
 
 		return new Function(name, wrapper, argNames);