diff --git a/lib/config/CMakeLists.txt b/lib/config/CMakeLists.txt index 80b8c2c44..0bc562e6b 100644 --- a/lib/config/CMakeLists.txt +++ b/lib/config/CMakeLists.txt @@ -28,6 +28,7 @@ set(config_SOURCES configitem.cpp configitem.hpp configitembuilder.cpp configitembuilder.hpp expression.cpp expression.hpp + generator-function.cpp generator-function.hpp objectrule.cpp objectrule.hpp vmops.hpp ${FLEX_config_lexer_OUTPUTS} ${BISON_config_parser_OUTPUTS} diff --git a/lib/config/config_lexer.ll b/lib/config/config_lexer.ll index abfdaffb7..47862e8d5 100644 --- a/lib/config/config_lexer.ll +++ b/lib/config/config_lexer.ll @@ -178,6 +178,7 @@ function return T_FUNCTION; return return T_RETURN; break return T_BREAK; continue return T_CONTINUE; +yield return T_YIELD; for return T_FOR; if return T_IF; else return T_ELSE; diff --git a/lib/config/config_parser.yy b/lib/config/config_parser.yy index 939681e68..cd6049fdc 100644 --- a/lib/config/config_parser.yy +++ b/lib/config/config_parser.yy @@ -151,6 +151,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig %token T_RETURN "return (T_RETURN)" %token T_BREAK "break (T_BREAK)" %token T_CONTINUE "continue (T_CONTINUE)" +%token T_YIELD "yield (T_YIELD)" %token T_FOR "for (T_FOR)" %token T_IF "if (T_IF)" %token T_ELSE "else (T_ELSE)" @@ -191,6 +192,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig %type use_specifier %type use_specifier_items %type use_specifier_item +%type generator_specifier %type object_declaration %right T_FOLLOWS @@ -200,7 +202,7 @@ static void MakeRBinaryOp(Expression** result, Expression *left, Expression *rig %right '?' ':' %left T_LOGICAL_OR %left T_LOGICAL_AND -%left T_RETURN T_BREAK T_CONTINUE +%left T_RETURN T_BREAK T_CONTINUE T_YIELD %left T_IDENTIFIER %left T_BINARY_OR %left T_XOR @@ -357,7 +359,7 @@ object: } object_declaration rterm optional_rterm use_specifier default_specifier ignore_specifier { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope_require_side_effect { @@ -493,7 +495,7 @@ lterm: T_LIBRARY rterm } | T_ASSIGN T_WHERE { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope %dprec 2 { @@ -529,7 +531,7 @@ lterm: T_LIBRARY rterm } | T_IGNORE T_WHERE { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope %dprec 2 { @@ -563,11 +565,21 @@ lterm: T_LIBRARY rterm $$ = MakeLiteralRaw(); } - | T_RETURN optional_rterm + | T_RETURN { - UseFlowControl(context, FlowControlReturn, @$); + UseFlowControl(context, FlowControlReturnVoid, @$); + $$ = new ReturnExpression(std::unique_ptr(MakeLiteralRaw()), @$); + } + | T_RETURN rterm + { + UseFlowControl(context, FlowControlReturnValue, @$); $$ = new ReturnExpression(std::unique_ptr($2), @$); } + | T_YIELD rterm + { + UseFlowControl(context, FlowControlYield, @$); + $$ = new YieldExpression(std::unique_ptr($2), @$); + } | T_BREAK { UseFlowControl(context, FlowControlBreak, @$); @@ -584,7 +596,7 @@ lterm: T_LIBRARY rterm } | T_NAMESPACE rterm { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope_require_side_effect { @@ -625,20 +637,20 @@ lterm: T_LIBRARY rterm $$ = new ForExpression(std::move(*$4), "", std::unique_ptr($6), std::unique_ptr($9), @$); delete $4; } - | T_FUNCTION identifier '(' identifier_items ')' use_specifier + | T_FUNCTION generator_specifier identifier '(' identifier_items ')' use_specifier { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | ($2 ? FlowControlYield : FlowControlReturnValue), false); } rterm_scope { EndFlowControlBlock(context); - std::unique_ptr fexpr{new FunctionExpression(*$2, std::move(*$4), std::move(*$6), std::unique_ptr($8), @$)}; - delete $4; - delete $6; + std::unique_ptr fexpr{new FunctionExpression(*$3, std::move(*$5), std::move(*$7), std::unique_ptr($9), $2, @$)}; + delete $5; + delete $7; - $$ = new SetExpression(MakeIndexer(ScopeThis, std::move(*$2)), OpSetLiteral, std::move(fexpr), @$); - delete $2; + $$ = new SetExpression(MakeIndexer(ScopeThis, std::move(*$3)), OpSetLiteral, std::move(fexpr), @$); + delete $3; } | T_CONST T_IDENTIFIER T_SET rterm { @@ -915,7 +927,7 @@ rterm_no_side_effect_no_dict: T_STRING } | identifier T_FOLLOWS { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope %dprec 2 { @@ -925,7 +937,7 @@ rterm_no_side_effect_no_dict: T_STRING args.emplace_back(std::move(*$1)); delete $1; - $$ = new FunctionExpression("", std::move(args), {}, std::unique_ptr($4), @$); + $$ = new FunctionExpression("", std::move(args), {}, std::unique_ptr($4), false, @$); } | identifier T_FOLLOWS rterm %dprec 1 { @@ -935,17 +947,17 @@ rterm_no_side_effect_no_dict: T_STRING args.emplace_back(std::move(*$1)); delete $1; - $$ = new FunctionExpression("", std::move(args), {}, std::unique_ptr($3), @$); + $$ = new FunctionExpression("", std::move(args), {}, std::unique_ptr($3), false, @$); } | '(' identifier_items ')' use_specifier T_FOLLOWS { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope %dprec 2 { EndFlowControlBlock(context); - $$ = new FunctionExpression("", std::move(*$2), std::move(*$4), std::unique_ptr($7), @$); + $$ = new FunctionExpression("", std::move(*$2), std::move(*$4), std::unique_ptr($7), false, @$); delete $2; delete $4; } @@ -953,7 +965,7 @@ rterm_no_side_effect_no_dict: T_STRING { ASSERT(!dynamic_cast($6)); - $$ = new FunctionExpression("", std::move(*$2), std::move(*$4), std::unique_ptr($6), @$); + $$ = new FunctionExpression("", std::move(*$2), std::move(*$4), std::unique_ptr($6), false, @$); delete $2; delete $4; } @@ -987,21 +999,21 @@ rterm_no_side_effect_no_dict: T_STRING | rterm T_DIVIDE_OP rterm { MakeRBinaryOp(&$$, $1, $3, @1, @3); } | rterm T_MODULO rterm { MakeRBinaryOp(&$$, $1, $3, @1, @3); } | rterm T_XOR rterm { MakeRBinaryOp(&$$, $1, $3, @1, @3); } - | T_FUNCTION '(' identifier_items ')' use_specifier + | T_FUNCTION generator_specifier '(' identifier_items ')' use_specifier { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | ($2 ? FlowControlYield : FlowControlReturnValue), false); } rterm_scope { EndFlowControlBlock(context); - $$ = new FunctionExpression("", std::move(*$3), std::move(*$5), std::unique_ptr($7), @$); - delete $3; - delete $5; + $$ = new FunctionExpression("", std::move(*$4), std::move(*$6), std::unique_ptr($8), $2, @$); + delete $4; + delete $6; } | T_NULLARY_LAMBDA_BEGIN { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } statements T_NULLARY_LAMBDA_END { @@ -1019,7 +1031,7 @@ rterm_no_side_effect_no_dict: T_STRING std::unique_ptr aexpr{new DictExpression(std::move(dlist), @$)}; aexpr->MakeInline(); - $$ = new FunctionExpression("", {}, {}, std::move(aexpr), @$); + $$ = new FunctionExpression("", {}, {}, std::move(aexpr), false, @$); } ; @@ -1105,6 +1117,16 @@ use_specifier_item: identifier } ; +generator_specifier: /* empty */ + { + $$ = false; + } + | T_MULTIPLY + { + $$ = true; + } + ; + apply_for_specifier: /* empty */ | T_FOR '(' optional_var identifier T_FOLLOWS optional_var identifier T_IN rterm ')' { @@ -1147,7 +1169,7 @@ apply: } T_APPLY identifier optional_rterm apply_for_specifier target_type_specifier use_specifier ignore_specifier { - BeginFlowControlBlock(context, FlowControlReturn, false); + BeginFlowControlBlock(context, FlowControlReturnVoid | FlowControlReturnValue, false); } rterm_scope_require_side_effect { diff --git a/lib/config/configcompiler.hpp b/lib/config/configcompiler.hpp index fe00bedfc..761a0e906 100644 --- a/lib/config/configcompiler.hpp +++ b/lib/config/configcompiler.hpp @@ -50,9 +50,11 @@ struct EItemInfo enum FlowControlType { - FlowControlReturn = 1, + FlowControlReturnVoid = 1, FlowControlContinue = 2, - FlowControlBreak = 4 + FlowControlBreak = 4, + FlowControlReturnValue = 8, + FlowControlYield = 16 }; struct ZoneFragment diff --git a/lib/config/expression.cpp b/lib/config/expression.cpp index 09b860cde..68015e624 100644 --- a/lib/config/expression.cpp +++ b/lib/config/expression.cpp @@ -3,6 +3,7 @@ #include "config/expression.hpp" #include "config/configitem.hpp" #include "config/configcompiler.hpp" +#include "config/generator-function.hpp" #include "config/vmops.hpp" #include "base/array.hpp" #include "base/json.hpp" @@ -734,6 +735,15 @@ ExpressionResult ContinueExpression::DoEvaluate(ScriptFrame& frame, DebugHint *d return ExpressionResult(Empty, ResultContinue); } +ExpressionResult YieldExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const +{ + ExpressionResult operand = m_Operand->Evaluate(frame); + CHECK_RESULT(operand); + + l_GeneratorFunction->YieldItem(operand.GetValue()); + return Empty; +} + ExpressionResult IndexerExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const { ExpressionResult operand1 = m_Operand1->Evaluate(frame, dhint); @@ -904,7 +914,7 @@ ExpressionResult ImportDefaultTemplatesExpression::DoEvaluate(ScriptFrame& frame ExpressionResult FunctionExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const { - return VMOps::NewFunction(frame, m_Name, m_Args, m_ClosedVars, m_Expression); + return VMOps::NewFunction(frame, m_Name, m_Args, m_ClosedVars, m_Expression, m_Generator); } ExpressionResult ApplyExpression::DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const diff --git a/lib/config/expression.hpp b/lib/config/expression.hpp index 644548d28..beacd6a90 100644 --- a/lib/config/expression.hpp +++ b/lib/config/expression.hpp @@ -734,6 +734,17 @@ protected: ExpressionResult DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const override; }; +class YieldExpression final : public UnaryExpression +{ +public: + YieldExpression(std::unique_ptr expression, const DebugInfo& debugInfo = DebugInfo()) + : UnaryExpression(std::move(expression), debugInfo) + { } + +protected: + ExpressionResult DoEvaluate(ScriptFrame& frame, DebugHint *dhint) const override; +}; + class GetScopeExpression final : public Expression { public: @@ -812,8 +823,9 @@ class FunctionExpression final : public DebuggableExpression { public: FunctionExpression(String name, std::vector args, - std::map >&& closedVars, std::unique_ptr expression, const DebugInfo& debugInfo = DebugInfo()) - : DebuggableExpression(debugInfo), m_Name(std::move(name)), m_Args(std::move(args)), m_ClosedVars(std::move(closedVars)), m_Expression(expression.release()) + std::map >&& closedVars, std::unique_ptr expression, bool generator, const DebugInfo& debugInfo = DebugInfo()) + : DebuggableExpression(debugInfo), m_Name(std::move(name)), m_Args(std::move(args)), + m_ClosedVars(std::move(closedVars)), m_Expression(expression.release()), m_Generator(generator) { } protected: @@ -824,6 +836,7 @@ private: std::vector m_Args; std::map > m_ClosedVars; Expression::Ptr m_Expression; + bool m_Generator; }; class ApplyExpression final : public DebuggableExpression diff --git a/lib/config/generator-function.cpp b/lib/config/generator-function.cpp new file mode 100644 index 000000000..74249a0e5 --- /dev/null +++ b/lib/config/generator-function.cpp @@ -0,0 +1,138 @@ +/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */ + +#include "base/debug.hpp" +#include "base/exception.hpp" +#include "base/logger.hpp" +#include "base/objectlock.hpp" +#include "base/scriptframe.hpp" +#include "base/shared.hpp" +#include "base/value.hpp" +#include "config/generator-function.hpp" +#include +#include +#include +#include +#include + +using namespace icinga; + +GeneratorFunction::GeneratorFunction(const Expression::Ptr& expression) +{ + auto originFrame (ScriptFrame::GetCurrentFrame()); + + std::promise promise; + auto future (promise.get_future()); + + m_Thread = std::thread([this, expression, originFrame, &promise]() { + ScriptFrame frame (false); + frame = *originFrame; + + promise.set_value(); + + frame.Depth = 0; + + for (std::unique_lock lock (m_ITC.Lock);;) { + if (!m_ITC.NextQueue.empty()) { + break; + } + + if (m_ITC.Shutdown) { + return; + } + + m_ITC.CV.wait(lock); + } + + l_GeneratorFunction = this; + + try { + expression->Evaluate(frame); + } catch (ForcedUnwind) { + } catch (...) { + std::unique_lock lock (m_ITC.Lock); + ASSERT(!m_ITC.NextQueue.empty()); + + auto next (std::move(m_ITC.NextQueue.front())); + m_ITC.NextQueue.pop(); + lock.unlock(); + + next.set_exception(std::current_exception()); + } + + std::unique_lock lock (m_ITC.Lock); + m_ITC.Shutdown = true; + + while (!m_ITC.NextQueue.empty()) { + auto next (std::move(m_ITC.NextQueue.front())); + m_ITC.NextQueue.pop(); + + next.set_value({Empty, false}); + } + }); + + future.wait(); +} + +GeneratorFunction::~GeneratorFunction() +{ + { + std::unique_lock lock (m_ITC.Lock); + m_ITC.Shutdown = true; + m_ITC.CV.notify_all(); + } + + m_Thread.join(); +} + +bool GeneratorFunction::GetNext(Value& out) +{ + std::unique_lock lock (m_ITC.Lock); + + if (m_ITC.Shutdown) { + return false; + } + + std::promise> promise; + auto future (promise.get_future()); + + m_ITC.NextQueue.emplace(std::move(promise)); + m_ITC.CV.notify_all(); + + lock.unlock(); + + future.wait(); + + auto res (future.get()); + + if (res.second) { + out = std::move(res.first); + } + + return res.second; +} + +void GeneratorFunction::YieldItem(Value item) +{ + std::unique_lock lock (m_ITC.Lock); + ASSERT(!m_ITC.NextQueue.empty()); + + auto next (std::move(m_ITC.NextQueue.front())); + m_ITC.NextQueue.pop(); + lock.unlock(); + + next.set_value({std::move(item), true}); + + for (lock.lock();;) { + if (!m_ITC.NextQueue.empty()) { + break; + } + + if (m_ITC.Shutdown) { + throw ForcedUnwind(); + } + + m_ITC.CV.wait(lock); + } +} + +thread_local GeneratorFunction* icinga::l_GeneratorFunction = nullptr; diff --git a/lib/config/generator-function.hpp b/lib/config/generator-function.hpp new file mode 100644 index 000000000..485711096 --- /dev/null +++ b/lib/config/generator-function.hpp @@ -0,0 +1,54 @@ +/* Icinga 2 | (c) 2020 Icinga GmbH | GPLv2+ */ + +#ifndef GENERATOR_FUNCTION_H +#define GENERATOR_FUNCTION_H + +#include "base/generator.hpp" +#include "base/value.hpp" +#include "config/expression.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace icinga +{ + +/** + * Supplier a generator function's items. + * + * @ingroup config + */ +class GeneratorFunction final : public Generator +{ +public: + class ForcedUnwind + { + }; + + GeneratorFunction(const Expression::Ptr& expression); + ~GeneratorFunction() override; + + bool GetNext(Value& out) override; + void YieldItem(Value item); + +private: + std::thread m_Thread; + + struct { + std::queue>> NextQueue; + bool Shutdown = false; + std::mutex Lock; + std::condition_variable CV; + std::exception_ptr Exception; + } m_ITC; +}; + +extern thread_local GeneratorFunction* l_GeneratorFunction; + +} + +#endif /* GENERATOR_FUNCTION_H */ diff --git a/lib/config/vmops.hpp b/lib/config/vmops.hpp index 6d6e840b6..30b0ad7b4 100644 --- a/lib/config/vmops.hpp +++ b/lib/config/vmops.hpp @@ -7,6 +7,7 @@ #include "config/expression.hpp" #include "config/configitembuilder.hpp" #include "config/applyrule.hpp" +#include "config/generator-function.hpp" #include "config/objectrule.hpp" #include "base/debuginfo.hpp" #include "base/array.hpp" @@ -93,11 +94,11 @@ public: } static inline Value NewFunction(ScriptFrame& frame, const String& name, const std::vector& argNames, - const std::map >& closedVars, const Expression::Ptr& expression) + const std::map >& closedVars, const Expression::Ptr& expression, bool generator) { auto evaluatedClosedVars = EvaluateClosedVars(frame, closedVars); - auto wrapper = [argNames, evaluatedClosedVars, expression](const std::vector& arguments) -> Value { + auto wrapper = [argNames, evaluatedClosedVars, expression, generator](const std::vector& arguments) -> Value { if (arguments.size() < argNames.size()) BOOST_THROW_EXCEPTION(std::invalid_argument("Too few arguments for function")); @@ -111,7 +112,11 @@ public: for (std::vector::size_type i = 0; i < std::min(arguments.size(), argNames.size()); i++) frame->Locals->Set(argNames[i], arguments[i]); - return expression->Evaluate(*frame); + if (generator) { + return new GeneratorFunction(expression); + } else { + return expression->Evaluate(*frame); + } }; return new Function(name, wrapper, argNames);