From 455d6fcde1765e69751db20694c13ca1bbc68bcf Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 1 Jul 2025 09:49:31 +0200 Subject: [PATCH 01/13] Introduce `ValueGenerator` class --- lib/base/CMakeLists.txt | 1 + lib/base/generator.hpp | 48 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/base/generator.hpp diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index d44254a35..1bb700c3d 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -37,6 +37,7 @@ set(base_SOURCES fifo.cpp fifo.hpp filelogger.cpp filelogger.hpp filelogger-ti.hpp function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp + generator.hpp initialize.cpp initialize.hpp intrusive-ptr.hpp io-engine.cpp io-engine.hpp diff --git a/lib/base/generator.hpp b/lib/base/generator.hpp new file mode 100644 index 000000000..fa41e67fc --- /dev/null +++ b/lib/base/generator.hpp @@ -0,0 +1,48 @@ +/* Icinga 2 | (c) 2025 Icinga GmbH | GPLv2+ */ + +#pragma once + +#include "base/i2-base.hpp" +#include "base/value.hpp" +#include + +namespace icinga +{ + +/** + * ValueGenerator is a class that defines a generator function type for producing Values on demand. + * + * This class is used to create generator functions that can yield any values that can be represented by the + * Icinga Value type. The generator function is exhausted when it returns `std::nullopt`, indicating that there + * are no more values to produce. Subsequent calls to `Next()` will always return `std::nullopt` after exhaustion. + * + * @ingroup base + */ +class ValueGenerator final : public Object +{ +public: + DECLARE_PTR_TYPEDEFS(ValueGenerator); + + /** + * Generates a Value using the provided generator function. + * + * The generator function should return an `std::optional` which contains the produced Value or + * `std::nullopt` when there are no more values to produce. After the generator function returns `std::nullopt`, + * the generator is considered exhausted, and further calls to `Next()` will always return `std::nullopt`. + */ + using GenFunc = std::function()>; + + explicit ValueGenerator(GenFunc generator): m_Generator(std::move(generator)) + { + } + + std::optional Next() const + { + return m_Generator(); + } + +private: + GenFunc m_Generator; // The generator function that produces Values. +}; + +} From 4c0628c24de529da3575ddbf4c10a02f81d3ac9e Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 3 Jul 2025 12:14:31 +0200 Subject: [PATCH 02/13] Allow to defer lock on `ObjectLock` --- lib/base/objectlock.cpp | 12 ++++++++++++ lib/base/objectlock.hpp | 1 + 2 files changed, 13 insertions(+) diff --git a/lib/base/objectlock.cpp b/lib/base/objectlock.cpp index fc0c7c631..fad59160b 100644 --- a/lib/base/objectlock.cpp +++ b/lib/base/objectlock.cpp @@ -18,6 +18,18 @@ ObjectLock::ObjectLock(const Object::Ptr& object) { } +/** + * Constructs a lock for the given object without locking it immediately. + * + * The user must call Lock() explicitly when needed. + * + * @param object The object to lock. + */ +ObjectLock::ObjectLock(const Object::Ptr& object, std::defer_lock_t) + : m_Object(object.get()), m_Locked(false) +{ +} + ObjectLock::ObjectLock(const Object *object) : m_Object(object), m_Locked(false) { diff --git a/lib/base/objectlock.hpp b/lib/base/objectlock.hpp index 8e98641db..abd071c66 100644 --- a/lib/base/objectlock.hpp +++ b/lib/base/objectlock.hpp @@ -15,6 +15,7 @@ struct ObjectLock { public: ObjectLock(const Object::Ptr& object); + ObjectLock(const Object::Ptr& object, std::defer_lock_t); ObjectLock(const Object *object); ObjectLock(const ObjectLock&) = delete; From 8ef921aa5e36080b83fdd3fb9acda73afa05123e Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 8 Jul 2025 17:23:34 +0200 Subject: [PATCH 03/13] Implement bool operator for `ObjectLock` --- lib/base/objectlock.cpp | 12 ++++++++++++ lib/base/objectlock.hpp | 2 ++ 2 files changed, 14 insertions(+) diff --git a/lib/base/objectlock.cpp b/lib/base/objectlock.cpp index fad59160b..9ed9d1f21 100644 --- a/lib/base/objectlock.cpp +++ b/lib/base/objectlock.cpp @@ -65,3 +65,15 @@ void ObjectLock::Unlock() m_Locked = false; } } + +/** + * Returns true if the object is locked, false otherwise. + * + * This operator allows using ObjectLock in boolean contexts. + * + * @returns true if the object is locked, false otherwise. + */ +ObjectLock::operator bool() const +{ + return m_Locked; +} diff --git a/lib/base/objectlock.hpp b/lib/base/objectlock.hpp index abd071c66..328887425 100644 --- a/lib/base/objectlock.hpp +++ b/lib/base/objectlock.hpp @@ -26,6 +26,8 @@ public: void Lock(); void Unlock(); + operator bool() const; + private: const Object *m_Object{nullptr}; bool m_Locked{false}; From 1c61bced03e68096fc467c5f806bff9f08373dcd Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 3 Jul 2025 13:40:10 +0200 Subject: [PATCH 04/13] Introduce `AsyncJsonWriter` output adapter interface --- lib/base/json.cpp | 1 - lib/base/json.hpp | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 56893308a..2d48970bb 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/lib/base/json.hpp b/lib/base/json.hpp index df0ea18a0..3df6e797e 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,10 +4,40 @@ #define JSON_H #include "base/i2-base.hpp" +#include +#include namespace icinga { +/** + * AsyncJsonWriter allows writing JSON data to any output stream asynchronously. + * + * All users of this class must ensure that the underlying output stream will not perform any asynchronous I/O + * operations when the @c write_character() or @c write_characters() methods are called. They shall only perform + * such ops when the @c JsonEncoder allows them to do so by calling the @c Flush() method. + * + * @ingroup base + */ +class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol +{ +public: + /** + * Flush instructs the underlying output stream to write any buffered data to wherever it is supposed to go. + * + * The @c JsonEncoder allows the stream to even perform asynchronous operations in a safe manner by calling + * this method with a dedicated @c boost::asio::yield_context object. The stream must not perform any async + * I/O operations triggered by methods other than this one. Any attempt to do so will result in undefined behavior. + * + * However, this doesn't necessarily enforce the stream to really flush its data immediately, but it's up + * to the implementation to do whatever it needs to. The encoder just gives it a chance to do so by calling + * this method. + * + * @param yield The yield context to use for asynchronous operations. + */ + virtual void Flush(boost::asio::yield_context& yield) = 0; +}; + class String; class Value; From 9dd2e2a3eca0d1d568bd587887ded9d63feff1f7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 22 Apr 2025 09:45:29 +0200 Subject: [PATCH 05/13] Introduce `JsonEncoder` class --- lib/base/json.cpp | 592 ++++++++++++++++++++-------------------------- lib/base/json.hpp | 66 ++++++ 2 files changed, 320 insertions(+), 338 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 2d48970bb..4f03e692e 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -2,21 +2,264 @@ #include "base/json.hpp" #include "base/debug.hpp" -#include "base/namespace.hpp" #include "base/dictionary.hpp" -#include "base/array.hpp" +#include "base/namespace.hpp" #include "base/objectlock.hpp" -#include "base/convert.hpp" -#include "base/utility.hpp" -#include -#include -#include +#include #include #include #include using namespace icinga; +JsonEncoder::JsonEncoder(std::string& output, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(output), prettify} +{ +} + +JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(stream), prettify} +{ +} + +JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify) + : m_Pretty(prettify), m_Writer(std::move(w)) +{ +} + +/** + * Encodes a single value into JSON and writes it to the underlying output stream. + * + * This method is the main entry point for encoding JSON data. It takes a value of any type that can + * be represented by our @c Value class recursively and encodes it into JSON in an efficient manner. + * If prettifying is enabled, the JSON output will be formatted with indentation and newlines for better + * readability, and the final JSON will also be terminated by a newline character. + * + * @param value The value to be JSON serialized. + */ +void JsonEncoder::Encode(const Value& value) +{ + switch (value.GetType()) { + case ValueEmpty: + Write("null"); + break; + case ValueBoolean: + Write(value.ToBool() ? "true" : "false"); + break; + case ValueString: + EncodeNlohmannJson(Utility::ValidateUTF8(value.Get())); + break; + case ValueNumber: + EncodeNumber(value.Get()); + break; + case ValueObject: { + const auto& obj = value.Get(); + const auto& type = obj->GetReflectionType(); + if (type == Namespace::TypeInstance) { + static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; + EncodeObject(static_pointer_cast(obj), extractor); + } else if (type == Dictionary::TypeInstance) { + static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; + EncodeObject(static_pointer_cast(obj), extractor); + } else if (type == Array::TypeInstance) { + EncodeArray(static_pointer_cast(obj)); + } else if (auto gen(dynamic_pointer_cast(obj)); gen) { + EncodeValueGenerator(gen); + } else { + // Some other non-serializable object type! + EncodeNlohmannJson(Utility::ValidateUTF8(obj->ToString())); + } + break; + } + default: + VERIFY(!"Invalid variant type."); + } + + // If we are at the top level of the JSON object and prettifying is enabled, we need to end + // the JSON with a newline character to ensure that the output is properly formatted. + if (m_Indent == 0 && m_Pretty) { + Write("\n"); + } +} + +/** + * Encodes an Array object into JSON and writes it to the output stream. + * + * @param array The Array object to be serialized into JSON. + */ +void JsonEncoder::EncodeArray(const Array::Ptr& array) +{ + BeginContainer('['); + ObjectLock olock(array); + bool isEmpty = true; + for (const auto& item : array) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(item); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes a ValueGenerator object into JSON and writes it to the output stream. + * + * This will iterate through the generator, encoding each value it produces until it is exhausted. + * + * @param generator The ValueGenerator object to be serialized into JSON. + */ +void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator) +{ + BeginContainer('['); + bool isEmpty = true; + while (auto result = generator->Next()) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(*result); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer. + * + * @tparam Iterable Type of the container (Namespace or Dictionary). + * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator. + * + * @param container The container to JSON serialize. + * @param extractor The value extractor function used to extract values from the container's iterator. + */ +template +void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor) +{ + static_assert(std::is_same_v || std::is_same_v, + "Container must be a Namespace or Dictionary"); + + BeginContainer('{'); + ObjectLock olock(container); + bool isEmpty = true; + for (const auto& [key, val] : container) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + + EncodeNlohmannJson(Utility::ValidateUTF8(key)); + Write(m_Pretty ? ": " : ":"); + Encode(extractor(val)); + } + EndContainer('}', isEmpty); +} + +/** + * Dumps a nlohmann::json object to the output stream using the serializer. + * + * This function uses the @c nlohmann::detail::serializer to dump the provided @c nlohmann::json + * object to the output stream managed by the @c JsonEncoder. + * + * @param json The nlohmann::json object to encode. + */ +void JsonEncoder::EncodeNlohmannJson(const nlohmann::json& json) const +{ + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::strict); + s.dump(json, m_Pretty, true, 0, 0); +} + +/** + * Encodes a double value into JSON format and writes it to the output stream. + * + * This function checks if the double value can be safely cast to an integer or unsigned integer type + * without loss of precision. If it can, it will serialize it as such; otherwise, it will serialize + * it as a double. This is particularly useful for ensuring that values like 0.0 are serialized as 0, + * which can be important for compatibility with clients like Icinga DB that expect integers in such cases. + * + * @param value The double value to encode as JSON. + */ +void JsonEncoder::EncodeNumber(double value) const +{ + try { + if (value < 0) { + if (auto ll(boost::numeric_cast(value)); ll == value) { + EncodeNlohmannJson(ll); + return; + } + } else if (auto ull(boost::numeric_cast(value)); ull == value) { + EncodeNlohmannJson(ull); + return; + } + // If we reach this point, the value cannot be safely cast to a signed or unsigned integer + // type because it would otherwise lose its precision. If the value was just too large to fit + // into the above types, then boost will throw an exception and end up in the below catch block. + // So, in either case, serialize the number as-is without any casting. + } catch (const boost::bad_numeric_cast&) {} + + EncodeNlohmannJson(value); +} + +/** + * Writes a string to the underlying output stream. + * + * This function writes the provided string view directly to the output stream without any additional formatting. + * + * @param sv The string view to write to the output stream. + */ +void JsonEncoder::Write(const std::string_view& sv) const +{ + m_Writer->write_characters(sv.data(), sv.size()); +} + +/** + * Begins a JSON container (object or array) by writing the opening character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param openChar The character that opens the container (either '{' for objects or '[' for arrays). + */ +void JsonEncoder::BeginContainer(char openChar) +{ + if (m_Pretty) { + m_Indent += m_IndentSize; + if (m_IndentStr.size() < m_Indent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + } + m_Writer->write_character(openChar); +} + +/** + * Ends a JSON container (object or array) by writing the closing character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param closeChar The character that closes the container (either '}' for objects or ']' for arrays). + * @param isContainerEmpty Whether the container is empty, used to determine if a newline should be written. + */ +void JsonEncoder::EndContainer(char closeChar, bool isContainerEmpty) +{ + if (m_Pretty) { + ASSERT(m_Indent >= m_IndentSize); // Ensure we don't underflow the indent size. + m_Indent -= m_IndentSize; + if (!isContainerEmpty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } + } + m_Writer->write_character(closeChar); +} + +/** + * Writes a separator (comma) and an indentation string if pretty-printing is enabled. + * + * This function is used to separate items in a JSON array or object and to maintain the correct indentation level. + * + * @param emitComma Whether to emit a comma. This is typically true for all but the first item in a container. + */ +void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const +{ + if (emitComma) { + Write(","); + } + if (m_Pretty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } +} + class JsonSax : public nlohmann::json_sax { public: @@ -44,165 +287,12 @@ private: void FillCurrentTarget(Value value); }; -const char l_Null[] = "null"; -const char l_False[] = "false"; -const char l_True[] = "true"; -const char l_Indent[] = " "; - -// https://github.com/nlohmann/json/issues/1512 -template -class JsonEncoder -{ -public: - void Null(); - void Boolean(bool value); - void NumberFloat(double value); - void Strng(String value); - void StartObject(); - void Key(String value); - void EndObject(); - void StartArray(); - void EndArray(); - - String GetResult(); - -private: - std::vector m_Result; - String m_CurrentKey; - std::stack> m_CurrentSubtree; - - void AppendChar(char c); - - template - void AppendChars(Iterator begin, Iterator end); - - void AppendJson(nlohmann::json json); - - void BeforeItem(); - - void FinishContainer(char terminator); -}; - -template -void Encode(JsonEncoder& stateMachine, const Value& value); - -template -inline -void EncodeNamespace(JsonEncoder& stateMachine, const Namespace::Ptr& ns) -{ - stateMachine.StartObject(); - - ObjectLock olock(ns); - for (const Namespace::Pair& kv : ns) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second.Val); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeDictionary(JsonEncoder& stateMachine, const Dictionary::Ptr& dict) -{ - stateMachine.StartObject(); - - ObjectLock olock(dict); - for (const Dictionary::Pair& kv : dict) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeArray(JsonEncoder& stateMachine, const Array::Ptr& arr) -{ - stateMachine.StartArray(); - - ObjectLock olock(arr); - for (const Value& value : arr) { - Encode(stateMachine, value); - } - - stateMachine.EndArray(); -} - -template -void Encode(JsonEncoder& stateMachine, const Value& value) -{ - switch (value.GetType()) { - case ValueNumber: - stateMachine.NumberFloat(value.Get()); - break; - - case ValueBoolean: - stateMachine.Boolean(value.ToBool()); - break; - - case ValueString: - stateMachine.Strng(Utility::ValidateUTF8(value.Get())); - break; - - case ValueObject: - { - const Object::Ptr& obj = value.Get(); - - { - Namespace::Ptr ns = dynamic_pointer_cast(obj); - if (ns) { - EncodeNamespace(stateMachine, ns); - break; - } - } - - { - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - if (dict) { - EncodeDictionary(stateMachine, dict); - break; - } - } - - { - Array::Ptr arr = dynamic_pointer_cast(obj); - if (arr) { - EncodeArray(stateMachine, arr); - break; - } - } - - // obj is most likely a function => "Object of type 'Function'" - Encode(stateMachine, obj->ToString()); - break; - } - - case ValueEmpty: - stateMachine.Null(); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - String icinga::JsonEncode(const Value& value, bool pretty_print) { - if (pretty_print) { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult() + "\n"; - } else { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult(); - } + std::string output; + JsonEncoder encoder(output, pretty_print); + encoder.Encode(value); + return String(std::move(output)); } Value icinga::JsonDecode(const String& data) @@ -348,177 +438,3 @@ void JsonSax::FillCurrentTarget(Value value) } } } - -template -inline -void JsonEncoder::Null() -{ - BeforeItem(); - AppendChars((const char*)l_Null, (const char*)l_Null + 4); -} - -template -inline -void JsonEncoder::Boolean(bool value) -{ - BeforeItem(); - - if (value) { - AppendChars((const char*)l_True, (const char*)l_True + 4); - } else { - AppendChars((const char*)l_False, (const char*)l_False + 5); - } -} - -template -inline -void JsonEncoder::NumberFloat(double value) -{ - BeforeItem(); - - // Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int. - if (value < 0) { - long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } else { - unsigned long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } -} - -template -inline -void JsonEncoder::Strng(String value) -{ - BeforeItem(); - AppendJson(std::move(value)); -} - -template -inline -void JsonEncoder::StartObject() -{ - BeforeItem(); - AppendChar('{'); - - m_CurrentSubtree.push(2); -} - -template -inline -void JsonEncoder::Key(String value) -{ - m_CurrentKey = std::move(value); -} - -template -inline -void JsonEncoder::EndObject() -{ - FinishContainer('}'); -} - -template -inline -void JsonEncoder::StartArray() -{ - BeforeItem(); - AppendChar('['); - - m_CurrentSubtree.push(0); -} - -template -inline -void JsonEncoder::EndArray() -{ - FinishContainer(']'); -} - -template -inline -String JsonEncoder::GetResult() -{ - return String(m_Result.begin(), m_Result.end()); -} - -template -inline -void JsonEncoder::AppendChar(char c) -{ - m_Result.emplace_back(c); -} - -template -template -inline -void JsonEncoder::AppendChars(Iterator begin, Iterator end) -{ - m_Result.insert(m_Result.end(), begin, end); -} - -template -inline -void JsonEncoder::AppendJson(nlohmann::json json) -{ - nlohmann::detail::serializer(nlohmann::detail::output_adapter(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0); -} - -template -inline -void JsonEncoder::BeforeItem() -{ - if (!m_CurrentSubtree.empty()) { - auto& node (m_CurrentSubtree.top()); - - if (node[0]) { - AppendChar(','); - } else { - node[0] = true; - } - - if (prettyPrint) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size()); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - if (node[1]) { - AppendJson(std::move(m_CurrentKey)); - AppendChar(':'); - - if (prettyPrint) { - AppendChar(' '); - } - } - } -} - -template -inline -void JsonEncoder::FinishContainer(char terminator) -{ - if (prettyPrint && m_CurrentSubtree.top()[0]) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size() - 1u); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - AppendChar(terminator); - - m_CurrentSubtree.pop(); -} diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 3df6e797e..984a8d1d4 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,6 +4,9 @@ #define JSON_H #include "base/i2-base.hpp" +#include "base/array.hpp" +#include "base/generator.hpp" +#include "base/utility.hpp" #include #include @@ -41,6 +44,69 @@ public: class String; class Value; +/** + * JSON encoder. + * + * This class can be used to encode Icinga Value types into JSON format and write them to an output stream. + * The supported stream types include any @c std::ostream like objects and our own @c AsyncJsonWriter, which + * allows writing JSON data to an Asio stream asynchronously. The nlohmann/json library already provides + * full support for the former stream type, while the latter is fully implemented by our own and satisfies the + * @c nlohmann::detail::output_adapter_protocol<> interface as well. Therefore, any concrete implementation of + * @c AsyncJsonWriter may be used to write the produced JSON directly to an Asio either TCP or TLS stream without + * any additional buffering other than the one used by the Asio buffered_stream<> class internally. + * + * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing + * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the + * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind + * of work to serializer<>. However, Strings are UTF-8 validated beforehand using the @c Utility::ValidateUTF8() + * function and only the validated (copy of the original) String is passed to the serializer. + * + * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object + * is indented with 4 spaces and grows linearly with the depth of the object tree. + * + * @ingroup base + */ +class JsonEncoder +{ +public: + explicit JsonEncoder(std::string& output, bool prettify = false); + explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false); + explicit JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify = false); + + void Encode(const Value& value); + +private: + void EncodeArray(const Array::Ptr& array); + void EncodeValueGenerator(const ValueGenerator::Ptr& generator); + + template + void EncodeObject(const Iterable& container, const ValExtractor& extractor); + + void EncodeNlohmannJson(const nlohmann::json& json) const; + void EncodeNumber(double value) const; + + void Write(const std::string_view& sv) const; + void BeginContainer(char openChar); + void EndContainer(char closeChar, bool isContainerEmpty = false); + void WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const; + + // The number of spaces to use for indentation in prettified JSON. + static constexpr uint8_t m_IndentSize = 4; + + bool m_Pretty; // Whether to pretty-print the JSON output. + unsigned m_Indent{0}; // The current indentation level for pretty-printing. + /** + * Pre-allocate for 8 levels of indentation for pretty-printing. + * + * This is used to avoid reallocating the string on every indent level change. + * The size of this string is dynamically adjusted if the indentation level exceeds its initial size at some point. + */ + std::string m_IndentStr{8*m_IndentSize, ' '}; + + // The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter. + nlohmann::detail::output_adapter_t m_Writer; +}; + String JsonEncode(const Value& value, bool pretty_print = false); Value JsonDecode(const String& data); From 2461e0415df6ea4b61105fe23c21e24b3c3cc284 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 22 Apr 2025 09:48:09 +0200 Subject: [PATCH 06/13] Introduce `JsonEncode` helper function It's just a wrapper around the `JsonEncoder` class to simplify its usage. --- lib/base/json.cpp | 17 +++++++++++++++-- lib/base/json.hpp | 3 ++- test/base-json.cpp | 39 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 4f03e692e..74bc9f63a 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -287,14 +287,27 @@ private: void FillCurrentTarget(Value value); }; -String icinga::JsonEncode(const Value& value, bool pretty_print) +String icinga::JsonEncode(const Value& value, bool prettify) { std::string output; - JsonEncoder encoder(output, pretty_print); + JsonEncoder encoder(output, prettify); encoder.Encode(value); return String(std::move(output)); } +/** + * Serializes an Icinga Value into a JSON object and writes it to the given output stream. + * + * @param value The value to be JSON serialized. + * @param os The output stream to write the JSON data to. + * @param prettify Whether to pretty print the serialized JSON. + */ +void icinga::JsonEncode(const Value& value, std::ostream& os, bool prettify) +{ + JsonEncoder encoder(os, prettify); + encoder.Encode(value); +} + Value icinga::JsonDecode(const String& data) { String sanitized (Utility::ValidateUTF8(data)); diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 984a8d1d4..63828316c 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -107,7 +107,8 @@ private: nlohmann::detail::output_adapter_t m_Writer; }; -String JsonEncode(const Value& value, bool pretty_print = false); +String JsonEncode(const Value& value, bool prettify = false); +void JsonEncode(const Value& value, std::ostream& os, bool prettify = false); Value JsonDecode(const String& data); } diff --git a/test/base-json.cpp b/test/base-json.cpp index 02bbebb6d..8df282c89 100644 --- a/test/base-json.cpp +++ b/test/base-json.cpp @@ -4,10 +4,13 @@ #include "base/function.hpp" #include "base/namespace.hpp" #include "base/array.hpp" +#include "base/generator.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" #include #include +#include +#include using namespace icinga; @@ -15,26 +18,51 @@ BOOST_AUTO_TEST_SUITE(base_json) BOOST_AUTO_TEST_CASE(encode) { + auto generate = []() -> std::optional { + static int count = 0; + if (++count == 4) { + count = 0; + return std::nullopt; + } + return Value(count); + }; + Dictionary::Ptr input (new Dictionary({ { "array", new Array({ new Namespace() }) }, { "false", false }, - { "float", -1.25 }, + // Use double max value to test JSON encoding of large numbers and trigger boost numeric_cast exceptions + { "max_double", std::numeric_limits::max() }, + // Test the maximum number that can be exact represented by a double is 2^64-2048. + { "max_int_in_double", std::nextafter(std::pow(2, 64), 0.0) }, + { "float", -1.25f }, + { "float_without_fraction", 23.0f }, { "fx", new Function("", []() {}) }, { "int", -42 }, { "null", Value() }, { "string", "LF\nTAB\tAUml\xC3\xA4Ill\xC3" }, { "true", true }, - { "uint", 23u } + { "uint", 23u }, + { "generator", new ValueGenerator(generate) }, + { "empty_generator", new ValueGenerator([]() -> std::optional { return std::nullopt; }) }, })); String output (R"EOF({ "array": [ {} ], + "empty_generator": [], "false": false, "float": -1.25, + "float_without_fraction": 23, "fx": "Object of type 'Function'", + "generator": [ + 1, + 2, + 3 + ], "int": -42, + "max_double": 1.7976931348623157e+308, + "max_int_in_double": 18446744073709549568, "null": null, "string": "LF\nTAB\tAUml\u00e4Ill\ufffd", "true": true, @@ -42,7 +70,12 @@ BOOST_AUTO_TEST_CASE(encode) } )EOF"); - BOOST_CHECK(JsonEncode(input, true) == output); + auto got(JsonEncode(input, true)); + BOOST_CHECK_EQUAL(output, got); + + std::ostringstream oss; + JsonEncode(input, oss, true); + BOOST_CHECK_EQUAL(output, oss.str()); boost::algorithm::replace_all(output, " ", ""); boost::algorithm::replace_all(output, "Objectoftype'Function'", "Object of type 'Function'"); From 57726fbb663fb6ee884a547bc997decc24a85787 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 3 Jul 2025 12:09:16 +0200 Subject: [PATCH 07/13] Do not require olock on frozen `Namespace`, `Dictionary` & `Array` --- lib/base/array.cpp | 11 ++++++++--- lib/base/array.hpp | 4 +++- lib/base/dictionary.cpp | 11 ++++++++--- lib/base/dictionary.hpp | 4 +++- lib/base/namespace.cpp | 11 ++++++++--- lib/base/namespace.hpp | 1 + 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/lib/base/array.cpp b/lib/base/array.cpp index 73adcb279..fe95f5d88 100644 --- a/lib/base/array.cpp +++ b/lib/base/array.cpp @@ -96,7 +96,7 @@ void Array::Add(Value value) */ Array::Iterator Array::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } @@ -110,7 +110,7 @@ Array::Iterator Array::Begin() */ Array::Iterator Array::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } @@ -327,7 +327,12 @@ Array::Ptr Array::Unique() const void Array::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Array::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); } Value Array::GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const diff --git a/lib/base/array.hpp b/lib/base/array.hpp index 9589d1901..fb8257c17 100644 --- a/lib/base/array.hpp +++ b/lib/base/array.hpp @@ -4,6 +4,7 @@ #define ARRAY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/objectlock.hpp" #include "base/value.hpp" #include @@ -98,13 +99,14 @@ public: Array::Ptr Unique() const; void Freeze(); + bool Frozen() const; Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; private: std::vector m_Data; /**< The data for the array. */ - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Array::Iterator begin(const Array::Ptr& x); diff --git a/lib/base/dictionary.cpp b/lib/base/dictionary.cpp index 89b6c4cbc..c45bee3d8 100644 --- a/lib/base/dictionary.cpp +++ b/lib/base/dictionary.cpp @@ -132,7 +132,7 @@ bool Dictionary::Contains(const String& key) const */ Dictionary::Iterator Dictionary::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } @@ -146,7 +146,7 @@ Dictionary::Iterator Dictionary::Begin() */ Dictionary::Iterator Dictionary::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } @@ -276,7 +276,12 @@ String Dictionary::ToString() const void Dictionary::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Dictionary::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); } Value Dictionary::GetFieldByName(const String& field, bool, const DebugInfo& debugInfo) const diff --git a/lib/base/dictionary.hpp b/lib/base/dictionary.hpp index ba2fbe82c..f95bcc699 100644 --- a/lib/base/dictionary.hpp +++ b/lib/base/dictionary.hpp @@ -4,6 +4,7 @@ #define DICTIONARY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/object.hpp" #include "base/value.hpp" #include @@ -69,6 +70,7 @@ public: String ToString() const override; void Freeze(); + bool Frozen() const; Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; @@ -78,7 +80,7 @@ public: private: std::map m_Data; /**< The data for the dictionary. */ mutable std::shared_timed_mutex m_DataMutex; - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Dictionary::Iterator begin(const Dictionary::Ptr& x); diff --git a/lib/base/namespace.cpp b/lib/base/namespace.cpp index 1f53efc92..85ed7fa1d 100644 --- a/lib/base/namespace.cpp +++ b/lib/base/namespace.cpp @@ -119,7 +119,12 @@ void Namespace::Remove(const String& field) void Namespace::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Namespace::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); } std::shared_lock Namespace::ReadLockUnlessFrozen() const @@ -160,14 +165,14 @@ bool Namespace::GetOwnField(const String& field, Value *result) const Namespace::Iterator Namespace::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } Namespace::Iterator Namespace::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } diff --git a/lib/base/namespace.hpp b/lib/base/namespace.hpp index 1a028e2c5..c7f1a7e0a 100644 --- a/lib/base/namespace.hpp +++ b/lib/base/namespace.hpp @@ -73,6 +73,7 @@ public: bool Contains(const String& field) const; void Remove(const String& field); void Freeze(); + bool Frozen() const; Iterator Begin(); Iterator End(); From 398b5e3193b9d5f8f93824602a28595afb332939 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 8 Jul 2025 17:27:34 +0200 Subject: [PATCH 08/13] Implement `LockIfRequired()` method for `Namespace`, `Dictionary` & `Array` --- lib/base/array.cpp | 14 ++++++++++++++ lib/base/array.hpp | 1 + lib/base/dictionary.cpp | 15 ++++++++++++++- lib/base/dictionary.hpp | 2 ++ lib/base/namespace.cpp | 15 ++++++++++++++- lib/base/namespace.hpp | 2 ++ 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib/base/array.cpp b/lib/base/array.cpp index fe95f5d88..9906d6496 100644 --- a/lib/base/array.cpp +++ b/lib/base/array.cpp @@ -335,6 +335,20 @@ bool Array::Frozen() const return m_Frozen.load(std::memory_order_acquire); } +/** + * Returns an already locked ObjectLock if the array is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Array::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); +} + Value Array::GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const { int index; diff --git a/lib/base/array.hpp b/lib/base/array.hpp index fb8257c17..f8f923494 100644 --- a/lib/base/array.hpp +++ b/lib/base/array.hpp @@ -100,6 +100,7 @@ public: Array::Ptr Unique() const; void Freeze(); bool Frozen() const; + ObjectLock LockIfRequired(); Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; diff --git a/lib/base/dictionary.cpp b/lib/base/dictionary.cpp index c45bee3d8..2599a1039 100644 --- a/lib/base/dictionary.cpp +++ b/lib/base/dictionary.cpp @@ -1,7 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/dictionary.hpp" -#include "base/objectlock.hpp" #include "base/debug.hpp" #include "base/primitivetype.hpp" #include "base/configwriter.hpp" @@ -284,6 +283,20 @@ bool Dictionary::Frozen() const return m_Frozen.load(std::memory_order_acquire); } +/** + * Returns an already locked ObjectLock if the dictionary is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Dictionary::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); +} + Value Dictionary::GetFieldByName(const String& field, bool, const DebugInfo& debugInfo) const { Value value; diff --git a/lib/base/dictionary.hpp b/lib/base/dictionary.hpp index f95bcc699..f0bbf1b46 100644 --- a/lib/base/dictionary.hpp +++ b/lib/base/dictionary.hpp @@ -6,6 +6,7 @@ #include "base/i2-base.hpp" #include "base/atomic.hpp" #include "base/object.hpp" +#include "base/objectlock.hpp" #include "base/value.hpp" #include #include @@ -71,6 +72,7 @@ public: void Freeze(); bool Frozen() const; + ObjectLock LockIfRequired(); Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; diff --git a/lib/base/namespace.cpp b/lib/base/namespace.cpp index 85ed7fa1d..383b2ecbd 100644 --- a/lib/base/namespace.cpp +++ b/lib/base/namespace.cpp @@ -1,7 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/namespace.hpp" -#include "base/objectlock.hpp" #include "base/debug.hpp" #include "base/primitivetype.hpp" #include "base/debuginfo.hpp" @@ -127,6 +126,20 @@ bool Namespace::Frozen() const return m_Frozen.load(std::memory_order_acquire); } +/** + * Returns an already locked ObjectLock if the namespace is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Namespace::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); +} + std::shared_lock Namespace::ReadLockUnlessFrozen() const { if (m_Frozen.load(std::memory_order_relaxed)) { diff --git a/lib/base/namespace.hpp b/lib/base/namespace.hpp index c7f1a7e0a..194671099 100644 --- a/lib/base/namespace.hpp +++ b/lib/base/namespace.hpp @@ -5,6 +5,7 @@ #include "base/i2-base.hpp" #include "base/object.hpp" +#include "base/objectlock.hpp" #include "base/shared-object.hpp" #include "base/value.hpp" #include "base/debuginfo.hpp" @@ -74,6 +75,7 @@ public: void Remove(const String& field); void Freeze(); bool Frozen() const; + ObjectLock LockIfRequired(); Iterator Begin(); Iterator End(); From dad4c0889f753781b64d67466bdef0d2c1f1e00d Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 3 Jul 2025 18:02:30 +0200 Subject: [PATCH 09/13] JsonEncoder: lock olock conditionally & flush output regularly --- lib/base/json.cpp | 78 ++++++++++++++++++++++++++++++++++++++--------- lib/base/json.hpp | 11 ++++--- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 74bc9f63a..e2749bbde 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -23,7 +23,7 @@ JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) } JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify) - : m_Pretty(prettify), m_Writer(std::move(w)) + : m_IsAsyncWriter{dynamic_cast(w.get()) != nullptr}, m_Pretty(prettify), m_Writer(std::move(w)) { } @@ -35,9 +35,18 @@ JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool pretti * If prettifying is enabled, the JSON output will be formatted with indentation and newlines for better * readability, and the final JSON will also be terminated by a newline character. * + * @note If the used output adapter performs asynchronous I/O operations (it's derived from @c AsyncJsonWriter), + * please provide a @c boost::asio::yield_context object to allow the encoder to flush the output stream in a + * safe manner. The encoder will try to regularly give the output stream a chance to flush its data when it is + * safe to do so, but for this to work, there must be a valid yield context provided. Otherwise, the encoder + * will not attempt to flush the output stream at all, which may lead to huge memory consumption when encoding + * large JSON objects or arrays. + * * @param value The value to be JSON serialized. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock on the parent containers. */ -void JsonEncoder::Encode(const Value& value) +void JsonEncoder::Encode(const Value& value, boost::asio::yield_context* yc) { switch (value.GetType()) { case ValueEmpty: @@ -57,14 +66,14 @@ void JsonEncoder::Encode(const Value& value) const auto& type = obj->GetReflectionType(); if (type == Namespace::TypeInstance) { static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; - EncodeObject(static_pointer_cast(obj), extractor); + EncodeObject(static_pointer_cast(obj), extractor, yc); } else if (type == Dictionary::TypeInstance) { static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; - EncodeObject(static_pointer_cast(obj), extractor); + EncodeObject(static_pointer_cast(obj), extractor, yc); } else if (type == Array::TypeInstance) { - EncodeArray(static_pointer_cast(obj)); + EncodeArray(static_pointer_cast(obj), yc); } else if (auto gen(dynamic_pointer_cast(obj)); gen) { - EncodeValueGenerator(gen); + EncodeValueGenerator(gen, yc); } else { // Some other non-serializable object type! EncodeNlohmannJson(Utility::ValidateUTF8(obj->ToString())); @@ -86,16 +95,23 @@ void JsonEncoder::Encode(const Value& value) * Encodes an Array object into JSON and writes it to the output stream. * * @param array The Array object to be serialized into JSON. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock. */ -void JsonEncoder::EncodeArray(const Array::Ptr& array) +void JsonEncoder::EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc) { BeginContainer('['); - ObjectLock olock(array); + auto olock = array->LockIfRequired(); + if (olock) { + yc = nullptr; // We've acquired an object lock, never allow asynchronous operations. + } + bool isEmpty = true; for (const auto& item : array) { WriteSeparatorAndIndentStrIfNeeded(!isEmpty); isEmpty = false; - Encode(item); + Encode(item, yc); + FlushIfSafe(yc); } EndContainer(']', isEmpty); } @@ -106,15 +122,18 @@ void JsonEncoder::EncodeArray(const Array::Ptr& array) * This will iterate through the generator, encoding each value it produces until it is exhausted. * * @param generator The ValueGenerator object to be serialized into JSON. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock on the parent containers. */ -void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator) +void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc) { BeginContainer('['); bool isEmpty = true; while (auto result = generator->Next()) { WriteSeparatorAndIndentStrIfNeeded(!isEmpty); isEmpty = false; - Encode(*result); + Encode(*result, yc); + FlushIfSafe(yc); } EndContainer(']', isEmpty); } @@ -127,15 +146,21 @@ void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator) * * @param container The container to JSON serialize. * @param extractor The value extractor function used to extract values from the container's iterator. + * @param yc The optional yield context for asynchronous operations. It will only be set when the encoder + * has not acquired any object lock on the parent containers, allowing safe asynchronous operations. */ template -void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor) +void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc) { static_assert(std::is_same_v || std::is_same_v, "Container must be a Namespace or Dictionary"); BeginContainer('{'); - ObjectLock olock(container); + auto olock = container->LockIfRequired(); + if (olock) { + yc = nullptr; // We've acquired an object lock, never allow asynchronous operations. + } + bool isEmpty = true; for (const auto& [key, val] : container) { WriteSeparatorAndIndentStrIfNeeded(!isEmpty); @@ -143,7 +168,9 @@ void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& ex EncodeNlohmannJson(Utility::ValidateUTF8(key)); Write(m_Pretty ? ": " : ":"); - Encode(extractor(val)); + + Encode(extractor(val), yc); + FlushIfSafe(yc); } EndContainer('}', isEmpty); } @@ -193,6 +220,29 @@ void JsonEncoder::EncodeNumber(double value) const EncodeNlohmannJson(value); } +/** + * Flushes the output stream if it is safe to do so. + * + * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired + * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations + * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed. + * + * When the @c yc parameter is provided, it indicates that it's safe to perform asynchronous operations, + * and the function will attempt to flush if the writer is an instance of @c AsyncJsonWriter. + * + * @param yc The yield context to use for asynchronous operations. + */ +void JsonEncoder::FlushIfSafe(boost::asio::yield_context* yc) const +{ + if (yc && m_IsAsyncWriter) { + // The m_IsAsyncWriter flag is a constant, and it will never change, so we can safely static + // cast the m_Writer to AsyncJsonWriter without any additional checks as it is guaranteed + // to be an instance of AsyncJsonWriter when m_IsAsyncWriter is true. + auto ajw(static_cast(m_Writer.get())); + ajw->Flush(*yc); + } +} + /** * Writes a string to the underlying output stream. * diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 63828316c..464a926f5 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -73,18 +73,20 @@ public: explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false); explicit JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify = false); - void Encode(const Value& value); + void Encode(const Value& value, boost::asio::yield_context* yc = nullptr); private: - void EncodeArray(const Array::Ptr& array); - void EncodeValueGenerator(const ValueGenerator::Ptr& generator); + void EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc); + void EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc); template - void EncodeObject(const Iterable& container, const ValExtractor& extractor); + void EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc); void EncodeNlohmannJson(const nlohmann::json& json) const; void EncodeNumber(double value) const; + void FlushIfSafe(boost::asio::yield_context* yc) const; + void Write(const std::string_view& sv) const; void BeginContainer(char openChar); void EndContainer(char closeChar, bool isContainerEmpty = false); @@ -93,6 +95,7 @@ private: // The number of spaces to use for indentation in prettified JSON. static constexpr uint8_t m_IndentSize = 4; + const bool m_IsAsyncWriter; // Whether the writer is an instance of AsyncJsonWriter. bool m_Pretty; // Whether to pretty-print the JSON output. unsigned m_Indent{0}; // The current indentation level for pretty-printing. /** From 89418f38ee845c1d67c006c4d3fff77791c1b6ff Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 4 Jul 2025 16:51:26 +0200 Subject: [PATCH 10/13] JsonEncoder: let the serializer replace invalid UTF-8 characters Replacing invalid UTF-8 characters beforehand by our selves doesn't make any sense, the serializer can literally perform the same replacement ops with the exact same Unicode replacement character (U+FFFD) on its own. So, why not just use it directly? Instead of wasting memory on a temporary `String` object to always UTF-8 validate every and each value, we just use the serializer to directly to dump the replaced char (if any) into the output writer. No memory waste, no fuss! --- lib/base/json.cpp | 13 ++++++++----- lib/base/json.hpp | 4 +--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index e2749bbde..00d114731 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -5,6 +5,7 @@ #include "base/dictionary.hpp" #include "base/namespace.hpp" #include "base/objectlock.hpp" +#include "base/utility.hpp" #include #include #include @@ -56,7 +57,7 @@ void JsonEncoder::Encode(const Value& value, boost::asio::yield_context* yc) Write(value.ToBool() ? "true" : "false"); break; case ValueString: - EncodeNlohmannJson(Utility::ValidateUTF8(value.Get())); + EncodeNlohmannJson(value.Get()); break; case ValueNumber: EncodeNumber(value.Get()); @@ -76,7 +77,7 @@ void JsonEncoder::Encode(const Value& value, boost::asio::yield_context* yc) EncodeValueGenerator(gen, yc); } else { // Some other non-serializable object type! - EncodeNlohmannJson(Utility::ValidateUTF8(obj->ToString())); + EncodeNlohmannJson(obj->ToString()); } break; } @@ -166,7 +167,7 @@ void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& ex WriteSeparatorAndIndentStrIfNeeded(!isEmpty); isEmpty = false; - EncodeNlohmannJson(Utility::ValidateUTF8(key)); + EncodeNlohmannJson(key); Write(m_Pretty ? ": " : ":"); Encode(extractor(val), yc); @@ -179,13 +180,15 @@ void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& ex * Dumps a nlohmann::json object to the output stream using the serializer. * * This function uses the @c nlohmann::detail::serializer to dump the provided @c nlohmann::json - * object to the output stream managed by the @c JsonEncoder. + * object to the output stream managed by the @c JsonEncoder. Strings will be properly escaped, and + * if any invalid UTF-8 sequences are encountered, it will replace them with the Unicode replacement + * character (U+FFFD). * * @param json The nlohmann::json object to encode. */ void JsonEncoder::EncodeNlohmannJson(const nlohmann::json& json) const { - nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::strict); + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::replace); s.dump(json, m_Pretty, true, 0, 0); } diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 464a926f5..238c9bc53 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -6,7 +6,6 @@ #include "base/i2-base.hpp" #include "base/array.hpp" #include "base/generator.hpp" -#include "base/utility.hpp" #include #include @@ -58,8 +57,7 @@ class Value; * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind - * of work to serializer<>. However, Strings are UTF-8 validated beforehand using the @c Utility::ValidateUTF8() - * function and only the validated (copy of the original) String is passed to the serializer. + * of work to serializer<>. * * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object * is indented with 4 spaces and grows linearly with the depth of the object tree. From cd1ab7548cc7d1486cae95870b4f8dde52fbee8b Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 11 Jul 2025 12:56:52 +0200 Subject: [PATCH 11/13] Rename `AsyncJsonWriter::Flush()` -> `MayFlush()` to reflect its usage --- lib/base/json.cpp | 2 +- lib/base/json.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 00d114731..8149e3f82 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -242,7 +242,7 @@ void JsonEncoder::FlushIfSafe(boost::asio::yield_context* yc) const // cast the m_Writer to AsyncJsonWriter without any additional checks as it is guaranteed // to be an instance of AsyncJsonWriter when m_IsAsyncWriter is true. auto ajw(static_cast(m_Writer.get())); - ajw->Flush(*yc); + ajw->MayFlush(*yc); } } diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 238c9bc53..badf4d7ef 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -17,7 +17,7 @@ namespace icinga * * All users of this class must ensure that the underlying output stream will not perform any asynchronous I/O * operations when the @c write_character() or @c write_characters() methods are called. They shall only perform - * such ops when the @c JsonEncoder allows them to do so by calling the @c Flush() method. + * such ops when the @c JsonEncoder allows them to do so by calling the @c MayFlush() method. * * @ingroup base */ @@ -25,7 +25,7 @@ class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol { public: /** - * Flush instructs the underlying output stream to write any buffered data to wherever it is supposed to go. + * It instructs the underlying output stream to write any buffered data to wherever it is supposed to go. * * The @c JsonEncoder allows the stream to even perform asynchronous operations in a safe manner by calling * this method with a dedicated @c boost::asio::yield_context object. The stream must not perform any async @@ -37,7 +37,7 @@ public: * * @param yield The yield context to use for asynchronous operations. */ - virtual void Flush(boost::asio::yield_context& yield) = 0; + virtual void MayFlush(boost::asio::yield_context& yield) = 0; }; class String; From 82b80e24c16497052e374f20dde46d7c912e611f Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 11 Jul 2025 12:58:01 +0200 Subject: [PATCH 12/13] fix comment --- lib/base/json.hpp | 4 +--- test/base-json.cpp | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/base/json.hpp b/lib/base/json.hpp index badf4d7ef..868f8f0ac 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -50,9 +50,7 @@ class Value; * The supported stream types include any @c std::ostream like objects and our own @c AsyncJsonWriter, which * allows writing JSON data to an Asio stream asynchronously. The nlohmann/json library already provides * full support for the former stream type, while the latter is fully implemented by our own and satisfies the - * @c nlohmann::detail::output_adapter_protocol<> interface as well. Therefore, any concrete implementation of - * @c AsyncJsonWriter may be used to write the produced JSON directly to an Asio either TCP or TLS stream without - * any additional buffering other than the one used by the Asio buffered_stream<> class internally. + * @c nlohmann::detail::output_adapter_protocol<> interface as well. * * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the diff --git a/test/base-json.cpp b/test/base-json.cpp index 8df282c89..4ae17bf09 100644 --- a/test/base-json.cpp +++ b/test/base-json.cpp @@ -32,7 +32,7 @@ BOOST_AUTO_TEST_CASE(encode) { "false", false }, // Use double max value to test JSON encoding of large numbers and trigger boost numeric_cast exceptions { "max_double", std::numeric_limits::max() }, - // Test the maximum number that can be exact represented by a double is 2^64-2048. + // Test the largest uint64_t value that has an exact double representation (2^64-2048). { "max_int_in_double", std::nextafter(std::pow(2, 64), 0.0) }, { "float", -1.25f }, { "float_without_fraction", 23.0f }, From 1f15f0ff07a8bcc1eb9824c969e59824bde3af82 Mon Sep 17 00:00:00 2001 From: Julian Brost Date: Fri, 11 Jul 2025 11:18:24 +0200 Subject: [PATCH 13/13] JsonEncoder: wrap writer for flushing This commit intruduces a small helper class that wraps any writer and provides a flush operation that performs the corresponding action if the writer is an AsyncJsonWriter and does nothing otherwise. --- lib/base/json.cpp | 61 ++++++++++++++++++++++++++--------------------- lib/base/json.hpp | 17 ++++++++++--- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 8149e3f82..531ab45b2 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -24,7 +24,7 @@ JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) } JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify) - : m_IsAsyncWriter{dynamic_cast(w.get()) != nullptr}, m_Pretty(prettify), m_Writer(std::move(w)) + : m_Pretty(prettify), m_Writer(std::move(w)), m_Flusher{m_Writer} { } @@ -112,7 +112,7 @@ void JsonEncoder::EncodeArray(const Array::Ptr& array, boost::asio::yield_contex WriteSeparatorAndIndentStrIfNeeded(!isEmpty); isEmpty = false; Encode(item, yc); - FlushIfSafe(yc); + m_Flusher.FlushIfSafe(yc); } EndContainer(']', isEmpty); } @@ -134,7 +134,7 @@ void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator, boo WriteSeparatorAndIndentStrIfNeeded(!isEmpty); isEmpty = false; Encode(*result, yc); - FlushIfSafe(yc); + m_Flusher.FlushIfSafe(yc); } EndContainer(']', isEmpty); } @@ -171,7 +171,7 @@ void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& ex Write(m_Pretty ? ": " : ":"); Encode(extractor(val), yc); - FlushIfSafe(yc); + m_Flusher.FlushIfSafe(yc); } EndContainer('}', isEmpty); } @@ -223,29 +223,6 @@ void JsonEncoder::EncodeNumber(double value) const EncodeNlohmannJson(value); } -/** - * Flushes the output stream if it is safe to do so. - * - * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired - * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations - * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed. - * - * When the @c yc parameter is provided, it indicates that it's safe to perform asynchronous operations, - * and the function will attempt to flush if the writer is an instance of @c AsyncJsonWriter. - * - * @param yc The yield context to use for asynchronous operations. - */ -void JsonEncoder::FlushIfSafe(boost::asio::yield_context* yc) const -{ - if (yc && m_IsAsyncWriter) { - // The m_IsAsyncWriter flag is a constant, and it will never change, so we can safely static - // cast the m_Writer to AsyncJsonWriter without any additional checks as it is guaranteed - // to be an instance of AsyncJsonWriter when m_IsAsyncWriter is true. - auto ajw(static_cast(m_Writer.get())); - ajw->MayFlush(*yc); - } -} - /** * Writes a string to the underlying output stream. * @@ -313,6 +290,36 @@ void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const } } +/** + * Wraps any writer of type @c nlohmann::detail::output_adapter_t into a Flusher + * + * @param w The writer to wrap. + */ +JsonEncoder::Flusher::Flusher(const nlohmann::detail::output_adapter_t& w) + : m_AsyncWriter(dynamic_cast(w.get())) +{ +} + +/** + * Flushes the underlying writer if it supports that operation and is safe to do so. + * + * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired + * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations + * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed. + * + * When the @c yc parameter is provided, it indicates that it's safe to perform asynchronous operations, + * and the function will attempt to flush if the writer is an instance of @c AsyncJsonWriter. Otherwise, + * this function does nothing. + * + * @param yc The yield context to use for asynchronous operations. + */ +void JsonEncoder::Flusher::FlushIfSafe(boost::asio::yield_context* yc) const +{ + if (yc && m_AsyncWriter) { + m_AsyncWriter->MayFlush(*yc); + } +} + class JsonSax : public nlohmann::json_sax { public: diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 868f8f0ac..a5ed46280 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -81,8 +81,6 @@ private: void EncodeNlohmannJson(const nlohmann::json& json) const; void EncodeNumber(double value) const; - void FlushIfSafe(boost::asio::yield_context* yc) const; - void Write(const std::string_view& sv) const; void BeginContainer(char openChar); void EndContainer(char closeChar, bool isContainerEmpty = false); @@ -91,7 +89,6 @@ private: // The number of spaces to use for indentation in prettified JSON. static constexpr uint8_t m_IndentSize = 4; - const bool m_IsAsyncWriter; // Whether the writer is an instance of AsyncJsonWriter. bool m_Pretty; // Whether to pretty-print the JSON output. unsigned m_Indent{0}; // The current indentation level for pretty-printing. /** @@ -104,6 +101,20 @@ private: // The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter. nlohmann::detail::output_adapter_t m_Writer; + + /** + * This class wraps any @c nlohmann::detail::output_adapter_t writer and provides a method to flush it as + * required. Only @c AsyncJsonWriter supports the flush operation, however, this class is also safe to use with + * other writer types and the flush method does nothing for them. + */ + class Flusher { + public: + explicit Flusher(const nlohmann::detail::output_adapter_t& w); + void FlushIfSafe(boost::asio::yield_context* yc) const; + + private: + AsyncJsonWriter* m_AsyncWriter; + } m_Flusher; }; String JsonEncode(const Value& value, bool prettify = false);