From 9dd2e2a3eca0d1d568bd587887ded9d63feff1f7 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 22 Apr 2025 09:45:29 +0200 Subject: [PATCH] Introduce `JsonEncoder` class --- lib/base/json.cpp | 592 ++++++++++++++++++++-------------------------- lib/base/json.hpp | 66 ++++++ 2 files changed, 320 insertions(+), 338 deletions(-) diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 2d48970bb..4f03e692e 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -2,21 +2,264 @@ #include "base/json.hpp" #include "base/debug.hpp" -#include "base/namespace.hpp" #include "base/dictionary.hpp" -#include "base/array.hpp" +#include "base/namespace.hpp" #include "base/objectlock.hpp" -#include "base/convert.hpp" -#include "base/utility.hpp" -#include -#include -#include +#include #include #include #include using namespace icinga; +JsonEncoder::JsonEncoder(std::string& output, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(output), prettify} +{ +} + +JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(stream), prettify} +{ +} + +JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify) + : m_Pretty(prettify), m_Writer(std::move(w)) +{ +} + +/** + * Encodes a single value into JSON and writes it to the underlying output stream. + * + * This method is the main entry point for encoding JSON data. It takes a value of any type that can + * be represented by our @c Value class recursively and encodes it into JSON in an efficient manner. + * If prettifying is enabled, the JSON output will be formatted with indentation and newlines for better + * readability, and the final JSON will also be terminated by a newline character. + * + * @param value The value to be JSON serialized. + */ +void JsonEncoder::Encode(const Value& value) +{ + switch (value.GetType()) { + case ValueEmpty: + Write("null"); + break; + case ValueBoolean: + Write(value.ToBool() ? "true" : "false"); + break; + case ValueString: + EncodeNlohmannJson(Utility::ValidateUTF8(value.Get())); + break; + case ValueNumber: + EncodeNumber(value.Get()); + break; + case ValueObject: { + const auto& obj = value.Get(); + const auto& type = obj->GetReflectionType(); + if (type == Namespace::TypeInstance) { + static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; + EncodeObject(static_pointer_cast(obj), extractor); + } else if (type == Dictionary::TypeInstance) { + static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; + EncodeObject(static_pointer_cast(obj), extractor); + } else if (type == Array::TypeInstance) { + EncodeArray(static_pointer_cast(obj)); + } else if (auto gen(dynamic_pointer_cast(obj)); gen) { + EncodeValueGenerator(gen); + } else { + // Some other non-serializable object type! + EncodeNlohmannJson(Utility::ValidateUTF8(obj->ToString())); + } + break; + } + default: + VERIFY(!"Invalid variant type."); + } + + // If we are at the top level of the JSON object and prettifying is enabled, we need to end + // the JSON with a newline character to ensure that the output is properly formatted. + if (m_Indent == 0 && m_Pretty) { + Write("\n"); + } +} + +/** + * Encodes an Array object into JSON and writes it to the output stream. + * + * @param array The Array object to be serialized into JSON. + */ +void JsonEncoder::EncodeArray(const Array::Ptr& array) +{ + BeginContainer('['); + ObjectLock olock(array); + bool isEmpty = true; + for (const auto& item : array) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(item); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes a ValueGenerator object into JSON and writes it to the output stream. + * + * This will iterate through the generator, encoding each value it produces until it is exhausted. + * + * @param generator The ValueGenerator object to be serialized into JSON. + */ +void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator) +{ + BeginContainer('['); + bool isEmpty = true; + while (auto result = generator->Next()) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(*result); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer. + * + * @tparam Iterable Type of the container (Namespace or Dictionary). + * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator. + * + * @param container The container to JSON serialize. + * @param extractor The value extractor function used to extract values from the container's iterator. + */ +template +void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor) +{ + static_assert(std::is_same_v || std::is_same_v, + "Container must be a Namespace or Dictionary"); + + BeginContainer('{'); + ObjectLock olock(container); + bool isEmpty = true; + for (const auto& [key, val] : container) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + + EncodeNlohmannJson(Utility::ValidateUTF8(key)); + Write(m_Pretty ? ": " : ":"); + Encode(extractor(val)); + } + EndContainer('}', isEmpty); +} + +/** + * Dumps a nlohmann::json object to the output stream using the serializer. + * + * This function uses the @c nlohmann::detail::serializer to dump the provided @c nlohmann::json + * object to the output stream managed by the @c JsonEncoder. + * + * @param json The nlohmann::json object to encode. + */ +void JsonEncoder::EncodeNlohmannJson(const nlohmann::json& json) const +{ + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::strict); + s.dump(json, m_Pretty, true, 0, 0); +} + +/** + * Encodes a double value into JSON format and writes it to the output stream. + * + * This function checks if the double value can be safely cast to an integer or unsigned integer type + * without loss of precision. If it can, it will serialize it as such; otherwise, it will serialize + * it as a double. This is particularly useful for ensuring that values like 0.0 are serialized as 0, + * which can be important for compatibility with clients like Icinga DB that expect integers in such cases. + * + * @param value The double value to encode as JSON. + */ +void JsonEncoder::EncodeNumber(double value) const +{ + try { + if (value < 0) { + if (auto ll(boost::numeric_cast(value)); ll == value) { + EncodeNlohmannJson(ll); + return; + } + } else if (auto ull(boost::numeric_cast(value)); ull == value) { + EncodeNlohmannJson(ull); + return; + } + // If we reach this point, the value cannot be safely cast to a signed or unsigned integer + // type because it would otherwise lose its precision. If the value was just too large to fit + // into the above types, then boost will throw an exception and end up in the below catch block. + // So, in either case, serialize the number as-is without any casting. + } catch (const boost::bad_numeric_cast&) {} + + EncodeNlohmannJson(value); +} + +/** + * Writes a string to the underlying output stream. + * + * This function writes the provided string view directly to the output stream without any additional formatting. + * + * @param sv The string view to write to the output stream. + */ +void JsonEncoder::Write(const std::string_view& sv) const +{ + m_Writer->write_characters(sv.data(), sv.size()); +} + +/** + * Begins a JSON container (object or array) by writing the opening character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param openChar The character that opens the container (either '{' for objects or '[' for arrays). + */ +void JsonEncoder::BeginContainer(char openChar) +{ + if (m_Pretty) { + m_Indent += m_IndentSize; + if (m_IndentStr.size() < m_Indent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + } + m_Writer->write_character(openChar); +} + +/** + * Ends a JSON container (object or array) by writing the closing character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param closeChar The character that closes the container (either '}' for objects or ']' for arrays). + * @param isContainerEmpty Whether the container is empty, used to determine if a newline should be written. + */ +void JsonEncoder::EndContainer(char closeChar, bool isContainerEmpty) +{ + if (m_Pretty) { + ASSERT(m_Indent >= m_IndentSize); // Ensure we don't underflow the indent size. + m_Indent -= m_IndentSize; + if (!isContainerEmpty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } + } + m_Writer->write_character(closeChar); +} + +/** + * Writes a separator (comma) and an indentation string if pretty-printing is enabled. + * + * This function is used to separate items in a JSON array or object and to maintain the correct indentation level. + * + * @param emitComma Whether to emit a comma. This is typically true for all but the first item in a container. + */ +void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const +{ + if (emitComma) { + Write(","); + } + if (m_Pretty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } +} + class JsonSax : public nlohmann::json_sax { public: @@ -44,165 +287,12 @@ private: void FillCurrentTarget(Value value); }; -const char l_Null[] = "null"; -const char l_False[] = "false"; -const char l_True[] = "true"; -const char l_Indent[] = " "; - -// https://github.com/nlohmann/json/issues/1512 -template -class JsonEncoder -{ -public: - void Null(); - void Boolean(bool value); - void NumberFloat(double value); - void Strng(String value); - void StartObject(); - void Key(String value); - void EndObject(); - void StartArray(); - void EndArray(); - - String GetResult(); - -private: - std::vector m_Result; - String m_CurrentKey; - std::stack> m_CurrentSubtree; - - void AppendChar(char c); - - template - void AppendChars(Iterator begin, Iterator end); - - void AppendJson(nlohmann::json json); - - void BeforeItem(); - - void FinishContainer(char terminator); -}; - -template -void Encode(JsonEncoder& stateMachine, const Value& value); - -template -inline -void EncodeNamespace(JsonEncoder& stateMachine, const Namespace::Ptr& ns) -{ - stateMachine.StartObject(); - - ObjectLock olock(ns); - for (const Namespace::Pair& kv : ns) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second.Val); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeDictionary(JsonEncoder& stateMachine, const Dictionary::Ptr& dict) -{ - stateMachine.StartObject(); - - ObjectLock olock(dict); - for (const Dictionary::Pair& kv : dict) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeArray(JsonEncoder& stateMachine, const Array::Ptr& arr) -{ - stateMachine.StartArray(); - - ObjectLock olock(arr); - for (const Value& value : arr) { - Encode(stateMachine, value); - } - - stateMachine.EndArray(); -} - -template -void Encode(JsonEncoder& stateMachine, const Value& value) -{ - switch (value.GetType()) { - case ValueNumber: - stateMachine.NumberFloat(value.Get()); - break; - - case ValueBoolean: - stateMachine.Boolean(value.ToBool()); - break; - - case ValueString: - stateMachine.Strng(Utility::ValidateUTF8(value.Get())); - break; - - case ValueObject: - { - const Object::Ptr& obj = value.Get(); - - { - Namespace::Ptr ns = dynamic_pointer_cast(obj); - if (ns) { - EncodeNamespace(stateMachine, ns); - break; - } - } - - { - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - if (dict) { - EncodeDictionary(stateMachine, dict); - break; - } - } - - { - Array::Ptr arr = dynamic_pointer_cast(obj); - if (arr) { - EncodeArray(stateMachine, arr); - break; - } - } - - // obj is most likely a function => "Object of type 'Function'" - Encode(stateMachine, obj->ToString()); - break; - } - - case ValueEmpty: - stateMachine.Null(); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - String icinga::JsonEncode(const Value& value, bool pretty_print) { - if (pretty_print) { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult() + "\n"; - } else { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult(); - } + std::string output; + JsonEncoder encoder(output, pretty_print); + encoder.Encode(value); + return String(std::move(output)); } Value icinga::JsonDecode(const String& data) @@ -348,177 +438,3 @@ void JsonSax::FillCurrentTarget(Value value) } } } - -template -inline -void JsonEncoder::Null() -{ - BeforeItem(); - AppendChars((const char*)l_Null, (const char*)l_Null + 4); -} - -template -inline -void JsonEncoder::Boolean(bool value) -{ - BeforeItem(); - - if (value) { - AppendChars((const char*)l_True, (const char*)l_True + 4); - } else { - AppendChars((const char*)l_False, (const char*)l_False + 5); - } -} - -template -inline -void JsonEncoder::NumberFloat(double value) -{ - BeforeItem(); - - // Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int. - if (value < 0) { - long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } else { - unsigned long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } -} - -template -inline -void JsonEncoder::Strng(String value) -{ - BeforeItem(); - AppendJson(std::move(value)); -} - -template -inline -void JsonEncoder::StartObject() -{ - BeforeItem(); - AppendChar('{'); - - m_CurrentSubtree.push(2); -} - -template -inline -void JsonEncoder::Key(String value) -{ - m_CurrentKey = std::move(value); -} - -template -inline -void JsonEncoder::EndObject() -{ - FinishContainer('}'); -} - -template -inline -void JsonEncoder::StartArray() -{ - BeforeItem(); - AppendChar('['); - - m_CurrentSubtree.push(0); -} - -template -inline -void JsonEncoder::EndArray() -{ - FinishContainer(']'); -} - -template -inline -String JsonEncoder::GetResult() -{ - return String(m_Result.begin(), m_Result.end()); -} - -template -inline -void JsonEncoder::AppendChar(char c) -{ - m_Result.emplace_back(c); -} - -template -template -inline -void JsonEncoder::AppendChars(Iterator begin, Iterator end) -{ - m_Result.insert(m_Result.end(), begin, end); -} - -template -inline -void JsonEncoder::AppendJson(nlohmann::json json) -{ - nlohmann::detail::serializer(nlohmann::detail::output_adapter(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0); -} - -template -inline -void JsonEncoder::BeforeItem() -{ - if (!m_CurrentSubtree.empty()) { - auto& node (m_CurrentSubtree.top()); - - if (node[0]) { - AppendChar(','); - } else { - node[0] = true; - } - - if (prettyPrint) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size()); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - if (node[1]) { - AppendJson(std::move(m_CurrentKey)); - AppendChar(':'); - - if (prettyPrint) { - AppendChar(' '); - } - } - } -} - -template -inline -void JsonEncoder::FinishContainer(char terminator) -{ - if (prettyPrint && m_CurrentSubtree.top()[0]) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size() - 1u); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - AppendChar(terminator); - - m_CurrentSubtree.pop(); -} diff --git a/lib/base/json.hpp b/lib/base/json.hpp index 3df6e797e..984a8d1d4 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,6 +4,9 @@ #define JSON_H #include "base/i2-base.hpp" +#include "base/array.hpp" +#include "base/generator.hpp" +#include "base/utility.hpp" #include #include @@ -41,6 +44,69 @@ public: class String; class Value; +/** + * JSON encoder. + * + * This class can be used to encode Icinga Value types into JSON format and write them to an output stream. + * The supported stream types include any @c std::ostream like objects and our own @c AsyncJsonWriter, which + * allows writing JSON data to an Asio stream asynchronously. The nlohmann/json library already provides + * full support for the former stream type, while the latter is fully implemented by our own and satisfies the + * @c nlohmann::detail::output_adapter_protocol<> interface as well. Therefore, any concrete implementation of + * @c AsyncJsonWriter may be used to write the produced JSON directly to an Asio either TCP or TLS stream without + * any additional buffering other than the one used by the Asio buffered_stream<> class internally. + * + * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing + * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the + * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind + * of work to serializer<>. However, Strings are UTF-8 validated beforehand using the @c Utility::ValidateUTF8() + * function and only the validated (copy of the original) String is passed to the serializer. + * + * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object + * is indented with 4 spaces and grows linearly with the depth of the object tree. + * + * @ingroup base + */ +class JsonEncoder +{ +public: + explicit JsonEncoder(std::string& output, bool prettify = false); + explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false); + explicit JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify = false); + + void Encode(const Value& value); + +private: + void EncodeArray(const Array::Ptr& array); + void EncodeValueGenerator(const ValueGenerator::Ptr& generator); + + template + void EncodeObject(const Iterable& container, const ValExtractor& extractor); + + void EncodeNlohmannJson(const nlohmann::json& json) const; + void EncodeNumber(double value) const; + + void Write(const std::string_view& sv) const; + void BeginContainer(char openChar); + void EndContainer(char closeChar, bool isContainerEmpty = false); + void WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const; + + // The number of spaces to use for indentation in prettified JSON. + static constexpr uint8_t m_IndentSize = 4; + + bool m_Pretty; // Whether to pretty-print the JSON output. + unsigned m_Indent{0}; // The current indentation level for pretty-printing. + /** + * Pre-allocate for 8 levels of indentation for pretty-printing. + * + * This is used to avoid reallocating the string on every indent level change. + * The size of this string is dynamically adjusted if the indentation level exceeds its initial size at some point. + */ + std::string m_IndentStr{8*m_IndentSize, ' '}; + + // The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter. + nlohmann::detail::output_adapter_t m_Writer; +}; + String JsonEncode(const Value& value, bool pretty_print = false); Value JsonDecode(const String& data);