diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index d44254a35..1bb700c3d 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -37,6 +37,7 @@ set(base_SOURCES fifo.cpp fifo.hpp filelogger.cpp filelogger.hpp filelogger-ti.hpp function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp + generator.hpp initialize.cpp initialize.hpp intrusive-ptr.hpp io-engine.cpp io-engine.hpp diff --git a/lib/base/array.cpp b/lib/base/array.cpp index 73adcb279..9906d6496 100644 --- a/lib/base/array.cpp +++ b/lib/base/array.cpp @@ -96,7 +96,7 @@ void Array::Add(Value value) */ Array::Iterator Array::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } @@ -110,7 +110,7 @@ Array::Iterator Array::Begin() */ Array::Iterator Array::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } @@ -327,7 +327,26 @@ Array::Ptr Array::Unique() const void Array::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Array::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); +} + +/** + * Returns an already locked ObjectLock if the array is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Array::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); } Value Array::GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const diff --git a/lib/base/array.hpp b/lib/base/array.hpp index 9589d1901..f8f923494 100644 --- a/lib/base/array.hpp +++ b/lib/base/array.hpp @@ -4,6 +4,7 @@ #define ARRAY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/objectlock.hpp" #include "base/value.hpp" #include @@ -98,13 +99,15 @@ public: Array::Ptr Unique() const; void Freeze(); + bool Frozen() const; + ObjectLock LockIfRequired(); Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; private: std::vector m_Data; /**< The data for the array. */ - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Array::Iterator begin(const Array::Ptr& x); diff --git a/lib/base/dictionary.cpp b/lib/base/dictionary.cpp index 89b6c4cbc..2599a1039 100644 --- a/lib/base/dictionary.cpp +++ b/lib/base/dictionary.cpp @@ -1,7 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/dictionary.hpp" -#include "base/objectlock.hpp" #include "base/debug.hpp" #include "base/primitivetype.hpp" #include "base/configwriter.hpp" @@ -132,7 +131,7 @@ bool Dictionary::Contains(const String& key) const */ Dictionary::Iterator Dictionary::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } @@ -146,7 +145,7 @@ Dictionary::Iterator Dictionary::Begin() */ Dictionary::Iterator Dictionary::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } @@ -276,7 +275,26 @@ String Dictionary::ToString() const void Dictionary::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Dictionary::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); +} + +/** + * Returns an already locked ObjectLock if the dictionary is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Dictionary::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); } Value Dictionary::GetFieldByName(const String& field, bool, const DebugInfo& debugInfo) const diff --git a/lib/base/dictionary.hpp b/lib/base/dictionary.hpp index ba2fbe82c..f0bbf1b46 100644 --- a/lib/base/dictionary.hpp +++ b/lib/base/dictionary.hpp @@ -4,7 +4,9 @@ #define DICTIONARY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/object.hpp" +#include "base/objectlock.hpp" #include "base/value.hpp" #include #include @@ -69,6 +71,8 @@ public: String ToString() const override; void Freeze(); + bool Frozen() const; + ObjectLock LockIfRequired(); Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override; @@ -78,7 +82,7 @@ public: private: std::map m_Data; /**< The data for the dictionary. */ mutable std::shared_timed_mutex m_DataMutex; - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Dictionary::Iterator begin(const Dictionary::Ptr& x); diff --git a/lib/base/generator.hpp b/lib/base/generator.hpp new file mode 100644 index 000000000..fa41e67fc --- /dev/null +++ b/lib/base/generator.hpp @@ -0,0 +1,48 @@ +/* Icinga 2 | (c) 2025 Icinga GmbH | GPLv2+ */ + +#pragma once + +#include "base/i2-base.hpp" +#include "base/value.hpp" +#include + +namespace icinga +{ + +/** + * ValueGenerator is a class that defines a generator function type for producing Values on demand. + * + * This class is used to create generator functions that can yield any values that can be represented by the + * Icinga Value type. The generator function is exhausted when it returns `std::nullopt`, indicating that there + * are no more values to produce. Subsequent calls to `Next()` will always return `std::nullopt` after exhaustion. + * + * @ingroup base + */ +class ValueGenerator final : public Object +{ +public: + DECLARE_PTR_TYPEDEFS(ValueGenerator); + + /** + * Generates a Value using the provided generator function. + * + * The generator function should return an `std::optional` which contains the produced Value or + * `std::nullopt` when there are no more values to produce. After the generator function returns `std::nullopt`, + * the generator is considered exhausted, and further calls to `Next()` will always return `std::nullopt`. + */ + using GenFunc = std::function()>; + + explicit ValueGenerator(GenFunc generator): m_Generator(std::move(generator)) + { + } + + std::optional Next() const + { + return m_Generator(); + } + +private: + GenFunc m_Generator; // The generator function that produces Values. +}; + +} diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 56893308a..531ab45b2 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -2,22 +2,324 @@ #include "base/json.hpp" #include "base/debug.hpp" -#include "base/namespace.hpp" #include "base/dictionary.hpp" -#include "base/array.hpp" +#include "base/namespace.hpp" #include "base/objectlock.hpp" -#include "base/convert.hpp" #include "base/utility.hpp" -#include -#include -#include -#include +#include #include #include #include using namespace icinga; +JsonEncoder::JsonEncoder(std::string& output, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(output), prettify} +{ +} + +JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(stream), prettify} +{ +} + +JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify) + : m_Pretty(prettify), m_Writer(std::move(w)), m_Flusher{m_Writer} +{ +} + +/** + * Encodes a single value into JSON and writes it to the underlying output stream. + * + * This method is the main entry point for encoding JSON data. It takes a value of any type that can + * be represented by our @c Value class recursively and encodes it into JSON in an efficient manner. + * If prettifying is enabled, the JSON output will be formatted with indentation and newlines for better + * readability, and the final JSON will also be terminated by a newline character. + * + * @note If the used output adapter performs asynchronous I/O operations (it's derived from @c AsyncJsonWriter), + * please provide a @c boost::asio::yield_context object to allow the encoder to flush the output stream in a + * safe manner. The encoder will try to regularly give the output stream a chance to flush its data when it is + * safe to do so, but for this to work, there must be a valid yield context provided. Otherwise, the encoder + * will not attempt to flush the output stream at all, which may lead to huge memory consumption when encoding + * large JSON objects or arrays. + * + * @param value The value to be JSON serialized. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock on the parent containers. + */ +void JsonEncoder::Encode(const Value& value, boost::asio::yield_context* yc) +{ + switch (value.GetType()) { + case ValueEmpty: + Write("null"); + break; + case ValueBoolean: + Write(value.ToBool() ? "true" : "false"); + break; + case ValueString: + EncodeNlohmannJson(value.Get()); + break; + case ValueNumber: + EncodeNumber(value.Get()); + break; + case ValueObject: { + const auto& obj = value.Get(); + const auto& type = obj->GetReflectionType(); + if (type == Namespace::TypeInstance) { + static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; + EncodeObject(static_pointer_cast(obj), extractor, yc); + } else if (type == Dictionary::TypeInstance) { + static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; + EncodeObject(static_pointer_cast(obj), extractor, yc); + } else if (type == Array::TypeInstance) { + EncodeArray(static_pointer_cast(obj), yc); + } else if (auto gen(dynamic_pointer_cast(obj)); gen) { + EncodeValueGenerator(gen, yc); + } else { + // Some other non-serializable object type! + EncodeNlohmannJson(obj->ToString()); + } + break; + } + default: + VERIFY(!"Invalid variant type."); + } + + // If we are at the top level of the JSON object and prettifying is enabled, we need to end + // the JSON with a newline character to ensure that the output is properly formatted. + if (m_Indent == 0 && m_Pretty) { + Write("\n"); + } +} + +/** + * Encodes an Array object into JSON and writes it to the output stream. + * + * @param array The Array object to be serialized into JSON. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock. + */ +void JsonEncoder::EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc) +{ + BeginContainer('['); + auto olock = array->LockIfRequired(); + if (olock) { + yc = nullptr; // We've acquired an object lock, never allow asynchronous operations. + } + + bool isEmpty = true; + for (const auto& item : array) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(item, yc); + m_Flusher.FlushIfSafe(yc); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes a ValueGenerator object into JSON and writes it to the output stream. + * + * This will iterate through the generator, encoding each value it produces until it is exhausted. + * + * @param generator The ValueGenerator object to be serialized into JSON. + * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder + * to flush the output stream safely when it has not acquired any object lock on the parent containers. + */ +void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc) +{ + BeginContainer('['); + bool isEmpty = true; + while (auto result = generator->Next()) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + Encode(*result, yc); + m_Flusher.FlushIfSafe(yc); + } + EndContainer(']', isEmpty); +} + +/** + * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer. + * + * @tparam Iterable Type of the container (Namespace or Dictionary). + * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator. + * + * @param container The container to JSON serialize. + * @param extractor The value extractor function used to extract values from the container's iterator. + * @param yc The optional yield context for asynchronous operations. It will only be set when the encoder + * has not acquired any object lock on the parent containers, allowing safe asynchronous operations. + */ +template +void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc) +{ + static_assert(std::is_same_v || std::is_same_v, + "Container must be a Namespace or Dictionary"); + + BeginContainer('{'); + auto olock = container->LockIfRequired(); + if (olock) { + yc = nullptr; // We've acquired an object lock, never allow asynchronous operations. + } + + bool isEmpty = true; + for (const auto& [key, val] : container) { + WriteSeparatorAndIndentStrIfNeeded(!isEmpty); + isEmpty = false; + + EncodeNlohmannJson(key); + Write(m_Pretty ? ": " : ":"); + + Encode(extractor(val), yc); + m_Flusher.FlushIfSafe(yc); + } + EndContainer('}', isEmpty); +} + +/** + * Dumps a nlohmann::json object to the output stream using the serializer. + * + * This function uses the @c nlohmann::detail::serializer to dump the provided @c nlohmann::json + * object to the output stream managed by the @c JsonEncoder. Strings will be properly escaped, and + * if any invalid UTF-8 sequences are encountered, it will replace them with the Unicode replacement + * character (U+FFFD). + * + * @param json The nlohmann::json object to encode. + */ +void JsonEncoder::EncodeNlohmannJson(const nlohmann::json& json) const +{ + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::replace); + s.dump(json, m_Pretty, true, 0, 0); +} + +/** + * Encodes a double value into JSON format and writes it to the output stream. + * + * This function checks if the double value can be safely cast to an integer or unsigned integer type + * without loss of precision. If it can, it will serialize it as such; otherwise, it will serialize + * it as a double. This is particularly useful for ensuring that values like 0.0 are serialized as 0, + * which can be important for compatibility with clients like Icinga DB that expect integers in such cases. + * + * @param value The double value to encode as JSON. + */ +void JsonEncoder::EncodeNumber(double value) const +{ + try { + if (value < 0) { + if (auto ll(boost::numeric_cast(value)); ll == value) { + EncodeNlohmannJson(ll); + return; + } + } else if (auto ull(boost::numeric_cast(value)); ull == value) { + EncodeNlohmannJson(ull); + return; + } + // If we reach this point, the value cannot be safely cast to a signed or unsigned integer + // type because it would otherwise lose its precision. If the value was just too large to fit + // into the above types, then boost will throw an exception and end up in the below catch block. + // So, in either case, serialize the number as-is without any casting. + } catch (const boost::bad_numeric_cast&) {} + + EncodeNlohmannJson(value); +} + +/** + * Writes a string to the underlying output stream. + * + * This function writes the provided string view directly to the output stream without any additional formatting. + * + * @param sv The string view to write to the output stream. + */ +void JsonEncoder::Write(const std::string_view& sv) const +{ + m_Writer->write_characters(sv.data(), sv.size()); +} + +/** + * Begins a JSON container (object or array) by writing the opening character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param openChar The character that opens the container (either '{' for objects or '[' for arrays). + */ +void JsonEncoder::BeginContainer(char openChar) +{ + if (m_Pretty) { + m_Indent += m_IndentSize; + if (m_IndentStr.size() < m_Indent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + } + m_Writer->write_character(openChar); +} + +/** + * Ends a JSON container (object or array) by writing the closing character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param closeChar The character that closes the container (either '}' for objects or ']' for arrays). + * @param isContainerEmpty Whether the container is empty, used to determine if a newline should be written. + */ +void JsonEncoder::EndContainer(char closeChar, bool isContainerEmpty) +{ + if (m_Pretty) { + ASSERT(m_Indent >= m_IndentSize); // Ensure we don't underflow the indent size. + m_Indent -= m_IndentSize; + if (!isContainerEmpty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } + } + m_Writer->write_character(closeChar); +} + +/** + * Writes a separator (comma) and an indentation string if pretty-printing is enabled. + * + * This function is used to separate items in a JSON array or object and to maintain the correct indentation level. + * + * @param emitComma Whether to emit a comma. This is typically true for all but the first item in a container. + */ +void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const +{ + if (emitComma) { + Write(","); + } + if (m_Pretty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } +} + +/** + * Wraps any writer of type @c nlohmann::detail::output_adapter_t into a Flusher + * + * @param w The writer to wrap. + */ +JsonEncoder::Flusher::Flusher(const nlohmann::detail::output_adapter_t& w) + : m_AsyncWriter(dynamic_cast(w.get())) +{ +} + +/** + * Flushes the underlying writer if it supports that operation and is safe to do so. + * + * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired + * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations + * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed. + * + * When the @c yc parameter is provided, it indicates that it's safe to perform asynchronous operations, + * and the function will attempt to flush if the writer is an instance of @c AsyncJsonWriter. Otherwise, + * this function does nothing. + * + * @param yc The yield context to use for asynchronous operations. + */ +void JsonEncoder::Flusher::FlushIfSafe(boost::asio::yield_context* yc) const +{ + if (yc && m_AsyncWriter) { + m_AsyncWriter->MayFlush(*yc); + } +} + class JsonSax : public nlohmann::json_sax { public: @@ -45,165 +347,25 @@ private: void FillCurrentTarget(Value value); }; -const char l_Null[] = "null"; -const char l_False[] = "false"; -const char l_True[] = "true"; -const char l_Indent[] = " "; - -// https://github.com/nlohmann/json/issues/1512 -template -class JsonEncoder +String icinga::JsonEncode(const Value& value, bool prettify) { -public: - void Null(); - void Boolean(bool value); - void NumberFloat(double value); - void Strng(String value); - void StartObject(); - void Key(String value); - void EndObject(); - void StartArray(); - void EndArray(); - - String GetResult(); - -private: - std::vector m_Result; - String m_CurrentKey; - std::stack> m_CurrentSubtree; - - void AppendChar(char c); - - template - void AppendChars(Iterator begin, Iterator end); - - void AppendJson(nlohmann::json json); - - void BeforeItem(); - - void FinishContainer(char terminator); -}; - -template -void Encode(JsonEncoder& stateMachine, const Value& value); - -template -inline -void EncodeNamespace(JsonEncoder& stateMachine, const Namespace::Ptr& ns) -{ - stateMachine.StartObject(); - - ObjectLock olock(ns); - for (const Namespace::Pair& kv : ns) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second.Val); - } - - stateMachine.EndObject(); + std::string output; + JsonEncoder encoder(output, prettify); + encoder.Encode(value); + return String(std::move(output)); } -template -inline -void EncodeDictionary(JsonEncoder& stateMachine, const Dictionary::Ptr& dict) +/** + * Serializes an Icinga Value into a JSON object and writes it to the given output stream. + * + * @param value The value to be JSON serialized. + * @param os The output stream to write the JSON data to. + * @param prettify Whether to pretty print the serialized JSON. + */ +void icinga::JsonEncode(const Value& value, std::ostream& os, bool prettify) { - stateMachine.StartObject(); - - ObjectLock olock(dict); - for (const Dictionary::Pair& kv : dict) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeArray(JsonEncoder& stateMachine, const Array::Ptr& arr) -{ - stateMachine.StartArray(); - - ObjectLock olock(arr); - for (const Value& value : arr) { - Encode(stateMachine, value); - } - - stateMachine.EndArray(); -} - -template -void Encode(JsonEncoder& stateMachine, const Value& value) -{ - switch (value.GetType()) { - case ValueNumber: - stateMachine.NumberFloat(value.Get()); - break; - - case ValueBoolean: - stateMachine.Boolean(value.ToBool()); - break; - - case ValueString: - stateMachine.Strng(Utility::ValidateUTF8(value.Get())); - break; - - case ValueObject: - { - const Object::Ptr& obj = value.Get(); - - { - Namespace::Ptr ns = dynamic_pointer_cast(obj); - if (ns) { - EncodeNamespace(stateMachine, ns); - break; - } - } - - { - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - if (dict) { - EncodeDictionary(stateMachine, dict); - break; - } - } - - { - Array::Ptr arr = dynamic_pointer_cast(obj); - if (arr) { - EncodeArray(stateMachine, arr); - break; - } - } - - // obj is most likely a function => "Object of type 'Function'" - Encode(stateMachine, obj->ToString()); - break; - } - - case ValueEmpty: - stateMachine.Null(); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - -String icinga::JsonEncode(const Value& value, bool pretty_print) -{ - if (pretty_print) { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult() + "\n"; - } else { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult(); - } + JsonEncoder encoder(os, prettify); + encoder.Encode(value); } Value icinga::JsonDecode(const String& data) @@ -349,177 +511,3 @@ void JsonSax::FillCurrentTarget(Value value) } } } - -template -inline -void JsonEncoder::Null() -{ - BeforeItem(); - AppendChars((const char*)l_Null, (const char*)l_Null + 4); -} - -template -inline -void JsonEncoder::Boolean(bool value) -{ - BeforeItem(); - - if (value) { - AppendChars((const char*)l_True, (const char*)l_True + 4); - } else { - AppendChars((const char*)l_False, (const char*)l_False + 5); - } -} - -template -inline -void JsonEncoder::NumberFloat(double value) -{ - BeforeItem(); - - // Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int. - if (value < 0) { - long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } else { - unsigned long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } -} - -template -inline -void JsonEncoder::Strng(String value) -{ - BeforeItem(); - AppendJson(std::move(value)); -} - -template -inline -void JsonEncoder::StartObject() -{ - BeforeItem(); - AppendChar('{'); - - m_CurrentSubtree.push(2); -} - -template -inline -void JsonEncoder::Key(String value) -{ - m_CurrentKey = std::move(value); -} - -template -inline -void JsonEncoder::EndObject() -{ - FinishContainer('}'); -} - -template -inline -void JsonEncoder::StartArray() -{ - BeforeItem(); - AppendChar('['); - - m_CurrentSubtree.push(0); -} - -template -inline -void JsonEncoder::EndArray() -{ - FinishContainer(']'); -} - -template -inline -String JsonEncoder::GetResult() -{ - return String(m_Result.begin(), m_Result.end()); -} - -template -inline -void JsonEncoder::AppendChar(char c) -{ - m_Result.emplace_back(c); -} - -template -template -inline -void JsonEncoder::AppendChars(Iterator begin, Iterator end) -{ - m_Result.insert(m_Result.end(), begin, end); -} - -template -inline -void JsonEncoder::AppendJson(nlohmann::json json) -{ - nlohmann::detail::serializer(nlohmann::detail::output_adapter(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0); -} - -template -inline -void JsonEncoder::BeforeItem() -{ - if (!m_CurrentSubtree.empty()) { - auto& node (m_CurrentSubtree.top()); - - if (node[0]) { - AppendChar(','); - } else { - node[0] = true; - } - - if (prettyPrint) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size()); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - if (node[1]) { - AppendJson(std::move(m_CurrentKey)); - AppendChar(':'); - - if (prettyPrint) { - AppendChar(' '); - } - } - } -} - -template -inline -void JsonEncoder::FinishContainer(char terminator) -{ - if (prettyPrint && m_CurrentSubtree.top()[0]) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size() - 1u); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - AppendChar(terminator); - - m_CurrentSubtree.pop(); -} diff --git a/lib/base/json.hpp b/lib/base/json.hpp index df0ea18a0..a5ed46280 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,14 +4,121 @@ #define JSON_H #include "base/i2-base.hpp" +#include "base/array.hpp" +#include "base/generator.hpp" +#include +#include namespace icinga { +/** + * AsyncJsonWriter allows writing JSON data to any output stream asynchronously. + * + * All users of this class must ensure that the underlying output stream will not perform any asynchronous I/O + * operations when the @c write_character() or @c write_characters() methods are called. They shall only perform + * such ops when the @c JsonEncoder allows them to do so by calling the @c MayFlush() method. + * + * @ingroup base + */ +class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol +{ +public: + /** + * It instructs the underlying output stream to write any buffered data to wherever it is supposed to go. + * + * The @c JsonEncoder allows the stream to even perform asynchronous operations in a safe manner by calling + * this method with a dedicated @c boost::asio::yield_context object. The stream must not perform any async + * I/O operations triggered by methods other than this one. Any attempt to do so will result in undefined behavior. + * + * However, this doesn't necessarily enforce the stream to really flush its data immediately, but it's up + * to the implementation to do whatever it needs to. The encoder just gives it a chance to do so by calling + * this method. + * + * @param yield The yield context to use for asynchronous operations. + */ + virtual void MayFlush(boost::asio::yield_context& yield) = 0; +}; + class String; class Value; -String JsonEncode(const Value& value, bool pretty_print = false); +/** + * JSON encoder. + * + * This class can be used to encode Icinga Value types into JSON format and write them to an output stream. + * The supported stream types include any @c std::ostream like objects and our own @c AsyncJsonWriter, which + * allows writing JSON data to an Asio stream asynchronously. The nlohmann/json library already provides + * full support for the former stream type, while the latter is fully implemented by our own and satisfies the + * @c nlohmann::detail::output_adapter_protocol<> interface as well. + * + * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing + * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the + * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind + * of work to serializer<>. + * + * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object + * is indented with 4 spaces and grows linearly with the depth of the object tree. + * + * @ingroup base + */ +class JsonEncoder +{ +public: + explicit JsonEncoder(std::string& output, bool prettify = false); + explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false); + explicit JsonEncoder(nlohmann::detail::output_adapter_t w, bool prettify = false); + + void Encode(const Value& value, boost::asio::yield_context* yc = nullptr); + +private: + void EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc); + void EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc); + + template + void EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc); + + void EncodeNlohmannJson(const nlohmann::json& json) const; + void EncodeNumber(double value) const; + + void Write(const std::string_view& sv) const; + void BeginContainer(char openChar); + void EndContainer(char closeChar, bool isContainerEmpty = false); + void WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const; + + // The number of spaces to use for indentation in prettified JSON. + static constexpr uint8_t m_IndentSize = 4; + + bool m_Pretty; // Whether to pretty-print the JSON output. + unsigned m_Indent{0}; // The current indentation level for pretty-printing. + /** + * Pre-allocate for 8 levels of indentation for pretty-printing. + * + * This is used to avoid reallocating the string on every indent level change. + * The size of this string is dynamically adjusted if the indentation level exceeds its initial size at some point. + */ + std::string m_IndentStr{8*m_IndentSize, ' '}; + + // The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter. + nlohmann::detail::output_adapter_t m_Writer; + + /** + * This class wraps any @c nlohmann::detail::output_adapter_t writer and provides a method to flush it as + * required. Only @c AsyncJsonWriter supports the flush operation, however, this class is also safe to use with + * other writer types and the flush method does nothing for them. + */ + class Flusher { + public: + explicit Flusher(const nlohmann::detail::output_adapter_t& w); + void FlushIfSafe(boost::asio::yield_context* yc) const; + + private: + AsyncJsonWriter* m_AsyncWriter; + } m_Flusher; +}; + +String JsonEncode(const Value& value, bool prettify = false); +void JsonEncode(const Value& value, std::ostream& os, bool prettify = false); Value JsonDecode(const String& data); } diff --git a/lib/base/namespace.cpp b/lib/base/namespace.cpp index 1f53efc92..383b2ecbd 100644 --- a/lib/base/namespace.cpp +++ b/lib/base/namespace.cpp @@ -1,7 +1,6 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/namespace.hpp" -#include "base/objectlock.hpp" #include "base/debug.hpp" #include "base/primitivetype.hpp" #include "base/debuginfo.hpp" @@ -119,7 +118,26 @@ void Namespace::Remove(const String& field) void Namespace::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true, std::memory_order_release); +} + +bool Namespace::Frozen() const +{ + return m_Frozen.load(std::memory_order_acquire); +} + +/** + * Returns an already locked ObjectLock if the namespace is frozen. + * Otherwise, returns an unlocked object lock. + * + * @returns An object lock. + */ +ObjectLock Namespace::LockIfRequired() +{ + if (Frozen()) { + return ObjectLock(this, std::defer_lock); + } + return ObjectLock(this); } std::shared_lock Namespace::ReadLockUnlessFrozen() const @@ -160,14 +178,14 @@ bool Namespace::GetOwnField(const String& field, Value *result) const Namespace::Iterator Namespace::Begin() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.begin(); } Namespace::Iterator Namespace::End() { - ASSERT(OwnsLock()); + ASSERT(Frozen() || OwnsLock()); return m_Data.end(); } diff --git a/lib/base/namespace.hpp b/lib/base/namespace.hpp index 1a028e2c5..194671099 100644 --- a/lib/base/namespace.hpp +++ b/lib/base/namespace.hpp @@ -5,6 +5,7 @@ #include "base/i2-base.hpp" #include "base/object.hpp" +#include "base/objectlock.hpp" #include "base/shared-object.hpp" #include "base/value.hpp" #include "base/debuginfo.hpp" @@ -73,6 +74,8 @@ public: bool Contains(const String& field) const; void Remove(const String& field); void Freeze(); + bool Frozen() const; + ObjectLock LockIfRequired(); Iterator Begin(); Iterator End(); diff --git a/lib/base/objectlock.cpp b/lib/base/objectlock.cpp index fc0c7c631..9ed9d1f21 100644 --- a/lib/base/objectlock.cpp +++ b/lib/base/objectlock.cpp @@ -18,6 +18,18 @@ ObjectLock::ObjectLock(const Object::Ptr& object) { } +/** + * Constructs a lock for the given object without locking it immediately. + * + * The user must call Lock() explicitly when needed. + * + * @param object The object to lock. + */ +ObjectLock::ObjectLock(const Object::Ptr& object, std::defer_lock_t) + : m_Object(object.get()), m_Locked(false) +{ +} + ObjectLock::ObjectLock(const Object *object) : m_Object(object), m_Locked(false) { @@ -53,3 +65,15 @@ void ObjectLock::Unlock() m_Locked = false; } } + +/** + * Returns true if the object is locked, false otherwise. + * + * This operator allows using ObjectLock in boolean contexts. + * + * @returns true if the object is locked, false otherwise. + */ +ObjectLock::operator bool() const +{ + return m_Locked; +} diff --git a/lib/base/objectlock.hpp b/lib/base/objectlock.hpp index 8e98641db..328887425 100644 --- a/lib/base/objectlock.hpp +++ b/lib/base/objectlock.hpp @@ -15,6 +15,7 @@ struct ObjectLock { public: ObjectLock(const Object::Ptr& object); + ObjectLock(const Object::Ptr& object, std::defer_lock_t); ObjectLock(const Object *object); ObjectLock(const ObjectLock&) = delete; @@ -25,6 +26,8 @@ public: void Lock(); void Unlock(); + operator bool() const; + private: const Object *m_Object{nullptr}; bool m_Locked{false}; diff --git a/test/base-json.cpp b/test/base-json.cpp index 02bbebb6d..4ae17bf09 100644 --- a/test/base-json.cpp +++ b/test/base-json.cpp @@ -4,10 +4,13 @@ #include "base/function.hpp" #include "base/namespace.hpp" #include "base/array.hpp" +#include "base/generator.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" #include #include +#include +#include using namespace icinga; @@ -15,26 +18,51 @@ BOOST_AUTO_TEST_SUITE(base_json) BOOST_AUTO_TEST_CASE(encode) { + auto generate = []() -> std::optional { + static int count = 0; + if (++count == 4) { + count = 0; + return std::nullopt; + } + return Value(count); + }; + Dictionary::Ptr input (new Dictionary({ { "array", new Array({ new Namespace() }) }, { "false", false }, - { "float", -1.25 }, + // Use double max value to test JSON encoding of large numbers and trigger boost numeric_cast exceptions + { "max_double", std::numeric_limits::max() }, + // Test the largest uint64_t value that has an exact double representation (2^64-2048). + { "max_int_in_double", std::nextafter(std::pow(2, 64), 0.0) }, + { "float", -1.25f }, + { "float_without_fraction", 23.0f }, { "fx", new Function("", []() {}) }, { "int", -42 }, { "null", Value() }, { "string", "LF\nTAB\tAUml\xC3\xA4Ill\xC3" }, { "true", true }, - { "uint", 23u } + { "uint", 23u }, + { "generator", new ValueGenerator(generate) }, + { "empty_generator", new ValueGenerator([]() -> std::optional { return std::nullopt; }) }, })); String output (R"EOF({ "array": [ {} ], + "empty_generator": [], "false": false, "float": -1.25, + "float_without_fraction": 23, "fx": "Object of type 'Function'", + "generator": [ + 1, + 2, + 3 + ], "int": -42, + "max_double": 1.7976931348623157e+308, + "max_int_in_double": 18446744073709549568, "null": null, "string": "LF\nTAB\tAUml\u00e4Ill\ufffd", "true": true, @@ -42,7 +70,12 @@ BOOST_AUTO_TEST_CASE(encode) } )EOF"); - BOOST_CHECK(JsonEncode(input, true) == output); + auto got(JsonEncode(input, true)); + BOOST_CHECK_EQUAL(output, got); + + std::ostringstream oss; + JsonEncode(input, oss, true); + BOOST_CHECK_EQUAL(output, oss.str()); boost::algorithm::replace_all(output, " ", ""); boost::algorithm::replace_all(output, "Objectoftype'Function'", "Object of type 'Function'");