Merge pull request #10414 from Icinga/support-json-serialization-into-ostream

Support JSON serialization into any stream like objects

lib/base/CMakeLists.txt

@@ -37,6 +37,7 @@ set(base_SOURCES
   fifo.cpp fifo.hpp
   filelogger.cpp filelogger.hpp filelogger-ti.hpp
   function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp
+  generator.hpp
   initialize.cpp initialize.hpp
   intrusive-ptr.hpp
   io-engine.cpp io-engine.hpp

lib/base/array.cpp

@@ -96,7 +96,7 @@ void Array::Add(Value value)
  */
 Array::Iterator Array::Begin()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.begin();
 }
@@ -110,7 +110,7 @@ Array::Iterator Array::Begin()
  */
 Array::Iterator Array::End()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.end();
 }
@@ -327,7 +327,26 @@ Array::Ptr Array::Unique() const
 void Array::Freeze()
 {
 	ObjectLock olock(this);
-	m_Frozen = true;
+	m_Frozen.store(true, std::memory_order_release);
+}
+
+bool Array::Frozen() const
+{
+	return m_Frozen.load(std::memory_order_acquire);
+}
+
+/**
+ * Returns an already locked ObjectLock if the array is frozen.
+ * Otherwise, returns an unlocked object lock.
+ *
+ * @returns An object lock.
+ */
+ObjectLock Array::LockIfRequired()
+{
+	if (Frozen()) {
+		return ObjectLock(this, std::defer_lock);
+	}
+	return ObjectLock(this);
 }
 
 Value Array::GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const

lib/base/array.hpp

@@ -4,6 +4,7 @@
 #define ARRAY_H
 
 #include "base/i2-base.hpp"
+#include "base/atomic.hpp"
 #include "base/objectlock.hpp"
 #include "base/value.hpp"
 #include <boost/range/iterator.hpp>
@@ -98,13 +99,15 @@ public:
 
 	Array::Ptr Unique() const;
 	void Freeze();
+	bool Frozen() const;
+	ObjectLock LockIfRequired();
 
 	Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override;
 	void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override;
 
 private:
 	std::vector<Value> m_Data; /**< The data for the array. */
-	bool m_Frozen{false};
+	Atomic<bool> m_Frozen{false};
 };
 
 Array::Iterator begin(const Array::Ptr& x);

lib/base/dictionary.cpp

@@ -1,7 +1,6 @@
 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
 
 #include "base/dictionary.hpp"
-#include "base/objectlock.hpp"
 #include "base/debug.hpp"
 #include "base/primitivetype.hpp"
 #include "base/configwriter.hpp"
@@ -132,7 +131,7 @@ bool Dictionary::Contains(const String& key) const
  */
 Dictionary::Iterator Dictionary::Begin()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.begin();
 }
@@ -146,7 +145,7 @@ Dictionary::Iterator Dictionary::Begin()
  */
 Dictionary::Iterator Dictionary::End()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.end();
 }
@@ -276,7 +275,26 @@ String Dictionary::ToString() const
 void Dictionary::Freeze()
 {
 	ObjectLock olock(this);
-	m_Frozen = true;
+	m_Frozen.store(true, std::memory_order_release);
+}
+
+bool Dictionary::Frozen() const
+{
+	return m_Frozen.load(std::memory_order_acquire);
+}
+
+/**
+ * Returns an already locked ObjectLock if the dictionary is frozen.
+ * Otherwise, returns an unlocked object lock.
+ *
+ * @returns An object lock.
+ */
+ObjectLock Dictionary::LockIfRequired()
+{
+	if (Frozen()) {
+		return ObjectLock(this, std::defer_lock);
+	}
+	return ObjectLock(this);
 }
 
 Value Dictionary::GetFieldByName(const String& field, bool, const DebugInfo& debugInfo) const

lib/base/dictionary.hpp

@@ -4,7 +4,9 @@
 #define DICTIONARY_H
 
 #include "base/i2-base.hpp"
+#include "base/atomic.hpp"
 #include "base/object.hpp"
+#include "base/objectlock.hpp"
 #include "base/value.hpp"
 #include <boost/range/iterator.hpp>
 #include <map>
@@ -69,6 +71,8 @@ public:
 	String ToString() const override;
 
 	void Freeze();
+	bool Frozen() const;
+	ObjectLock LockIfRequired();
 
 	Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override;
 	void SetFieldByName(const String& field, const Value& value, const DebugInfo& debugInfo) override;
@@ -78,7 +82,7 @@ public:
 private:
 	std::map<String, Value> m_Data; /**< The data for the dictionary. */
 	mutable std::shared_timed_mutex m_DataMutex;
-	bool m_Frozen{false};
+	Atomic<bool> m_Frozen{false};
 };
 
 Dictionary::Iterator begin(const Dictionary::Ptr& x);

lib/base/generator.hpp

@@ -0,0 +1,48 @@
+/* Icinga 2 | (c) 2025 Icinga GmbH | GPLv2+ */
+
+#pragma once
+
+#include "base/i2-base.hpp"
+#include "base/value.hpp"
+#include <optional>
+
+namespace icinga
+{
+
+/**
+ * ValueGenerator is a class that defines a generator function type for producing Values on demand.
+ *
+ * This class is used to create generator functions that can yield any values that can be represented by the
+ * Icinga Value type. The generator function is exhausted when it returns `std::nullopt`, indicating that there
+ * are no more values to produce. Subsequent calls to `Next()` will always return `std::nullopt` after exhaustion.
+ *
+ * @ingroup base
+ */
+class ValueGenerator final : public Object
+{
+public:
+	DECLARE_PTR_TYPEDEFS(ValueGenerator);
+
+	/**
+	 * Generates a Value using the provided generator function.
+	 *
+	 * The generator function should return an `std::optional<Value>` which contains the produced Value or
+	 * `std::nullopt` when there are no more values to produce. After the generator function returns `std::nullopt`,
+	 * the generator is considered exhausted, and further calls to `Next()` will always return `std::nullopt`.
+	 */
+	using GenFunc = std::function<std::optional<Value>()>;
+
+	explicit ValueGenerator(GenFunc generator): m_Generator(std::move(generator))
+	{
+	}
+
+	std::optional<Value> Next() const
+	{
+		return m_Generator();
+	}
+
+private:
+	GenFunc m_Generator; // The generator function that produces Values.
+};
+
+}

lib/base/json.cpp

@@ -2,22 +2,324 @@
 
 #include "base/json.hpp"
 #include "base/debug.hpp"
-#include "base/namespace.hpp"
 #include "base/dictionary.hpp"
-#include "base/array.hpp"
+#include "base/namespace.hpp"
 #include "base/objectlock.hpp"
-#include "base/convert.hpp"
 #include "base/utility.hpp"
-#include <bitset>
+#include <boost/numeric/conversion/cast.hpp>
-#include <boost/exception_ptr.hpp>
-#include <cstdint>
-#include <json.hpp>
 #include <stack>
 #include <utility>
 #include <vector>
 
 using namespace icinga;
 
+JsonEncoder::JsonEncoder(std::string& output, bool prettify)
+	: JsonEncoder{nlohmann::detail::output_adapter<char>(output), prettify}
+{
+}
+
+JsonEncoder::JsonEncoder(std::basic_ostream<char>& stream, bool prettify)
+	: JsonEncoder{nlohmann::detail::output_adapter<char>(stream), prettify}
+{
+}
+
+JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t<char> w, bool prettify)
+	: m_Pretty(prettify), m_Writer(std::move(w)), m_Flusher{m_Writer}
+{
+}
+
+/**
+ * Encodes a single value into JSON and writes it to the underlying output stream.
+ *
+ * This method is the main entry point for encoding JSON data. It takes a value of any type that can
+ * be represented by our @c Value class recursively and encodes it into JSON in an efficient manner.
+ * If prettifying is enabled, the JSON output will be formatted with indentation and newlines for better
+ * readability, and the final JSON will also be terminated by a newline character.
+ *
+ * @note If the used output adapter performs asynchronous I/O operations (it's derived from @c AsyncJsonWriter),
+ * please provide a @c boost::asio::yield_context object to allow the encoder to flush the output stream in a
+ * safe manner. The encoder will try to regularly give the output stream a chance to flush its data when it is
+ * safe to do so, but for this to work, there must be a valid yield context provided. Otherwise, the encoder
+ * will not attempt to flush the output stream at all, which may lead to huge memory consumption when encoding
+ * large JSON objects or arrays.
+ *
+ * @param value The value to be JSON serialized.
+ * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder
+ * to flush the output stream safely when it has not acquired any object lock on the parent containers.
+ */
+void JsonEncoder::Encode(const Value& value, boost::asio::yield_context* yc)
+{
+	switch (value.GetType()) {
+		case ValueEmpty:
+			Write("null");
+			break;
+		case ValueBoolean:
+			Write(value.ToBool() ? "true" : "false");
+			break;
+		case ValueString:
+			EncodeNlohmannJson(value.Get<String>());
+			break;
+		case ValueNumber:
+			EncodeNumber(value.Get<double>());
+			break;
+		case ValueObject: {
+			const auto& obj = value.Get<Object::Ptr>();
+			const auto& type = obj->GetReflectionType();
+			if (type == Namespace::TypeInstance) {
+				static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; };
+				EncodeObject(static_pointer_cast<Namespace>(obj), extractor, yc);
+			} else if (type == Dictionary::TypeInstance) {
+				static constexpr auto extractor = [](const Value& v) -> const Value& { return v; };
+				EncodeObject(static_pointer_cast<Dictionary>(obj), extractor, yc);
+			} else if (type == Array::TypeInstance) {
+				EncodeArray(static_pointer_cast<Array>(obj), yc);
+			} else if (auto gen(dynamic_pointer_cast<ValueGenerator>(obj)); gen) {
+				EncodeValueGenerator(gen, yc);
+			} else {
+				// Some other non-serializable object type!
+				EncodeNlohmannJson(obj->ToString());
+			}
+			break;
+		}
+		default:
+			VERIFY(!"Invalid variant type.");
+	}
+
+	// If we are at the top level of the JSON object and prettifying is enabled, we need to end
+	// the JSON with a newline character to ensure that the output is properly formatted.
+	if (m_Indent == 0 && m_Pretty) {
+		Write("\n");
+	}
+}
+
+/**
+ * Encodes an Array object into JSON and writes it to the output stream.
+ *
+ * @param array The Array object to be serialized into JSON.
+ * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder
+ * to flush the output stream safely when it has not acquired any object lock.
+ */
+void JsonEncoder::EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc)
+{
+	BeginContainer('[');
+	auto olock = array->LockIfRequired();
+	if (olock) {
+		yc = nullptr; // We've acquired an object lock, never allow asynchronous operations.
+	}
+
+	bool isEmpty = true;
+	for (const auto& item : array) {
+		WriteSeparatorAndIndentStrIfNeeded(!isEmpty);
+		isEmpty = false;
+		Encode(item, yc);
+		m_Flusher.FlushIfSafe(yc);
+	}
+	EndContainer(']', isEmpty);
+}
+
+/**
+ * Encodes a ValueGenerator object into JSON and writes it to the output stream.
+ *
+ * This will iterate through the generator, encoding each value it produces until it is exhausted.
+ *
+ * @param generator The ValueGenerator object to be serialized into JSON.
+ * @param yc The optional yield context for asynchronous operations. If provided, it allows the encoder
+ * to flush the output stream safely when it has not acquired any object lock on the parent containers.
+ */
+void JsonEncoder::EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc)
+{
+	BeginContainer('[');
+	bool isEmpty = true;
+	while (auto result = generator->Next()) {
+		WriteSeparatorAndIndentStrIfNeeded(!isEmpty);
+		isEmpty = false;
+		Encode(*result, yc);
+		m_Flusher.FlushIfSafe(yc);
+	}
+	EndContainer(']', isEmpty);
+}
+
+/**
+ * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer.
+ *
+ * @tparam Iterable Type of the container (Namespace or Dictionary).
+ * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator.
+ *
+ * @param container The container to JSON serialize.
+ * @param extractor The value extractor function used to extract values from the container's iterator.
+ * @param yc The optional yield context for asynchronous operations. It will only be set when the encoder
+ * has not acquired any object lock on the parent containers, allowing safe asynchronous operations.
+ */
+template<typename Iterable, typename ValExtractor>
+void JsonEncoder::EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc)
+{
+	static_assert(std::is_same_v<Iterable, Namespace::Ptr> || std::is_same_v<Iterable, Dictionary::Ptr>,
+		"Container must be a Namespace or Dictionary");
+
+	BeginContainer('{');
+	auto olock = container->LockIfRequired();
+	if (olock) {
+		yc = nullptr; // We've acquired an object lock, never allow asynchronous operations.
+	}
+
+	bool isEmpty = true;
+	for (const auto& [key, val] : container) {
+		WriteSeparatorAndIndentStrIfNeeded(!isEmpty);
+		isEmpty = false;
+
+		EncodeNlohmannJson(key);
+		Write(m_Pretty ? ": " : ":");
+
+		Encode(extractor(val), yc);
+		m_Flusher.FlushIfSafe(yc);
+	}
+	EndContainer('}', isEmpty);
+}
+
+/**
+ * Dumps a nlohmann::json object to the output stream using the serializer.
+ *
+ * This function uses the @c nlohmann::detail::serializer to dump the provided @c nlohmann::json
+ * object to the output stream managed by the @c JsonEncoder. Strings will be properly escaped, and
+ * if any invalid UTF-8 sequences are encountered, it will replace them with the Unicode replacement
+ * character (U+FFFD).
+ *
+ * @param json The nlohmann::json object to encode.
+ */
+void JsonEncoder::EncodeNlohmannJson(const nlohmann::json& json) const
+{
+	nlohmann::detail::serializer<nlohmann::json> s(m_Writer, ' ', nlohmann::json::error_handler_t::replace);
+	s.dump(json, m_Pretty, true, 0, 0);
+}
+
+/**
+ * Encodes a double value into JSON format and writes it to the output stream.
+ *
+ * This function checks if the double value can be safely cast to an integer or unsigned integer type
+ * without loss of precision. If it can, it will serialize it as such; otherwise, it will serialize
+ * it as a double. This is particularly useful for ensuring that values like 0.0 are serialized as 0,
+ * which can be important for compatibility with clients like Icinga DB that expect integers in such cases.
+ *
+ * @param value The double value to encode as JSON.
+ */
+void JsonEncoder::EncodeNumber(double value) const
+{
+	try {
+		if (value < 0) {
+			if (auto ll(boost::numeric_cast<nlohmann::json::number_integer_t>(value)); ll == value) {
+				EncodeNlohmannJson(ll);
+				return;
+			}
+		} else if (auto ull(boost::numeric_cast<nlohmann::json::number_unsigned_t>(value)); ull == value) {
+			EncodeNlohmannJson(ull);
+			return;
+		}
+		// If we reach this point, the value cannot be safely cast to a signed or unsigned integer
+		// type because it would otherwise lose its precision. If the value was just too large to fit
+		// into the above types, then boost will throw an exception and end up in the below catch block.
+		// So, in either case, serialize the number as-is without any casting.
+	} catch (const boost::bad_numeric_cast&) {}
+
+	EncodeNlohmannJson(value);
+}
+
+/**
+ * Writes a string to the underlying output stream.
+ *
+ * This function writes the provided string view directly to the output stream without any additional formatting.
+ *
+ * @param sv The string view to write to the output stream.
+ */
+void JsonEncoder::Write(const std::string_view& sv) const
+{
+	m_Writer->write_characters(sv.data(), sv.size());
+}
+
+/**
+ * Begins a JSON container (object or array) by writing the opening character and adjusting the
+ * indentation level if pretty-printing is enabled.
+ *
+ * @param openChar The character that opens the container (either '{' for objects or '[' for arrays).
+ */
+void JsonEncoder::BeginContainer(char openChar)
+{
+	if (m_Pretty) {
+		m_Indent += m_IndentSize;
+		if (m_IndentStr.size() < m_Indent) {
+			m_IndentStr.resize(m_IndentStr.size() * 2, ' ');
+		}
+	}
+	m_Writer->write_character(openChar);
+}
+
+/**
+ * Ends a JSON container (object or array) by writing the closing character and adjusting the
+ * indentation level if pretty-printing is enabled.
+ *
+ * @param closeChar The character that closes the container (either '}' for objects or ']' for arrays).
+ * @param isContainerEmpty Whether the container is empty, used to determine if a newline should be written.
+ */
+void JsonEncoder::EndContainer(char closeChar, bool isContainerEmpty)
+{
+	if (m_Pretty) {
+		ASSERT(m_Indent >= m_IndentSize); // Ensure we don't underflow the indent size.
+		m_Indent -= m_IndentSize;
+		if (!isContainerEmpty) {
+			Write("\n");
+			m_Writer->write_characters(m_IndentStr.c_str(), m_Indent);
+		}
+	}
+	m_Writer->write_character(closeChar);
+}
+
+/**
+ * Writes a separator (comma) and an indentation string if pretty-printing is enabled.
+ *
+ * This function is used to separate items in a JSON array or object and to maintain the correct indentation level.
+ *
+ * @param emitComma Whether to emit a comma. This is typically true for all but the first item in a container.
+ */
+void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const
+{
+	if (emitComma) {
+		Write(",");
+	}
+	if (m_Pretty) {
+		Write("\n");
+		m_Writer->write_characters(m_IndentStr.c_str(), m_Indent);
+	}
+}
+
+/**
+ * Wraps any writer of type @c nlohmann::detail::output_adapter_t<char> into a Flusher
+ *
+ * @param w The writer to wrap.
+ */
+JsonEncoder::Flusher::Flusher(const nlohmann::detail::output_adapter_t<char>& w)
+	: m_AsyncWriter(dynamic_cast<AsyncJsonWriter*>(w.get()))
+{
+}
+
+/**
+ * Flushes the underlying writer if it supports that operation and is safe to do so.
+ *
+ * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired
+ * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations
+ * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed.
+ *
+ * When the @c yc parameter is provided, it indicates that it's safe to perform asynchronous operations,
+ * and the function will attempt to flush if the writer is an instance of @c AsyncJsonWriter. Otherwise,
+ * this function does nothing.
+ *
+ * @param yc The yield context to use for asynchronous operations.
+ */
+void JsonEncoder::Flusher::FlushIfSafe(boost::asio::yield_context* yc) const
+{
+	if (yc && m_AsyncWriter) {
+		m_AsyncWriter->MayFlush(*yc);
+	}
+}
+
 class JsonSax : public nlohmann::json_sax<nlohmann::json>
 {
 public:
@@ -45,165 +347,25 @@ private:
 	void FillCurrentTarget(Value value);
 };
 
-const char l_Null[] = "null";
+String icinga::JsonEncode(const Value& value, bool prettify)
-const char l_False[] = "false";
-const char l_True[] = "true";
-const char l_Indent[] = "    ";
-
-// https://github.com/nlohmann/json/issues/1512
-template<bool prettyPrint>
-class JsonEncoder
 {
-public:
+	std::string output;
-	void Null();
+	JsonEncoder encoder(output, prettify);
-	void Boolean(bool value);
+	encoder.Encode(value);
-	void NumberFloat(double value);
+	return String(std::move(output));
-	void Strng(String value);
-	void StartObject();
-	void Key(String value);
-	void EndObject();
-	void StartArray();
-	void EndArray();
-
-	String GetResult();
-
-private:
-	std::vector<char> m_Result;
-	String m_CurrentKey;
-	std::stack<std::bitset<2>> m_CurrentSubtree;
-
-	void AppendChar(char c);
-
-	template<class Iterator>
-	void AppendChars(Iterator begin, Iterator end);
-
-	void AppendJson(nlohmann::json json);
-
-	void BeforeItem();
-
-	void FinishContainer(char terminator);
-};
-
-template<bool prettyPrint>
-void Encode(JsonEncoder<prettyPrint>& stateMachine, const Value& value);
-
-template<bool prettyPrint>
-inline
-void EncodeNamespace(JsonEncoder<prettyPrint>& stateMachine, const Namespace::Ptr& ns)
-{
-	stateMachine.StartObject();
-
-	ObjectLock olock(ns);
-	for (const Namespace::Pair& kv : ns) {
-		stateMachine.Key(Utility::ValidateUTF8(kv.first));
-		Encode(stateMachine, kv.second.Val);
-	}
-
-	stateMachine.EndObject();
 }
 
-template<bool prettyPrint>
+/**
-inline
+ * Serializes an Icinga Value into a JSON object and writes it to the given output stream.
-void EncodeDictionary(JsonEncoder<prettyPrint>& stateMachine, const Dictionary::Ptr& dict)
+ *
+ * @param value The value to be JSON serialized.
+ * @param os The output stream to write the JSON data to.
+ * @param prettify Whether to pretty print the serialized JSON.
+ */
+void icinga::JsonEncode(const Value& value, std::ostream& os, bool prettify)
 {
-	stateMachine.StartObject();
+	JsonEncoder encoder(os, prettify);
-
+	encoder.Encode(value);
-	ObjectLock olock(dict);
-	for (const Dictionary::Pair& kv : dict) {
-		stateMachine.Key(Utility::ValidateUTF8(kv.first));
-		Encode(stateMachine, kv.second);
-	}
-
-	stateMachine.EndObject();
-}
-
-template<bool prettyPrint>
-inline
-void EncodeArray(JsonEncoder<prettyPrint>& stateMachine, const Array::Ptr& arr)
-{
-	stateMachine.StartArray();
-
-	ObjectLock olock(arr);
-	for (const Value& value : arr) {
-		Encode(stateMachine, value);
-	}
-
-	stateMachine.EndArray();
-}
-
-template<bool prettyPrint>
-void Encode(JsonEncoder<prettyPrint>& stateMachine, const Value& value)
-{
-	switch (value.GetType()) {
-		case ValueNumber:
-			stateMachine.NumberFloat(value.Get<double>());
-			break;
-
-		case ValueBoolean:
-			stateMachine.Boolean(value.ToBool());
-			break;
-
-		case ValueString:
-			stateMachine.Strng(Utility::ValidateUTF8(value.Get<String>()));
-			break;
-
-		case ValueObject:
-			{
-				const Object::Ptr& obj = value.Get<Object::Ptr>();
-
-				{
-					Namespace::Ptr ns = dynamic_pointer_cast<Namespace>(obj);
-					if (ns) {
-						EncodeNamespace(stateMachine, ns);
-						break;
-					}
-				}
-
-				{
-					Dictionary::Ptr dict = dynamic_pointer_cast<Dictionary>(obj);
-					if (dict) {
-						EncodeDictionary(stateMachine, dict);
-						break;
-					}
-				}
-
-				{
-					Array::Ptr arr = dynamic_pointer_cast<Array>(obj);
-					if (arr) {
-						EncodeArray(stateMachine, arr);
-						break;
-					}
-				}
-
-				// obj is most likely a function => "Object of type 'Function'"
-				Encode(stateMachine, obj->ToString());
-				break;
-			}
-
-		case ValueEmpty:
-			stateMachine.Null();
-			break;
-
-		default:
-			VERIFY(!"Invalid variant type.");
-	}
-}
-
-String icinga::JsonEncode(const Value& value, bool pretty_print)
-{
-	if (pretty_print) {
-		JsonEncoder<true> stateMachine;
-
-		Encode(stateMachine, value);
-
-		return stateMachine.GetResult() + "\n";
-	} else {
-		JsonEncoder<false> stateMachine;
-
-		Encode(stateMachine, value);
-
-		return stateMachine.GetResult();
-	}
 }
 
 Value icinga::JsonDecode(const String& data)
@@ -349,177 +511,3 @@ void JsonSax::FillCurrentTarget(Value value)
 		}
 	}
 }
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::Null()
-{
-	BeforeItem();
-	AppendChars((const char*)l_Null, (const char*)l_Null + 4);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::Boolean(bool value)
-{
-	BeforeItem();
-
-	if (value) {
-		AppendChars((const char*)l_True, (const char*)l_True + 4);
-	} else {
-		AppendChars((const char*)l_False, (const char*)l_False + 5);
-	}
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::NumberFloat(double value)
-{
-	BeforeItem();
-
-	// Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int.
-	if (value < 0) {
-		long long i = value;
-
-		if (i == value) {
-			AppendJson(i);
-		} else {
-			AppendJson(value);
-		}
-	} else {
-		unsigned long long i = value;
-
-		if (i == value) {
-			AppendJson(i);
-		} else {
-			AppendJson(value);
-		}
-	}
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::Strng(String value)
-{
-	BeforeItem();
-	AppendJson(std::move(value));
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::StartObject()
-{
-	BeforeItem();
-	AppendChar('{');
-
-	m_CurrentSubtree.push(2);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::Key(String value)
-{
-	m_CurrentKey = std::move(value);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::EndObject()
-{
-	FinishContainer('}');
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::StartArray()
-{
-	BeforeItem();
-	AppendChar('[');
-
-	m_CurrentSubtree.push(0);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::EndArray()
-{
-	FinishContainer(']');
-}
-
-template<bool prettyPrint>
-inline
-String JsonEncoder<prettyPrint>::GetResult()
-{
-	return String(m_Result.begin(), m_Result.end());
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::AppendChar(char c)
-{
-	m_Result.emplace_back(c);
-}
-
-template<bool prettyPrint>
-template<class Iterator>
-inline
-void JsonEncoder<prettyPrint>::AppendChars(Iterator begin, Iterator end)
-{
-	m_Result.insert(m_Result.end(), begin, end);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::AppendJson(nlohmann::json json)
-{
-	nlohmann::detail::serializer<nlohmann::json>(nlohmann::detail::output_adapter<char>(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0);
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::BeforeItem()
-{
-	if (!m_CurrentSubtree.empty()) {
-		auto& node (m_CurrentSubtree.top());
-
-		if (node[0]) {
-			AppendChar(',');
-		} else {
-			node[0] = true;
-		}
-
-		if (prettyPrint) {
-			AppendChar('\n');
-
-			for (auto i (m_CurrentSubtree.size()); i; --i) {
-				AppendChars((const char*)l_Indent, (const char*)l_Indent + 4);
-			}
-		}
-
-		if (node[1]) {
-			AppendJson(std::move(m_CurrentKey));
-			AppendChar(':');
-
-			if (prettyPrint) {
-				AppendChar(' ');
-			}
-		}
-	}
-}
-
-template<bool prettyPrint>
-inline
-void JsonEncoder<prettyPrint>::FinishContainer(char terminator)
-{
-	if (prettyPrint && m_CurrentSubtree.top()[0]) {
-		AppendChar('\n');
-
-		for (auto i (m_CurrentSubtree.size() - 1u); i; --i) {
-			AppendChars((const char*)l_Indent, (const char*)l_Indent + 4);
-		}
-	}
-
-	AppendChar(terminator);
-
-	m_CurrentSubtree.pop();
-}

lib/base/json.hpp

@@ -4,14 +4,121 @@
 #define JSON_H
 
 #include "base/i2-base.hpp"
+#include "base/array.hpp"
+#include "base/generator.hpp"
+#include <boost/asio/spawn.hpp>
+#include <json.hpp>
 
 namespace icinga
 {
 
+/**
+ * AsyncJsonWriter allows writing JSON data to any output stream asynchronously.
+ *
+ * All users of this class must ensure that the underlying output stream will not perform any asynchronous I/O
+ * operations when the @c write_character() or @c write_characters() methods are called. They shall only perform
+ * such ops when the @c JsonEncoder allows them to do so by calling the @c MayFlush() method.
+ *
+ * @ingroup base
+ */
+class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol<char>
+{
+public:
+	/**
+	 * It instructs the underlying output stream to write any buffered data to wherever it is supposed to go.
+	 *
+	 * The @c JsonEncoder allows the stream to even perform asynchronous operations in a safe manner by calling
+	 * this method with a dedicated @c boost::asio::yield_context object. The stream must not perform any async
+	 * I/O operations triggered by methods other than this one. Any attempt to do so will result in undefined behavior.
+	 *
+	 * However, this doesn't necessarily enforce the stream to really flush its data immediately, but it's up
+	 * to the implementation to do whatever it needs to. The encoder just gives it a chance to do so by calling
+	 * this method.
+	 *
+	 * @param yield The yield context to use for asynchronous operations.
+	 */
+	virtual void MayFlush(boost::asio::yield_context& yield) = 0;
+};
+
 class String;
 class Value;
 
-String JsonEncode(const Value& value, bool pretty_print = false);
+/**
+ * JSON encoder.
+ *
+ * This class can be used to encode Icinga Value types into JSON format and write them to an output stream.
+ * The supported stream types include any @c std::ostream like objects and our own @c AsyncJsonWriter, which
+ * allows writing JSON data to an Asio stream asynchronously. The nlohmann/json library already provides
+ * full support for the former stream type, while the latter is fully implemented by our own and satisfies the
+ * @c nlohmann::detail::output_adapter_protocol<> interface as well.
+ *
+ * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing
+ * @c nlohmann::detail::serializer<> class to dump numbers and ASCII validated JSON strings. This means that the
+ * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind
+ * of work to serializer<>.
+ *
+ * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object
+ * is indented with 4 spaces and grows linearly with the depth of the object tree.
+ *
+ * @ingroup base
+ */
+class JsonEncoder
+{
+public:
+	explicit JsonEncoder(std::string& output, bool prettify = false);
+	explicit JsonEncoder(std::basic_ostream<char>& stream, bool prettify = false);
+	explicit JsonEncoder(nlohmann::detail::output_adapter_t<char> w, bool prettify = false);
+
+	void Encode(const Value& value, boost::asio::yield_context* yc = nullptr);
+
+private:
+	void EncodeArray(const Array::Ptr& array, boost::asio::yield_context* yc);
+	void EncodeValueGenerator(const ValueGenerator::Ptr& generator, boost::asio::yield_context* yc);
+
+	template<typename Iterable, typename ValExtractor>
+	void EncodeObject(const Iterable& container, const ValExtractor& extractor, boost::asio::yield_context* yc);
+
+	void EncodeNlohmannJson(const nlohmann::json& json) const;
+	void EncodeNumber(double value) const;
+
+	void Write(const std::string_view& sv) const;
+	void BeginContainer(char openChar);
+	void EndContainer(char closeChar, bool isContainerEmpty = false);
+	void WriteSeparatorAndIndentStrIfNeeded(bool emitComma) const;
+
+	// The number of spaces to use for indentation in prettified JSON.
+	static constexpr uint8_t m_IndentSize = 4;
+
+	bool m_Pretty; // Whether to pretty-print the JSON output.
+	unsigned m_Indent{0}; // The current indentation level for pretty-printing.
+	/**
+	 * Pre-allocate for 8 levels of indentation for pretty-printing.
+	 *
+	 * This is used to avoid reallocating the string on every indent level change.
+	 * The size of this string is dynamically adjusted if the indentation level exceeds its initial size at some point.
+	 */
+	std::string m_IndentStr{8*m_IndentSize, ' '};
+
+	// The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter.
+	nlohmann::detail::output_adapter_t<char> m_Writer;
+
+	/**
+	 * This class wraps any @c nlohmann::detail::output_adapter_t<char> writer and provides a method to flush it as
+	 * required. Only @c AsyncJsonWriter supports the flush operation, however, this class is also safe to use with
+	 * other writer types and the flush method does nothing for them.
+	 */
+	class Flusher {
+	public:
+		explicit Flusher(const nlohmann::detail::output_adapter_t<char>& w);
+		void FlushIfSafe(boost::asio::yield_context* yc) const;
+
+	private:
+		AsyncJsonWriter* m_AsyncWriter;
+	} m_Flusher;
+};
+
+String JsonEncode(const Value& value, bool prettify = false);
+void JsonEncode(const Value& value, std::ostream& os, bool prettify = false);
 Value JsonDecode(const String& data);
 
 }

lib/base/namespace.cpp

@@ -1,7 +1,6 @@
 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
 
 #include "base/namespace.hpp"
-#include "base/objectlock.hpp"
 #include "base/debug.hpp"
 #include "base/primitivetype.hpp"
 #include "base/debuginfo.hpp"
@@ -119,7 +118,26 @@ void Namespace::Remove(const String& field)
 void Namespace::Freeze() {
 	ObjectLock olock(this);
 
-	m_Frozen = true;
+	m_Frozen.store(true, std::memory_order_release);
+}
+
+bool Namespace::Frozen() const
+{
+	return m_Frozen.load(std::memory_order_acquire);
+}
+
+/**
+ * Returns an already locked ObjectLock if the namespace is frozen.
+ * Otherwise, returns an unlocked object lock.
+ *
+ * @returns An object lock.
+ */
+ObjectLock Namespace::LockIfRequired()
+{
+	if (Frozen()) {
+		return ObjectLock(this, std::defer_lock);
+	}
+	return ObjectLock(this);
 }
 
 std::shared_lock<std::shared_timed_mutex> Namespace::ReadLockUnlessFrozen() const
@@ -160,14 +178,14 @@ bool Namespace::GetOwnField(const String& field, Value *result) const
 
 Namespace::Iterator Namespace::Begin()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.begin();
 }
 
 Namespace::Iterator Namespace::End()
 {
-	ASSERT(OwnsLock());
+	ASSERT(Frozen() || OwnsLock());
 
 	return m_Data.end();
 }

lib/base/namespace.hpp

@@ -5,6 +5,7 @@
 
 #include "base/i2-base.hpp"
 #include "base/object.hpp"
+#include "base/objectlock.hpp"
 #include "base/shared-object.hpp"
 #include "base/value.hpp"
 #include "base/debuginfo.hpp"
@@ -73,6 +74,8 @@ public:
 	bool Contains(const String& field) const;
 	void Remove(const String& field);
 	void Freeze();
+	bool Frozen() const;
+	ObjectLock LockIfRequired();
 
 	Iterator Begin();
 	Iterator End();

lib/base/objectlock.cpp

@@ -18,6 +18,18 @@ ObjectLock::ObjectLock(const Object::Ptr& object)
 {
 }
 
+/**
+ * Constructs a lock for the given object without locking it immediately.
+ *
+ * The user must call Lock() explicitly when needed.
+ *
+ * @param object The object to lock.
+ */
+ObjectLock::ObjectLock(const Object::Ptr& object, std::defer_lock_t)
+	: m_Object(object.get()), m_Locked(false)
+{
+}
+
 ObjectLock::ObjectLock(const Object *object)
 	: m_Object(object), m_Locked(false)
 {
@@ -53,3 +65,15 @@ void ObjectLock::Unlock()
 		m_Locked = false;
 	}
 }
+
+/**
+ * Returns true if the object is locked, false otherwise.
+ *
+ * This operator allows using ObjectLock in boolean contexts.
+ *
+ * @returns true if the object is locked, false otherwise.
+ */
+ObjectLock::operator bool() const
+{
+	return m_Locked;
+}

lib/base/objectlock.hpp

@@ -15,6 +15,7 @@ struct ObjectLock
 {
 public:
 	ObjectLock(const Object::Ptr& object);
+	ObjectLock(const Object::Ptr& object, std::defer_lock_t);
 	ObjectLock(const Object *object);
 
 	ObjectLock(const ObjectLock&) = delete;
@@ -25,6 +26,8 @@ public:
 	void Lock();
 	void Unlock();
 
+	operator bool() const;
+
 private:
 	const Object *m_Object{nullptr};
 	bool m_Locked{false};

test/base-json.cpp

@@ -4,10 +4,13 @@
 #include "base/function.hpp"
 #include "base/namespace.hpp"
 #include "base/array.hpp"
+#include "base/generator.hpp"
 #include "base/objectlock.hpp"
 #include "base/json.hpp"
 #include <boost/algorithm/string/replace.hpp>
 #include <BoostTestTargetConfig.h>
+#include <limits>
+#include <cmath>
 
 using namespace icinga;
 
@@ -15,26 +18,51 @@ BOOST_AUTO_TEST_SUITE(base_json)
 
 BOOST_AUTO_TEST_CASE(encode)
 {
+	auto generate = []() -> std::optional<Value> {
+		static int count = 0;
+		if (++count == 4) {
+			count = 0;
+			return std::nullopt;
+		}
+		return Value(count);
+	};
+
 	Dictionary::Ptr input (new Dictionary({
 		{ "array", new Array({ new Namespace() }) },
 		{ "false", false },
-		{ "float", -1.25 },
+		// Use double max value to test JSON encoding of large numbers and trigger boost numeric_cast exceptions
+		{ "max_double", std::numeric_limits<double>::max() },
+		// Test the largest uint64_t value that has an exact double representation (2^64-2048).
+		{ "max_int_in_double", std::nextafter(std::pow(2, 64), 0.0) },
+		{ "float", -1.25f },
+		{ "float_without_fraction", 23.0f },
 		{ "fx", new Function("<test>", []() {}) },
 		{ "int", -42 },
 		{ "null", Value() },
 		{ "string", "LF\nTAB\tAUml\xC3\xA4Ill\xC3" },
 		{ "true", true },
-		{ "uint", 23u }
+		{ "uint", 23u },
+		{ "generator", new ValueGenerator(generate) },
+		{ "empty_generator", new ValueGenerator([]() -> std::optional<Value> { return std::nullopt; }) },
 	}));
 
 	String output (R"EOF({
     "array": [
         {}
     ],
+    "empty_generator": [],
     "false": false,
     "float": -1.25,
+    "float_without_fraction": 23,
     "fx": "Object of type 'Function'",
+    "generator": [
+        1,
+        2,
+        3
+    ],
     "int": -42,
+    "max_double": 1.7976931348623157e+308,
+    "max_int_in_double": 18446744073709549568,
     "null": null,
     "string": "LF\nTAB\tAUml\u00e4Ill\ufffd",
     "true": true,
@@ -42,7 +70,12 @@ BOOST_AUTO_TEST_CASE(encode)
 }
 )EOF");
 
-	BOOST_CHECK(JsonEncode(input, true) == output);
+	auto got(JsonEncode(input, true));
+	BOOST_CHECK_EQUAL(output, got);
+
+	std::ostringstream oss;
+	JsonEncode(input, oss, true);
+	BOOST_CHECK_EQUAL(output, oss.str());
 
 	boost::algorithm::replace_all(output, " ", "");
 	boost::algorithm::replace_all(output, "Objectoftype'Function'", "Object of type 'Function'");