Implement support for CLIENT_MULTI_STATEMENTS

fixes #8738

lib/db_ido_mysql/idomysqlconnection.cpp

@@ -100,8 +100,6 @@ void IdoMysqlConnection::ExceptionHandler(boost::exception_ptr exp)
 	Log(LogDebug, "IdoMysqlConnection")
 	    << "Exception during database operation: " << DiagnosticInformation(exp);
 
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
-
 	if (GetConnected()) {
 		mysql_close(&m_Connection);
 
@@ -118,8 +116,6 @@ void IdoMysqlConnection::Disconnect(void)
 {
 	AssertOnWorkQueue();
 
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
-
 	if (!GetConnected())
 		return;
 
@@ -136,12 +132,13 @@ void IdoMysqlConnection::TxTimerHandler(void)
 
 void IdoMysqlConnection::NewTransaction(void)
 {
+	m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::FinishAsyncQueries, this, true));
 	m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::InternalNewTransaction, this));
 }
 
 void IdoMysqlConnection::InternalNewTransaction(void)
 {
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
+	AssertOnWorkQueue();
 
 	if (!GetConnected())
 		return;
@@ -165,9 +162,6 @@ void IdoMysqlConnection::Reconnect(void)
 
 	std::vector<DbObject::Ptr> active_dbobjs;
 
-	{
-		boost::mutex::scoped_lock lock(m_ConnectionMutex);
-
 	bool reconnect = false;
 
 	if (GetConnected()) {
@@ -207,7 +201,7 @@ void IdoMysqlConnection::Reconnect(void)
 		BOOST_THROW_EXCEPTION(std::bad_alloc());
 	}
 
-		if (!mysql_real_connect(&m_Connection, host, user, passwd, db, port, socket_path, CLIENT_FOUND_ROWS)) {
+	if (!mysql_real_connect(&m_Connection, host, user, passwd, db, port, socket_path, CLIENT_FOUND_ROWS | CLIENT_MULTI_STATEMENTS)) {
 		Log(LogCritical, "IdoMysqlConnection")
 		    << "Connection to database '" << db << "' with user '" << user << "' on '" << host << ":" << port
 		    << "' failed: \"" << mysql_error(&m_Connection) << "\"";
@@ -217,11 +211,22 @@ void IdoMysqlConnection::Reconnect(void)
 
 	SetConnected(true);
 
-		String dbVersionName = "idoutils";
+	IdoMysqlResult result = Query("SELECT @@global.max_allowed_packet AS max_allowed_packet");
-		IdoMysqlResult result = Query("SELECT version FROM " + GetTablePrefix() + "dbversion WHERE name='" + Escape(dbVersionName) + "'");
 
 	Dictionary::Ptr row = FetchRow(result);
 
+	if (row)
+		m_MaxPacketSize = row->Get("max_allowed_packet");
+	else
+		m_MaxPacketSize = 64 * 1024;
+
+	DiscardRows(result);
+
+	String dbVersionName = "idoutils";
+	result = Query("SELECT version FROM " + GetTablePrefix() + "dbversion WHERE name='" + Escape(dbVersionName) + "'");
+
+	row = FetchRow(result);
+
 	if (!row) {
 		mysql_close(&m_Connection);
 		SetConnected(false);
@@ -350,7 +355,6 @@ void IdoMysqlConnection::Reconnect(void)
 	}
 
 	Query("BEGIN");
-	}
 
 	UpdateAllObjects();
 
@@ -370,10 +374,119 @@ void IdoMysqlConnection::ClearConfigTable(const String& table)
 	Query("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " + Convert::ToString(static_cast<long>(m_InstanceID)));
 }
 
+void IdoMysqlConnection::AsyncQuery(const String& query, const boost::function<void (const IdoMysqlResult&)>& callback)
+{
+	AssertOnWorkQueue();
+
+	IdoAsyncQuery aq;
+	aq.Query = query;
+	aq.Callback = callback;
+	m_AsyncQueries.push_back(aq);
+	m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::FinishAsyncQueries, this, false));
+}
+
+void IdoMysqlConnection::FinishAsyncQueries(bool force)
+{
+	if (m_AsyncQueries.size() < 10 && !force)
+		return;
+
+	std::vector<IdoAsyncQuery> queries;
+	m_AsyncQueries.swap(queries);
+
+	std::vector<IdoAsyncQuery>::size_type offset = 0;
+
+	while (offset < queries.size()) {
+		std::ostringstream querybuf;
+
+		std::vector<IdoAsyncQuery>::size_type count = 0;
+		size_t num_bytes = 0;
+
+		for (std::vector<IdoAsyncQuery>::size_type i = offset; i < queries.size(); i++) {
+			const IdoAsyncQuery& aq = queries[i];
+
+			size_t size_query = aq.Query.GetLength() + 1;
+
+			if (num_bytes + size_query > m_MaxPacketSize - 512)
+				break;
+
+			if (count > 0)
+				querybuf << ";";
+
+			IncreaseQueryCount();
+			count++;
+
+			querybuf << aq.Query;
+			num_bytes += size_query;
+		}
+
+		String query = querybuf.str();
+
+		if (mysql_query(&m_Connection, query.CStr()) != 0) {
+			std::ostringstream msgbuf;
+			String message = mysql_error(&m_Connection);
+			msgbuf << "Error \"" << message << "\" when executing query \"" << query << "\"";
+			Log(LogCritical, "IdoMysqlConnection", msgbuf.str());
+
+			BOOST_THROW_EXCEPTION(
+			    database_error()
+				<< errinfo_message(mysql_error(&m_Connection))
+				<< errinfo_database_query(query)
+			);
+		}
+
+		for (std::vector<IdoAsyncQuery>::size_type i = offset; i < offset + count; i++) {
+			const IdoAsyncQuery& aq = queries[i];
+
+			m_AffectedRows = mysql_affected_rows(&m_Connection);
+
+			MYSQL_RES *result = mysql_use_result(&m_Connection);
+
+			IdoMysqlResult iresult;
+
+			if (!result) {
+				if (mysql_field_count(&m_Connection) > 0) {
+					std::ostringstream msgbuf;
+					String message = mysql_error(&m_Connection);
+					msgbuf << "Error \"" << message << "\" when executing query \"" << aq.Query << "\"";
+					Log(LogCritical, "IdoMysqlConnection", msgbuf.str());
+
+					BOOST_THROW_EXCEPTION(
+					    database_error()
+						<< errinfo_message(mysql_error(&m_Connection))
+						<< errinfo_database_query(query)
+					);
+				}
+			} else
+				iresult = IdoMysqlResult(result, std::ptr_fun(mysql_free_result));
+
+			if (aq.Callback)
+				aq.Callback(iresult);
+
+			if (mysql_next_result(&m_Connection) > 0) {
+				std::ostringstream msgbuf;
+				String message = mysql_error(&m_Connection);
+				msgbuf << "Error \"" << message << "\" when executing query \"" << query << "\"";
+				Log(LogCritical, "IdoMysqlConnection", msgbuf.str());
+
+				BOOST_THROW_EXCEPTION(
+				    database_error()
+					<< errinfo_message(mysql_error(&m_Connection))
+					<< errinfo_database_query(query)
+				);
+			}
+		}
+
+		offset += count;
+	}
+}
+
 IdoMysqlResult IdoMysqlConnection::Query(const String& query)
 {
 	AssertOnWorkQueue();
 
+	/* finish all async queries to maintain the right order for queries */
+	FinishAsyncQueries(true);
+
 	Log(LogDebug, "IdoMysqlConnection")
 	    << "Query: " << query;
 
@@ -483,12 +596,13 @@ void IdoMysqlConnection::DiscardRows(const IdoMysqlResult& result)
 
 void IdoMysqlConnection::ActivateObject(const DbObject::Ptr& dbobj)
 {
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
+	m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::InternalActivateObject, this, dbobj));
-	InternalActivateObject(dbobj);
 }
 
 void IdoMysqlConnection::InternalActivateObject(const DbObject::Ptr& dbobj)
 {
+	AssertOnWorkQueue();
+
 	if (!GetConnected())
 		return;
 
@@ -510,13 +624,18 @@ void IdoMysqlConnection::InternalActivateObject(const DbObject::Ptr& dbobj)
 		SetObjectID(dbobj, GetLastInsertID());
 	} else {
 		qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 1 WHERE object_id = " << static_cast<long>(dbref);
-		Query(qbuf.str());
+		AsyncQuery(qbuf.str());
 	}
 }
 
 void IdoMysqlConnection::DeactivateObject(const DbObject::Ptr& dbobj)
 {
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
+	m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::InternalDeactivateObject, this, dbobj));
+}
+
+void IdoMysqlConnection::InternalDeactivateObject(const DbObject::Ptr& dbobj)
+{
+	AssertOnWorkQueue();
 
 	if (!GetConnected())
 		return;
@@ -528,13 +647,12 @@ void IdoMysqlConnection::DeactivateObject(const DbObject::Ptr& dbobj)
 
 	std::ostringstream qbuf;
 	qbuf << "UPDATE " + GetTablePrefix() + "objects SET is_active = 0 WHERE object_id = " << static_cast<long>(dbref);
-	Query(qbuf.str());
+	AsyncQuery(qbuf.str());
 
 	/* Note that we're _NOT_ clearing the db refs via SetReference/SetConfigUpdate/SetStatusUpdate
 	 * because the object is still in the database. */
 }
 
-/* caller must hold m_ConnectionMutex */
 bool IdoMysqlConnection::FieldToEscapedString(const String& key, const Value& value, Value *result)
 {
 	if (key == "instance_id") {
@@ -606,7 +724,7 @@ void IdoMysqlConnection::ExecuteQuery(const DbQuery& query)
 
 void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride)
 {
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
+	AssertOnWorkQueue();
 
 	if ((query.Category & GetCategories()) == 0)
 		return;
@@ -716,11 +834,12 @@ void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query, DbQueryType
 	if (type != DbQueryInsert)
 		qbuf << where.str();
 
-	Query(qbuf.str());
+	AsyncQuery(qbuf.str(), boost::bind(&IdoMysqlConnection::FinishExecuteQuery, this, query, type, upsert));
+}
 
+void IdoMysqlConnection::FinishExecuteQuery(const DbQuery& query, int type, bool upsert)
+{
 	if (upsert && GetAffectedRows() == 0) {
-		lock.unlock();
-
 		DbQueryType to = DbQueryInsert;
 		InternalExecuteQuery(query, &to);
 
@@ -749,12 +868,12 @@ void IdoMysqlConnection::CleanUpExecuteQuery(const String& table, const String&
 
 void IdoMysqlConnection::InternalCleanUpExecuteQuery(const String& table, const String& time_column, double max_age)
 {
-	boost::mutex::scoped_lock lock(m_ConnectionMutex);
+	AssertOnWorkQueue();
 
 	if (!GetConnected())
 		return;
 
-	Query("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " +
+	AsyncQuery("DELETE FROM " + GetTablePrefix() + table + " WHERE instance_id = " +
 	    Convert::ToString(static_cast<long>(m_InstanceID)) + " AND " + time_column +
 	    " < FROM_UNIXTIME(" + Convert::ToString(static_cast<long>(max_age)) + ")");
 }

lib/db_ido_mysql/idomysqlconnection.hpp

@@ -31,6 +31,14 @@ namespace icinga
 
 typedef boost::shared_ptr<MYSQL_RES> IdoMysqlResult;
 
+typedef boost::function<void (const IdoMysqlResult&)> IdoAsyncCallback;
+
+struct IdoAsyncQuery
+{
+	String Query;
+	IdoAsyncCallback Callback;
+};
+
 /**
  * An IDO MySQL database connection.
  *
@@ -64,9 +72,11 @@ private:
 
 	WorkQueue m_QueryQueue;
 
-	boost::mutex m_ConnectionMutex;
 	MYSQL m_Connection;
 	int m_AffectedRows;
+	int m_MaxPacketSize;
+
+	std::vector<IdoAsyncQuery> m_AsyncQueries;
 
 	Timer::Ptr m_ReconnectTimer;
 	Timer::Ptr m_TxTimer;
@@ -78,8 +88,12 @@ private:
 	Dictionary::Ptr FetchRow(const IdoMysqlResult& result);
 	void DiscardRows(const IdoMysqlResult& result);
 
+	void AsyncQuery(const String& query, const IdoAsyncCallback& callback = IdoAsyncCallback());
+	void FinishAsyncQueries(bool force = false);
+
 	bool FieldToEscapedString(const String& key, const Value& value, Value *result);
 	void InternalActivateObject(const DbObject::Ptr& dbobj);
+	void InternalDeactivateObject(const DbObject::Ptr& dbobj);
 
 	void Disconnect(void);
 	void Reconnect(void);
@@ -90,6 +104,7 @@ private:
 	void ReconnectTimerHandler(void);
 
 	void InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride = NULL);
+	void FinishExecuteQuery(const DbQuery& query, int type, bool upsert);
 	void InternalCleanUpExecuteQuery(const String& table, const String& time_key, double time_value);
 	void InternalNewTransaction(void);