CheckerComponent sometimes fails to schedule checks in time

fixes #12180
This commit is contained in:
Gunnar Beutner 2016-07-28 12:50:48 +02:00
parent 256d936661
commit 21cebe97a1
2 changed files with 36 additions and 12 deletions

View File

@ -117,9 +117,9 @@ void CheckerComponent::CheckThreadProc(void)
break; break;
CheckTimeView::iterator it = idx.begin(); CheckTimeView::iterator it = idx.begin();
Checkable::Ptr checkable = *it; CheckableScheduleInfo csi = *it;
double wait = checkable->GetNextCheck() - Utility::GetTime(); double wait = csi.NextCheck - Utility::GetTime();
if (Checkable::GetPendingChecks() >= GetConcurrentChecks()) if (Checkable::GetPendingChecks() >= GetConcurrentChecks())
wait = 0.5; wait = 0.5;
@ -131,6 +131,8 @@ void CheckerComponent::CheckThreadProc(void)
continue; continue;
} }
Checkable::Ptr checkable = csi.Object;
m_IdleCheckables.erase(checkable); m_IdleCheckables.erase(checkable);
bool forced = checkable->GetForceNextCheck(); bool forced = checkable->GetForceNextCheck();
@ -170,7 +172,7 @@ void CheckerComponent::CheckThreadProc(void)
/* reschedule the checkable if checks are disabled */ /* reschedule the checkable if checks are disabled */
if (!check) { if (!check) {
m_IdleCheckables.insert(checkable); m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
lock.unlock(); lock.unlock();
checkable->UpdateNextCheck(); checkable->UpdateNextCheck();
@ -180,7 +182,7 @@ void CheckerComponent::CheckThreadProc(void)
continue; continue;
} }
m_PendingCheckables.insert(checkable); m_PendingCheckables.insert(GetCheckableScheduleInfo(checkable));
lock.unlock(); lock.unlock();
@ -232,7 +234,7 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
m_PendingCheckables.erase(it); m_PendingCheckables.erase(it);
if (checkable->IsActive()) if (checkable->IsActive())
m_IdleCheckables.insert(checkable); m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
m_CV.notify_all(); m_CV.notify_all();
} }
@ -250,7 +252,7 @@ void CheckerComponent::ResultTimerHandler(void)
boost::mutex::scoped_lock lock(m_Mutex); boost::mutex::scoped_lock lock(m_Mutex);
msgbuf << "Pending checkables: " << m_PendingCheckables.size() << "; Idle checkables: " << m_IdleCheckables.size() << "; Checks/s: " msgbuf << "Pending checkables: " << m_PendingCheckables.size() << "; Idle checkables: " << m_IdleCheckables.size() << "; Checks/s: "
<< (CIB::GetActiveHostChecksStatistics(5) + CIB::GetActiveServiceChecksStatistics(5)) / 5.0; << (CIB::GetActiveHostChecksStatistics(60) + CIB::GetActiveServiceChecksStatistics(60)) / 60.0;
} }
Log(LogNotice, "CheckerComponent", msgbuf.str()); Log(LogNotice, "CheckerComponent", msgbuf.str());
@ -273,7 +275,7 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
if (m_PendingCheckables.find(checkable) != m_PendingCheckables.end()) if (m_PendingCheckables.find(checkable) != m_PendingCheckables.end())
return; return;
m_IdleCheckables.insert(checkable); m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
} else { } else {
m_IdleCheckables.erase(checkable); m_IdleCheckables.erase(checkable);
m_PendingCheckables.erase(checkable); m_PendingCheckables.erase(checkable);
@ -283,6 +285,14 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
} }
} }
CheckableScheduleInfo CheckerComponent::GetCheckableScheduleInfo(const Checkable::Ptr& checkable)
{
CheckableScheduleInfo csi;
csi.Object = checkable;
csi.NextCheck = checkable->GetNextCheck();
return csi;
}
void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable) void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
{ {
boost::mutex::scoped_lock lock(m_Mutex); boost::mutex::scoped_lock lock(m_Mutex);
@ -296,7 +306,10 @@ void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
return; return;
idx.erase(checkable); idx.erase(checkable);
idx.insert(checkable);
CheckableScheduleInfo csi = GetCheckableScheduleInfo(checkable);
idx.insert(csi);
m_CV.notify_all(); m_CV.notify_all();
} }

View File

@ -35,6 +35,15 @@
namespace icinga namespace icinga
{ {
/**
* @ingroup checker
*/
struct CheckableScheduleInfo
{
Checkable::Ptr Object;
double NextCheck;
};
/** /**
* @ingroup checker * @ingroup checker
*/ */
@ -45,9 +54,9 @@ struct CheckableNextCheckExtractor
/** /**
* @threadsafety Always. * @threadsafety Always.
*/ */
double operator()(const Checkable::Ptr& checkable) double operator()(const CheckableScheduleInfo& csi)
{ {
return checkable->GetNextCheck(); return csi.NextCheck;
} }
}; };
@ -61,9 +70,9 @@ public:
DECLARE_OBJECTNAME(CheckerComponent); DECLARE_OBJECTNAME(CheckerComponent);
typedef boost::multi_index_container< typedef boost::multi_index_container<
Checkable::Ptr, CheckableScheduleInfo,
boost::multi_index::indexed_by< boost::multi_index::indexed_by<
boost::multi_index::ordered_unique<boost::multi_index::identity<Checkable::Ptr> >, boost::multi_index::ordered_unique<boost::multi_index::member<CheckableScheduleInfo, Checkable::Ptr, &CheckableScheduleInfo::Object> >,
boost::multi_index::ordered_non_unique<CheckableNextCheckExtractor> boost::multi_index::ordered_non_unique<CheckableNextCheckExtractor>
> >
> CheckableSet; > CheckableSet;
@ -100,6 +109,8 @@ private:
void NextCheckChangedHandler(const Checkable::Ptr& checkable); void NextCheckChangedHandler(const Checkable::Ptr& checkable);
void RescheduleCheckTimer(void); void RescheduleCheckTimer(void);
static CheckableScheduleInfo GetCheckableScheduleInfo(const Checkable::Ptr& checkable);
}; };
} }