2013-08-14 16:39:12 -04:00
|
|
|
|
|
|
|
// DeadlockDetect.h
|
|
|
|
|
|
|
|
// Declares the cDeadlockDetect class that tries to detect deadlocks and aborts the server when it detects one
|
|
|
|
|
|
|
|
/*
|
|
|
|
This class simply monitors each world's m_WorldAge, which is expected to grow on each tick.
|
|
|
|
If the world age doesn't grow for several seconds, it's either because the server is super-overloaded,
|
|
|
|
or because the world tick thread hangs in a deadlock. We presume the latter and therefore kill the server.
|
|
|
|
Once we learn to write crashdumps programmatically, we should do so just before killing, to enable debugging.
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include "OSSupport/IsThread.h"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class cDeadlockDetect :
|
|
|
|
public cIsThread
|
|
|
|
{
|
|
|
|
typedef cIsThread super;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2013-08-14 16:39:12 -04:00
|
|
|
public:
|
|
|
|
cDeadlockDetect(void);
|
2017-05-20 02:16:28 -04:00
|
|
|
virtual ~cDeadlockDetect() override;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2014-10-20 13:59:40 -04:00
|
|
|
/** Starts the detection. Hides cIsThread's Start, because we need some initialization */
|
2013-11-30 16:14:47 -05:00
|
|
|
bool Start(int a_IntervalSec);
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2017-01-17 16:38:04 -05:00
|
|
|
/** Adds the critical section for tracking.
|
|
|
|
Tracked CSs are listed, together with ownership details, when a deadlock is detected.
|
|
|
|
A tracked CS must be untracked before it is destroyed.
|
|
|
|
a_Name is an arbitrary name that is listed along with the CS in the output. */
|
|
|
|
void TrackCriticalSection(cCriticalSection & a_CS, const AString & a_Name);
|
|
|
|
|
|
|
|
/** Removes the CS from the tracking. */
|
|
|
|
void UntrackCriticalSection(cCriticalSection & a_CS);
|
|
|
|
|
2013-08-14 16:39:12 -04:00
|
|
|
protected:
|
|
|
|
struct sWorldAge
|
|
|
|
{
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Last m_WorldAge that has been detected in this world */
|
2013-08-14 16:39:12 -04:00
|
|
|
Int64 m_Age;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Number of cycles for which the age has been the same */
|
2013-08-14 16:39:12 -04:00
|
|
|
int m_NumCyclesSame;
|
|
|
|
} ;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Maps world name -> sWorldAge */
|
2013-08-14 16:39:12 -04:00
|
|
|
typedef std::map<AString, sWorldAge> WorldAges;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2017-01-17 16:38:04 -05:00
|
|
|
/** Protects m_TrackedCriticalSections from multithreaded access. */
|
|
|
|
cCriticalSection m_CS;
|
|
|
|
|
|
|
|
/** CriticalSections that are tracked (their status output on deadlock).
|
|
|
|
Protected against multithreaded access by m_CS. */
|
|
|
|
std::vector<std::pair<cCriticalSection *, AString>> m_TrackedCriticalSections;
|
|
|
|
|
2013-08-14 16:39:12 -04:00
|
|
|
WorldAges m_WorldAges;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Number of secods for which the ages must be the same for the detection to trigger */
|
2013-11-30 16:14:47 -05:00
|
|
|
int m_IntervalSec;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
|
|
|
|
2013-08-14 16:39:12 -04:00
|
|
|
// cIsThread overrides:
|
|
|
|
virtual void Execute(void) override;
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Sets the initial world age */
|
2013-08-14 16:39:12 -04:00
|
|
|
void SetWorldAge(const AString & a_WorldName, Int64 a_Age);
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2015-07-31 10:49:10 -04:00
|
|
|
/** Checks if the world's age has changed, updates the world's stats; calls DeadlockDetected() if deadlock detected */
|
2013-08-14 16:39:12 -04:00
|
|
|
void CheckWorldAge(const AString & a_WorldName, Int64 a_Age);
|
2016-02-05 16:45:45 -05:00
|
|
|
|
2017-01-17 08:10:02 -05:00
|
|
|
/** Called when a deadlock is detected in a world. Aborts the server.
|
|
|
|
a_WorldName is the name of the world whose age has triggered the detection.
|
|
|
|
a_WorldAge is the age (in ticks) in which the world is stuck. */
|
|
|
|
NORETURN void DeadlockDetected(const AString & a_WorldName, Int64 a_WorldAge);
|
2017-01-17 16:38:04 -05:00
|
|
|
|
|
|
|
/** Outputs a listing of the tracked CSs, together with their name and state. */
|
|
|
|
void ListTrackedCSs();
|
2013-08-14 16:39:12 -04:00
|
|
|
} ;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|