Merge pull request #3544 from cuberite/DeadlockDetectMoreInfo
This commit is contained in:
commit
0256daa7ca
@ -2420,6 +2420,34 @@ end
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
function HandleConsoleDeadlock(a_Split)
|
||||||
|
-- If given a parameter, assume it's a world name and simulate a deadlock in the world's tick thread
|
||||||
|
if (a_Split[2]) then
|
||||||
|
local world = cRoot:Get():GetWorld(a_Split[2])
|
||||||
|
if (world) then
|
||||||
|
world:ScheduleTask(0,
|
||||||
|
function()
|
||||||
|
-- Make a live-lock:
|
||||||
|
while (true) do
|
||||||
|
end
|
||||||
|
end
|
||||||
|
)
|
||||||
|
return true, "Deadlock in world tick thread for world " .. a_Split[2] .. " has been scheduled."
|
||||||
|
end
|
||||||
|
LOG("Not a world name: " .. a_Split[2] .. "; simulating a deadlock in the command execution thread instead.")
|
||||||
|
else
|
||||||
|
LOG("Simulating a deadlock in the command execution thread.")
|
||||||
|
end
|
||||||
|
|
||||||
|
-- Make a live-lock in the command execution thread:
|
||||||
|
while(true) do
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function HandleConsoleDownload(a_Split)
|
function HandleConsoleDownload(a_Split)
|
||||||
-- Check params:
|
-- Check params:
|
||||||
local url = a_Split[2]
|
local url = a_Split[2]
|
||||||
|
@ -266,6 +266,12 @@ g_PluginInfo =
|
|||||||
HelpString = "Performs cBoundingBox API tests",
|
HelpString = "Performs cBoundingBox API tests",
|
||||||
},
|
},
|
||||||
|
|
||||||
|
["deadlock"] =
|
||||||
|
{
|
||||||
|
Handler = HandleConsoleDeadlock,
|
||||||
|
HelpString = "Simulates a deadlock, either on the command execution thread, or on a world tick thread",
|
||||||
|
},
|
||||||
|
|
||||||
["download"] =
|
["download"] =
|
||||||
{
|
{
|
||||||
Handler = HandleConsoleDownload,
|
Handler = HandleConsoleDownload,
|
||||||
|
@ -19,6 +19,7 @@ extern "C"
|
|||||||
#include "LuaJson.h"
|
#include "LuaJson.h"
|
||||||
#include "../Entities/Entity.h"
|
#include "../Entities/Entity.h"
|
||||||
#include "../BlockEntities/BlockEntity.h"
|
#include "../BlockEntities/BlockEntity.h"
|
||||||
|
#include "../DeadlockDetect.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -2225,6 +2226,24 @@ void cLuaState::LogApiCallParamFailure(const char * a_FnName, const char * a_Par
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cLuaState::TrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect)
|
||||||
|
{
|
||||||
|
a_DeadlockDetect.TrackCriticalSection(m_CS, Printf("cLuaState %s", m_SubsystemName.c_str()));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cLuaState::UntrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect)
|
||||||
|
{
|
||||||
|
a_DeadlockDetect.UntrackCriticalSection(m_CS);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
int cLuaState::ReportFnCallErrors(lua_State * a_LuaState)
|
int cLuaState::ReportFnCallErrors(lua_State * a_LuaState)
|
||||||
{
|
{
|
||||||
LOGWARNING("LUA: %s", lua_tostring(a_LuaState, -1));
|
LOGWARNING("LUA: %s", lua_tostring(a_LuaState, -1));
|
||||||
|
@ -46,6 +46,7 @@ class cLuaServerHandle;
|
|||||||
class cLuaTCPLink;
|
class cLuaTCPLink;
|
||||||
class cLuaUDPEndpoint;
|
class cLuaUDPEndpoint;
|
||||||
class cPluginLua;
|
class cPluginLua;
|
||||||
|
class cDeadlockDetect;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -822,6 +823,12 @@ public:
|
|||||||
logs the stack trace and stack values. */
|
logs the stack trace and stack values. */
|
||||||
void LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames);
|
void LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames);
|
||||||
|
|
||||||
|
/** Adds this object's CS to the DeadlockDetect's tracked CSs. */
|
||||||
|
void TrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
|
/** Removes this object's CS from the DeadlockDetect's tracked CSs. */
|
||||||
|
void UntrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
cCriticalSection m_CS;
|
cCriticalSection m_CS;
|
||||||
|
@ -33,10 +33,12 @@ extern "C"
|
|||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
// cPluginLua:
|
// cPluginLua:
|
||||||
|
|
||||||
cPluginLua::cPluginLua(const AString & a_PluginDirectory) :
|
cPluginLua::cPluginLua(const AString & a_PluginDirectory, cDeadlockDetect & a_DeadlockDetect) :
|
||||||
cPlugin(a_PluginDirectory),
|
cPlugin(a_PluginDirectory),
|
||||||
m_LuaState(Printf("plugin %s", a_PluginDirectory.c_str()))
|
m_LuaState(Printf("plugin %s", a_PluginDirectory.c_str())),
|
||||||
|
m_DeadlockDetect(a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
|
m_LuaState.TrackInDeadlockDetect(a_DeadlockDetect);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -46,6 +48,7 @@ cPluginLua::cPluginLua(const AString & a_PluginDirectory) :
|
|||||||
cPluginLua::~cPluginLua()
|
cPluginLua::~cPluginLua()
|
||||||
{
|
{
|
||||||
Close();
|
Close();
|
||||||
|
m_LuaState.UntrackInDeadlockDetect(m_DeadlockDetect);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -62,7 +62,7 @@ public:
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
cPluginLua(const AString & a_PluginDirectory);
|
cPluginLua(const AString & a_PluginDirectory, cDeadlockDetect & a_DeadlockDetect);
|
||||||
~cPluginLua();
|
~cPluginLua();
|
||||||
|
|
||||||
virtual void OnDisable(void) override;
|
virtual void OnDisable(void) override;
|
||||||
@ -179,6 +179,9 @@ protected:
|
|||||||
/** Hooks that the plugin has registered. */
|
/** Hooks that the plugin has registered. */
|
||||||
cHookMap m_HookMap;
|
cHookMap m_HookMap;
|
||||||
|
|
||||||
|
/** The DeadlockDetect object to which the plugin's CS is tracked. */
|
||||||
|
cDeadlockDetect & m_DeadlockDetect;
|
||||||
|
|
||||||
|
|
||||||
/** Releases all Lua references, notifies and removes all m_Resettables[] and closes the m_LuaState. */
|
/** Releases all Lua references, notifies and removes all m_Resettables[] and closes the m_LuaState. */
|
||||||
void Close(void);
|
void Close(void);
|
||||||
|
@ -32,8 +32,9 @@ cPluginManager * cPluginManager::Get(void)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
cPluginManager::cPluginManager(void) :
|
cPluginManager::cPluginManager(cDeadlockDetect & a_DeadlockDetect) :
|
||||||
m_bReloadPlugins(false)
|
m_bReloadPlugins(false),
|
||||||
|
m_DeadlockDetect(a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -98,7 +99,7 @@ void cPluginManager::RefreshPluginList(void)
|
|||||||
} // for plugin - m_Plugins[]
|
} // for plugin - m_Plugins[]
|
||||||
if (!hasFound)
|
if (!hasFound)
|
||||||
{
|
{
|
||||||
m_Plugins.push_back(std::make_shared<cPluginLua>(folder));
|
m_Plugins.push_back(std::make_shared<cPluginLua>(folder, m_DeadlockDetect));
|
||||||
}
|
}
|
||||||
} // for folder - Folders[]
|
} // for folder - Folders[]
|
||||||
}
|
}
|
||||||
@ -601,6 +602,30 @@ bool cPluginManager::CallHookEntityChangedWorld(cEntity & a_Entity, cWorld & a_W
|
|||||||
|
|
||||||
bool cPluginManager::CallHookExecuteCommand(cPlayer * a_Player, const AStringVector & a_Split, const AString & a_EntireCommand, CommandResult & a_Result)
|
bool cPluginManager::CallHookExecuteCommand(cPlayer * a_Player, const AStringVector & a_Split, const AString & a_EntireCommand, CommandResult & a_Result)
|
||||||
{
|
{
|
||||||
|
// Output the command being executed to log (for troubleshooting deadlocks-in-commands):
|
||||||
|
if (a_Player != nullptr)
|
||||||
|
{
|
||||||
|
auto world = a_Player->GetWorld();
|
||||||
|
AString worldName;
|
||||||
|
Int64 worldAge;
|
||||||
|
if (world != nullptr)
|
||||||
|
{
|
||||||
|
worldName = world->GetName();
|
||||||
|
worldAge = world->GetWorldAge();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
worldName = "<no world>";
|
||||||
|
worldAge = 0;
|
||||||
|
}
|
||||||
|
LOG("Player %s is executing command \"%s\" in world \"%s\" at world age %lld.",
|
||||||
|
a_Player->GetName().c_str(),
|
||||||
|
a_EntireCommand.c_str(),
|
||||||
|
worldName.c_str(),
|
||||||
|
worldAge
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
FIND_HOOK(HOOK_EXECUTE_COMMAND);
|
FIND_HOOK(HOOK_EXECUTE_COMMAND);
|
||||||
VERIFY_HOOK;
|
VERIFY_HOOK;
|
||||||
|
|
||||||
|
@ -26,6 +26,7 @@ class cPlugin;
|
|||||||
class cProjectileEntity;
|
class cProjectileEntity;
|
||||||
class cWorld;
|
class cWorld;
|
||||||
class cSettingsRepositoryInterface;
|
class cSettingsRepositoryInterface;
|
||||||
|
class cDeadlockDetect;
|
||||||
struct TakeDamageInfo;
|
struct TakeDamageInfo;
|
||||||
|
|
||||||
typedef SharedPtr<cPlugin> cPluginPtr;
|
typedef SharedPtr<cPlugin> cPluginPtr;
|
||||||
@ -413,8 +414,11 @@ private:
|
|||||||
/** If set to true, all the plugins will be reloaded within the next call to Tick(). */
|
/** If set to true, all the plugins will be reloaded within the next call to Tick(). */
|
||||||
bool m_bReloadPlugins;
|
bool m_bReloadPlugins;
|
||||||
|
|
||||||
|
/** The deadlock detect in which all plugins should track their CSs. */
|
||||||
|
cDeadlockDetect & m_DeadlockDetect;
|
||||||
|
|
||||||
cPluginManager();
|
|
||||||
|
cPluginManager(cDeadlockDetect & a_DeadlockDetect);
|
||||||
virtual ~cPluginManager();
|
virtual ~cPluginManager();
|
||||||
|
|
||||||
/** Reloads all plugins, defaulting to settings.ini for settings location */
|
/** Reloads all plugins, defaulting to settings.ini for settings location */
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
#include "SetChunkData.h"
|
#include "SetChunkData.h"
|
||||||
#include "Blocks/ChunkInterface.h"
|
#include "Blocks/ChunkInterface.h"
|
||||||
#include "Entities/Pickup.h"
|
#include "Entities/Pickup.h"
|
||||||
|
#include "DeadlockDetect.h"
|
||||||
|
|
||||||
#ifndef _WIN32
|
#ifndef _WIN32
|
||||||
#include <cstdlib> // abs
|
#include <cstdlib> // abs
|
||||||
@ -2783,6 +2784,24 @@ void cChunkMap::SetChunkAlwaysTicked(int a_ChunkX, int a_ChunkZ, bool a_AlwaysTi
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cChunkMap::TrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect, const AString & a_WorldName)
|
||||||
|
{
|
||||||
|
a_DeadlockDetect.TrackCriticalSection(m_CSChunks, Printf("World %s chunkmap", a_WorldName.c_str()));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cChunkMap::UntrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect)
|
||||||
|
{
|
||||||
|
a_DeadlockDetect.UntrackCriticalSection(m_CSChunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cChunkMap::FastSetBlock(int a_BlockX, int a_BlockY, int a_BlockZ, BLOCKTYPE a_BlockType, NIBBLETYPE a_BlockMeta)
|
void cChunkMap::FastSetBlock(int a_BlockX, int a_BlockY, int a_BlockZ, BLOCKTYPE a_BlockType, NIBBLETYPE a_BlockMeta)
|
||||||
{
|
{
|
||||||
int ChunkX, ChunkZ, X = a_BlockX, Y = a_BlockY, Z = a_BlockZ;
|
int ChunkX, ChunkZ, X = a_BlockX, Y = a_BlockY, Z = a_BlockZ;
|
||||||
|
@ -38,6 +38,7 @@ class cMobCensus;
|
|||||||
class cMobSpawner;
|
class cMobSpawner;
|
||||||
class cSetChunkData;
|
class cSetChunkData;
|
||||||
class cBoundingBox;
|
class cBoundingBox;
|
||||||
|
class cDeadlockDetect;
|
||||||
|
|
||||||
typedef std::list<cClientHandle *> cClientHandleList;
|
typedef std::list<cClientHandle *> cClientHandleList;
|
||||||
typedef cChunk * cChunkPtr;
|
typedef cChunk * cChunkPtr;
|
||||||
@ -411,6 +412,12 @@ public:
|
|||||||
as at least one requests is active the chunk will be ticked). */
|
as at least one requests is active the chunk will be ticked). */
|
||||||
void SetChunkAlwaysTicked(int a_ChunkX, int a_ChunkZ, bool a_AlwaysTicked);
|
void SetChunkAlwaysTicked(int a_ChunkX, int a_ChunkZ, bool a_AlwaysTicked);
|
||||||
|
|
||||||
|
/** Adds this chunkmap's CS to the DeadlockDetect's tracked CSs. */
|
||||||
|
void TrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect, const AString & a_WorldName);
|
||||||
|
|
||||||
|
/** Removes this chunkmap's CS from the DeadlockDetect's tracked CSs. */
|
||||||
|
void UntrackInDeadlockDetect(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
// The chunks can manipulate neighbors while in their Tick() method, using LockedGetBlock() and LockedSetBlock()
|
// The chunks can manipulate neighbors while in their Tick() method, using LockedGetBlock() and LockedSetBlock()
|
||||||
|
@ -30,6 +30,27 @@ cDeadlockDetect::cDeadlockDetect(void) :
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
cDeadlockDetect::~cDeadlockDetect()
|
||||||
|
{
|
||||||
|
// Check that all tracked CSs have been removed, report any remaining:
|
||||||
|
cCSLock lock(m_CS);
|
||||||
|
if (!m_TrackedCriticalSections.empty())
|
||||||
|
{
|
||||||
|
LOGWARNING("DeadlockDetect: Some CS objects (%u) haven't been removed from tracking", static_cast<unsigned>(m_TrackedCriticalSections.size()));
|
||||||
|
for (const auto & tcs: m_TrackedCriticalSections)
|
||||||
|
{
|
||||||
|
LOGWARNING(" CS %p / %s",
|
||||||
|
static_cast<void *>(tcs.first),
|
||||||
|
tcs.second.c_str()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
bool cDeadlockDetect::Start(int a_IntervalSec)
|
bool cDeadlockDetect::Start(int a_IntervalSec)
|
||||||
{
|
{
|
||||||
m_IntervalSec = a_IntervalSec;
|
m_IntervalSec = a_IntervalSec;
|
||||||
@ -61,6 +82,33 @@ bool cDeadlockDetect::Start(int a_IntervalSec)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::TrackCriticalSection(cCriticalSection & a_CS, const AString & a_Name)
|
||||||
|
{
|
||||||
|
cCSLock lock(m_CS);
|
||||||
|
m_TrackedCriticalSections.emplace_back(std::make_pair(&a_CS, a_Name));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::UntrackCriticalSection(cCriticalSection & a_CS)
|
||||||
|
{
|
||||||
|
cCSLock lock(m_CS);
|
||||||
|
for (auto itr = m_TrackedCriticalSections.begin(), end = m_TrackedCriticalSections.end(); itr != end; ++itr)
|
||||||
|
{
|
||||||
|
if (itr->first == &a_CS)
|
||||||
|
{
|
||||||
|
m_TrackedCriticalSections.erase(itr);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cDeadlockDetect::Execute(void)
|
void cDeadlockDetect::Execute(void)
|
||||||
{
|
{
|
||||||
// Loop until the signal to terminate:
|
// Loop until the signal to terminate:
|
||||||
@ -121,7 +169,7 @@ void cDeadlockDetect::CheckWorldAge(const AString & a_WorldName, Int64 a_Age)
|
|||||||
WorldAge.m_NumCyclesSame += 1;
|
WorldAge.m_NumCyclesSame += 1;
|
||||||
if (WorldAge.m_NumCyclesSame > (m_IntervalSec * 1000) / CYCLE_MILLISECONDS)
|
if (WorldAge.m_NumCyclesSame > (m_IntervalSec * 1000) / CYCLE_MILLISECONDS)
|
||||||
{
|
{
|
||||||
DeadlockDetected();
|
DeadlockDetected(a_WorldName, a_Age);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -135,9 +183,12 @@ void cDeadlockDetect::CheckWorldAge(const AString & a_WorldName, Int64 a_Age)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cDeadlockDetect::DeadlockDetected(void)
|
void cDeadlockDetect::DeadlockDetected(const AString & a_WorldName, Int64 a_WorldAge)
|
||||||
{
|
{
|
||||||
LOGERROR("Deadlock detected, aborting the server");
|
LOGERROR("Deadlock detected: world %s has been stuck at age %lld. Aborting the server.",
|
||||||
|
a_WorldName.c_str(), static_cast<long long>(a_WorldAge)
|
||||||
|
);
|
||||||
|
ListTrackedCSs();
|
||||||
ASSERT(!"Deadlock detected");
|
ASSERT(!"Deadlock detected");
|
||||||
abort();
|
abort();
|
||||||
}
|
}
|
||||||
@ -145,3 +196,19 @@ void cDeadlockDetect::DeadlockDetected(void)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::ListTrackedCSs(void)
|
||||||
|
{
|
||||||
|
cCSLock lock(m_CS);
|
||||||
|
for (const auto & cs: m_TrackedCriticalSections)
|
||||||
|
{
|
||||||
|
LOG("CS at %p, %s: RecursionCount = %d, ThreadIDHash = %04llx",
|
||||||
|
static_cast<void *>(cs.first), cs.second.c_str(),
|
||||||
|
cs.first->m_RecursionCount, static_cast<UInt64>(std::hash<std::thread::id>()(cs.first->m_OwningThreadID))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -27,10 +27,20 @@ class cDeadlockDetect :
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
cDeadlockDetect(void);
|
cDeadlockDetect(void);
|
||||||
|
~cDeadlockDetect();
|
||||||
|
|
||||||
/** Starts the detection. Hides cIsThread's Start, because we need some initialization */
|
/** Starts the detection. Hides cIsThread's Start, because we need some initialization */
|
||||||
bool Start(int a_IntervalSec);
|
bool Start(int a_IntervalSec);
|
||||||
|
|
||||||
|
/** Adds the critical section for tracking.
|
||||||
|
Tracked CSs are listed, together with ownership details, when a deadlock is detected.
|
||||||
|
A tracked CS must be untracked before it is destroyed.
|
||||||
|
a_Name is an arbitrary name that is listed along with the CS in the output. */
|
||||||
|
void TrackCriticalSection(cCriticalSection & a_CS, const AString & a_Name);
|
||||||
|
|
||||||
|
/** Removes the CS from the tracking. */
|
||||||
|
void UntrackCriticalSection(cCriticalSection & a_CS);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
struct sWorldAge
|
struct sWorldAge
|
||||||
{
|
{
|
||||||
@ -44,6 +54,13 @@ protected:
|
|||||||
/** Maps world name -> sWorldAge */
|
/** Maps world name -> sWorldAge */
|
||||||
typedef std::map<AString, sWorldAge> WorldAges;
|
typedef std::map<AString, sWorldAge> WorldAges;
|
||||||
|
|
||||||
|
/** Protects m_TrackedCriticalSections from multithreaded access. */
|
||||||
|
cCriticalSection m_CS;
|
||||||
|
|
||||||
|
/** CriticalSections that are tracked (their status output on deadlock).
|
||||||
|
Protected against multithreaded access by m_CS. */
|
||||||
|
std::vector<std::pair<cCriticalSection *, AString>> m_TrackedCriticalSections;
|
||||||
|
|
||||||
WorldAges m_WorldAges;
|
WorldAges m_WorldAges;
|
||||||
|
|
||||||
/** Number of secods for which the ages must be the same for the detection to trigger */
|
/** Number of secods for which the ages must be the same for the detection to trigger */
|
||||||
@ -59,8 +76,13 @@ protected:
|
|||||||
/** Checks if the world's age has changed, updates the world's stats; calls DeadlockDetected() if deadlock detected */
|
/** Checks if the world's age has changed, updates the world's stats; calls DeadlockDetected() if deadlock detected */
|
||||||
void CheckWorldAge(const AString & a_WorldName, Int64 a_Age);
|
void CheckWorldAge(const AString & a_WorldName, Int64 a_Age);
|
||||||
|
|
||||||
/** Called when a deadlock is detected. Aborts the server. */
|
/** Called when a deadlock is detected in a world. Aborts the server.
|
||||||
NORETURN void DeadlockDetected(void);
|
a_WorldName is the name of the world whose age has triggered the detection.
|
||||||
|
a_WorldAge is the age (in ticks) in which the world is stuck. */
|
||||||
|
NORETURN void DeadlockDetected(const AString & a_WorldName, Int64 a_WorldAge);
|
||||||
|
|
||||||
|
/** Outputs a listing of the tracked CSs, together with their name and state. */
|
||||||
|
void ListTrackedCSs();
|
||||||
} ;
|
} ;
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,12 +9,10 @@
|
|||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
// cCriticalSection:
|
// cCriticalSection:
|
||||||
|
|
||||||
#ifdef _DEBUG
|
cCriticalSection::cCriticalSection():
|
||||||
cCriticalSection::cCriticalSection()
|
m_RecursionCount(0)
|
||||||
{
|
{
|
||||||
m_IsLocked = 0;
|
|
||||||
}
|
}
|
||||||
#endif // _DEBUG
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -24,10 +22,8 @@ void cCriticalSection::Lock()
|
|||||||
{
|
{
|
||||||
m_Mutex.lock();
|
m_Mutex.lock();
|
||||||
|
|
||||||
#ifdef _DEBUG
|
m_RecursionCount += 1;
|
||||||
m_IsLocked += 1;
|
|
||||||
m_OwningThreadID = std::this_thread::get_id();
|
m_OwningThreadID = std::this_thread::get_id();
|
||||||
#endif // _DEBUG
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -36,10 +32,8 @@ void cCriticalSection::Lock()
|
|||||||
|
|
||||||
void cCriticalSection::Unlock()
|
void cCriticalSection::Unlock()
|
||||||
{
|
{
|
||||||
#ifdef _DEBUG
|
ASSERT(IsLockedByCurrentThread());
|
||||||
ASSERT(m_IsLocked > 0);
|
m_RecursionCount -= 1;
|
||||||
m_IsLocked -= 1;
|
|
||||||
#endif // _DEBUG
|
|
||||||
|
|
||||||
m_Mutex.unlock();
|
m_Mutex.unlock();
|
||||||
}
|
}
|
||||||
@ -48,10 +42,9 @@ void cCriticalSection::Unlock()
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
#ifdef _DEBUG
|
|
||||||
bool cCriticalSection::IsLocked(void)
|
bool cCriticalSection::IsLocked(void)
|
||||||
{
|
{
|
||||||
return (m_IsLocked > 0);
|
return (m_RecursionCount > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -60,9 +53,8 @@ bool cCriticalSection::IsLocked(void)
|
|||||||
|
|
||||||
bool cCriticalSection::IsLockedByCurrentThread(void)
|
bool cCriticalSection::IsLockedByCurrentThread(void)
|
||||||
{
|
{
|
||||||
return ((m_IsLocked > 0) && (m_OwningThreadID == std::this_thread::get_id()));
|
return ((m_RecursionCount > 0) && (m_OwningThreadID == std::this_thread::get_id()));
|
||||||
}
|
}
|
||||||
#endif // _DEBUG
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,27 +9,40 @@
|
|||||||
|
|
||||||
class cCriticalSection
|
class cCriticalSection
|
||||||
{
|
{
|
||||||
|
friend class cDeadlockDetect; // Allow the DeadlockDetect to read the internals, so that it may output some statistics
|
||||||
|
|
||||||
public:
|
public:
|
||||||
void Lock(void);
|
void Lock(void);
|
||||||
void Unlock(void);
|
void Unlock(void);
|
||||||
|
|
||||||
// IsLocked / IsLockedByCurrentThread are only used in ASSERT statements, but because of the changes with ASSERT they must always be defined
|
|
||||||
// The fake versions (in Release) will not effect the program in any way
|
|
||||||
#ifdef _DEBUG
|
|
||||||
cCriticalSection(void);
|
cCriticalSection(void);
|
||||||
|
|
||||||
|
/** Returns true if the CS is currently locked.
|
||||||
|
Note that since it relies on the m_RecursionCount value, it is inherently thread-unsafe, prone to false positives.
|
||||||
|
Also, due to multithreading, the state can change between this when function is evaluated and the returned value is used.
|
||||||
|
To be used in ASSERT(IsLocked()) only. */
|
||||||
bool IsLocked(void);
|
bool IsLocked(void);
|
||||||
|
|
||||||
|
/** Returns true if the CS is currently locked by the thread calling this function.
|
||||||
|
Note that since it relies on the m_RecursionCount value, it is inherently thread-unsafe, prone to false positives.
|
||||||
|
Also, due to multithreading, the state can change between this when function is evaluated and the returned value is used.
|
||||||
|
To be used in ASSERT(IsLockedByCurrentThread()) only. */
|
||||||
bool IsLockedByCurrentThread(void);
|
bool IsLockedByCurrentThread(void);
|
||||||
#else
|
|
||||||
bool IsLocked(void) { return false; }
|
|
||||||
bool IsLockedByCurrentThread(void) { return false; }
|
|
||||||
#endif // _DEBUG
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
#ifdef _DEBUG
|
/** Number of times that this CS is currently locked (levels of recursion). Zero if not locked.
|
||||||
int m_IsLocked; // Number of times this CS is locked
|
Note that this value should be considered true only when the CS is locked; without the lock, it is UndefinedBehavior to even read it,
|
||||||
|
but making it std::atomic would impose too much of a runtime penalty.
|
||||||
|
It is only ever read without the lock in the DeadlockDetect, where the server is terminating anyway. */
|
||||||
|
int m_RecursionCount;
|
||||||
|
|
||||||
|
/** ID of the thread that is currently holding the CS.
|
||||||
|
Note that this value should be considered true only when the CS is locked; without the lock, it is UndefinedBehavior to even read it,
|
||||||
|
but making it std::atomic would impose too much of a runtime penalty.
|
||||||
|
When unlocked, the value stored here has no meaning, it may be an ID of a previous holder, or it could be any garbage.
|
||||||
|
It is only ever read without the lock in the DeadlockDetect, where the server is terminating anyway. */
|
||||||
std::thread::id m_OwningThreadID;
|
std::thread::id m_OwningThreadID;
|
||||||
#endif // _DEBUG
|
|
||||||
|
|
||||||
std::recursive_mutex m_Mutex;
|
std::recursive_mutex m_Mutex;
|
||||||
} ALIGN_8;
|
} ALIGN_8;
|
||||||
|
14
src/Root.cpp
14
src/Root.cpp
@ -182,7 +182,7 @@ void cRoot::Start(std::unique_ptr<cSettingsRepositoryInterface> a_OverridesRepo)
|
|||||||
LoadWorlds(*settingsRepo, IsNewIniFile);
|
LoadWorlds(*settingsRepo, IsNewIniFile);
|
||||||
|
|
||||||
LOGD("Loading plugin manager...");
|
LOGD("Loading plugin manager...");
|
||||||
m_PluginManager = new cPluginManager();
|
m_PluginManager = new cPluginManager(dd);
|
||||||
m_PluginManager->ReloadPluginsNow(*settingsRepo);
|
m_PluginManager->ReloadPluginsNow(*settingsRepo);
|
||||||
|
|
||||||
LOGD("Loading MonsterConfig...");
|
LOGD("Loading MonsterConfig...");
|
||||||
@ -193,7 +193,7 @@ void cRoot::Start(std::unique_ptr<cSettingsRepositoryInterface> a_OverridesRepo)
|
|||||||
m_Authenticator.Start(*settingsRepo);
|
m_Authenticator.Start(*settingsRepo);
|
||||||
|
|
||||||
LOGD("Starting worlds...");
|
LOGD("Starting worlds...");
|
||||||
StartWorlds();
|
StartWorlds(dd);
|
||||||
|
|
||||||
if (settingsRepo->GetValueSetB("DeadlockDetect", "Enabled", true))
|
if (settingsRepo->GetValueSetB("DeadlockDetect", "Enabled", true))
|
||||||
{
|
{
|
||||||
@ -248,7 +248,7 @@ void cRoot::Start(std::unique_ptr<cSettingsRepositoryInterface> a_OverridesRepo)
|
|||||||
dd.Stop();
|
dd.Stop();
|
||||||
|
|
||||||
LOGD("Stopping world threads...");
|
LOGD("Stopping world threads...");
|
||||||
StopWorlds();
|
StopWorlds(dd);
|
||||||
|
|
||||||
LOGD("Stopping authenticator...");
|
LOGD("Stopping authenticator...");
|
||||||
m_Authenticator.Stop();
|
m_Authenticator.Stop();
|
||||||
@ -486,11 +486,11 @@ void cRoot::LoadWorlds(cSettingsRepositoryInterface & a_Settings, bool a_IsNewIn
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cRoot::StartWorlds(void)
|
void cRoot::StartWorlds(cDeadlockDetect & a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
for (WorldMap::iterator itr = m_WorldsByName.begin(); itr != m_WorldsByName.end(); ++itr)
|
for (WorldMap::iterator itr = m_WorldsByName.begin(); itr != m_WorldsByName.end(); ++itr)
|
||||||
{
|
{
|
||||||
itr->second->Start();
|
itr->second->Start(a_DeadlockDetect);
|
||||||
itr->second->InitializeSpawn();
|
itr->second->InitializeSpawn();
|
||||||
m_PluginManager->CallHookWorldStarted(*itr->second);
|
m_PluginManager->CallHookWorldStarted(*itr->second);
|
||||||
}
|
}
|
||||||
@ -500,11 +500,11 @@ void cRoot::StartWorlds(void)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cRoot::StopWorlds(void)
|
void cRoot::StopWorlds(cDeadlockDetect & a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
for (WorldMap::iterator itr = m_WorldsByName.begin(); itr != m_WorldsByName.end(); ++itr)
|
for (WorldMap::iterator itr = m_WorldsByName.begin(); itr != m_WorldsByName.end(); ++itr)
|
||||||
{
|
{
|
||||||
itr->second->Stop();
|
itr->second->Stop(a_DeadlockDetect);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -27,6 +27,7 @@ class cPlayer;
|
|||||||
class cCommandOutputCallback;
|
class cCommandOutputCallback;
|
||||||
class cCompositeChat;
|
class cCompositeChat;
|
||||||
class cSettingsRepositoryInterface;
|
class cSettingsRepositoryInterface;
|
||||||
|
class cDeadlockDetect;
|
||||||
|
|
||||||
typedef cItemCallback<cPlayer> cPlayerListCallback;
|
typedef cItemCallback<cPlayer> cPlayerListCallback;
|
||||||
typedef cItemCallback<cWorld> cWorldListCallback;
|
typedef cItemCallback<cWorld> cWorldListCallback;
|
||||||
@ -226,10 +227,10 @@ private:
|
|||||||
void LoadWorlds(cSettingsRepositoryInterface & a_Settings, bool a_IsNewIniFile);
|
void LoadWorlds(cSettingsRepositoryInterface & a_Settings, bool a_IsNewIniFile);
|
||||||
|
|
||||||
/** Starts each world's life */
|
/** Starts each world's life */
|
||||||
void StartWorlds(void);
|
void StartWorlds(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
/** Stops each world's threads, so that it's safe to unload them */
|
/** Stops each world's threads, so that it's safe to unload them */
|
||||||
void StopWorlds(void);
|
void StopWorlds(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
/** Unloads all worlds from memory */
|
/** Unloads all worlds from memory */
|
||||||
void UnloadWorlds(void);
|
void UnloadWorlds(void);
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
#include "ChunkMap.h"
|
#include "ChunkMap.h"
|
||||||
#include "Generating/ChunkDesc.h"
|
#include "Generating/ChunkDesc.h"
|
||||||
#include "SetChunkData.h"
|
#include "SetChunkData.h"
|
||||||
|
#include "DeadlockDetect.h"
|
||||||
|
|
||||||
// Serializers
|
// Serializers
|
||||||
#include "WorldStorage/ScoreboardSerializer.h"
|
#include "WorldStorage/ScoreboardSerializer.h"
|
||||||
@ -433,8 +434,13 @@ void cWorld::InitializeSpawn(void)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cWorld::Start(void)
|
void cWorld::Start(cDeadlockDetect & a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
|
// Track the CSs used by this world in the deadlock detector:
|
||||||
|
a_DeadlockDetect.TrackCriticalSection(m_CSClients, Printf("World %s clients", m_WorldName.c_str()));
|
||||||
|
a_DeadlockDetect.TrackCriticalSection(m_CSPlayers, Printf("World %s players", m_WorldName.c_str()));
|
||||||
|
a_DeadlockDetect.TrackCriticalSection(m_CSTasks, Printf("World %s tasks", m_WorldName.c_str()));
|
||||||
|
|
||||||
m_SpawnX = 0;
|
m_SpawnX = 0;
|
||||||
m_SpawnY = cChunkDef::Height;
|
m_SpawnY = cChunkDef::Height;
|
||||||
m_SpawnZ = 0;
|
m_SpawnZ = 0;
|
||||||
@ -603,6 +609,7 @@ void cWorld::Start(void)
|
|||||||
SetTimeOfDay(IniFile.GetValueSetI("General", "TimeInTicks", GetTimeOfDay()));
|
SetTimeOfDay(IniFile.GetValueSetI("General", "TimeInTicks", GetTimeOfDay()));
|
||||||
|
|
||||||
m_ChunkMap = cpp14::make_unique<cChunkMap>(this);
|
m_ChunkMap = cpp14::make_unique<cChunkMap>(this);
|
||||||
|
m_ChunkMap->TrackInDeadlockDetect(a_DeadlockDetect, m_WorldName);
|
||||||
|
|
||||||
// preallocate some memory for ticking blocks so we don't need to allocate that often
|
// preallocate some memory for ticking blocks so we don't need to allocate that often
|
||||||
m_BlockTickQueue.reserve(1000);
|
m_BlockTickQueue.reserve(1000);
|
||||||
@ -953,7 +960,7 @@ void cWorld::InitialiseAndLoadMobSpawningValues(cIniFile & a_IniFile)
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
void cWorld::Stop(void)
|
void cWorld::Stop(cDeadlockDetect & a_DeadlockDetect)
|
||||||
{
|
{
|
||||||
// Delete the clients that have been in this world:
|
// Delete the clients that have been in this world:
|
||||||
{
|
{
|
||||||
@ -990,6 +997,11 @@ void cWorld::Stop(void)
|
|||||||
m_Generator.Stop();
|
m_Generator.Stop();
|
||||||
m_ChunkSender.Stop();
|
m_ChunkSender.Stop();
|
||||||
m_Storage.Stop();
|
m_Storage.Stop();
|
||||||
|
|
||||||
|
a_DeadlockDetect.UntrackCriticalSection(m_CSClients);
|
||||||
|
a_DeadlockDetect.UntrackCriticalSection(m_CSPlayers);
|
||||||
|
a_DeadlockDetect.UntrackCriticalSection(m_CSTasks);
|
||||||
|
m_ChunkMap->UntrackInDeadlockDetect(a_DeadlockDetect);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
12
src/World.h
12
src/World.h
@ -58,7 +58,7 @@ class cCompositeChat;
|
|||||||
class cCuboid;
|
class cCuboid;
|
||||||
class cSetChunkData;
|
class cSetChunkData;
|
||||||
class cBroadcaster;
|
class cBroadcaster;
|
||||||
|
class cDeadlockDetect;
|
||||||
|
|
||||||
typedef std::list< cPlayer * > cPlayerList;
|
typedef std::list< cPlayer * > cPlayerList;
|
||||||
typedef std::list< std::pair< cPlayer *, cWorld * > > cAwaitingPlayerList;
|
typedef std::list< std::pair< cPlayer *, cWorld * > > cAwaitingPlayerList;
|
||||||
@ -703,11 +703,13 @@ public:
|
|||||||
|
|
||||||
void InitializeSpawn(void);
|
void InitializeSpawn(void);
|
||||||
|
|
||||||
/** Starts threads that belong to this world */
|
/** Starts threads that belong to this world.
|
||||||
void Start(void);
|
a_DeadlockDetect is used for tracking this world's age, detecting a possible deadlock. */
|
||||||
|
void Start(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
/** Stops threads that belong to this world (part of deinit) */
|
/** Stops threads that belong to this world (part of deinit).
|
||||||
void Stop(void);
|
a_DeadlockDetect is used for tracking this world's age, detecting a possible deadlock. */
|
||||||
|
void Stop(cDeadlockDetect & a_DeadlockDetect);
|
||||||
|
|
||||||
/** Processes the blocks queued for ticking with a delay (m_BlockTickQueue[]) */
|
/** Processes the blocks queued for ticking with a delay (m_BlockTickQueue[]) */
|
||||||
void TickQueuedBlocks(void);
|
void TickQueuedBlocks(void);
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
#include "BlockEntities/BlockEntity.h"
|
#include "BlockEntities/BlockEntity.h"
|
||||||
#include "Blocks/BlockHandler.h"
|
#include "Blocks/BlockHandler.h"
|
||||||
#include "Generating/ChunkDesc.h"
|
#include "Generating/ChunkDesc.h"
|
||||||
|
#include "DeadlockDetect.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -271,3 +272,19 @@ cBlockEntity * cBlockEntity::CreateByBlockType(BLOCKTYPE a_BlockType, NIBBLETYPE
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::TrackCriticalSection(cCriticalSection & a_CS, const AString & a_Name)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::UntrackCriticalSection(cCriticalSection & a_CS)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
#include "BlockEntities/BlockEntity.h"
|
#include "BlockEntities/BlockEntity.h"
|
||||||
#include "Blocks/BlockHandler.h"
|
#include "Blocks/BlockHandler.h"
|
||||||
#include "Generating/ChunkDesc.h"
|
#include "Generating/ChunkDesc.h"
|
||||||
|
#include "DeadlockDetect.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -271,3 +272,19 @@ cBlockEntity * cBlockEntity::CreateByBlockType(BLOCKTYPE a_BlockType, NIBBLETYPE
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::TrackCriticalSection(cCriticalSection & a_CS, const AString & a_Name)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
void cDeadlockDetect::UntrackCriticalSection(cCriticalSection & a_CS)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user