flewkey/cuberite-2a
1
0
Fork 0
Code Releases Activity

AnvilStats: initial project import, can do block and biome statistics

Browse Source 
git-svn-id: http://mc-server.googlecode.com/svn/trunk@895 0a769ca7-a7f5-676a-18bf-c427514a06d6
This commit is contained in:
madmaxoft@gmail.com 2012-09-27 20:02:25 +00:00
parent 6c7e123373
commit 6ccc4e3674
11 changed files with 1512 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
AnvilStats/AnvilStats.cpp Normal file
View File

@ -0,0 +1,53 @@
// AnvilStats.cpp
// Implements the main app entrypoint
#include "Globals.h"
#include "Statistics.h"
#include "Processor.h"
int main(int argc, char * argv[])
{
if (argc < 2)
{
LOG("Usage: %s <method number> [<world folder>]", argv[0]);
LOG("\nNo method number present, aborting.");
return -1;
}
AString WorldFolder;
if (argc > 2)
{
WorldFolder = argv[2];
}
else
{
WorldFolder = "." + cFile::PathSeparator;
}
cCallbackFactory * Factory = NULL;
switch (atol(argv[1]))
{
case 0: Factory = new cStatisticsFactory; break;
default:
{
LOG("Unknown method \"%s\", aborting.", argv[1]);
return -2;
}
}
cProcessor Processor;
Processor.ProcessWorld(WorldFolder, *Factory);
delete Factory;
LOG("Done");
}

34
AnvilStats/AnvilStats.sln Normal file
View File

@ -0,0 +1,34 @@

Microsoft Visual Studio Solution File, Format Version 10.00
# Visual C++ Express 2008
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "AnvilStats", "AnvilStats.vcproj", "{CF996A5E-0A86-4004-9710-682B06B5AEBA}"
ProjectSection(ProjectDependencies) = postProject
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA} = {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "..\VC2008\zlib.vcproj", "{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Release profiled|Win32 = Release profiled|Win32
Release|Win32 = Release|Win32
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Debug|Win32.ActiveCfg = Debug|Win32
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Debug|Win32.Build.0 = Debug|Win32
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release profiled|Win32.ActiveCfg = Release|Win32
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release profiled|Win32.Build.0 = Release|Win32
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release|Win32.ActiveCfg = Release|Win32
{CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release|Win32.Build.0 = Release|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Debug|Win32.ActiveCfg = Debug|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Debug|Win32.Build.0 = Debug|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release profiled|Win32.ActiveCfg = Release profiled|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release profiled|Win32.Build.0 = Release profiled|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release|Win32.ActiveCfg = Release|Win32
{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release|Win32.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

27
AnvilStats/AnvilStats.txt Normal file
View File

@ -0,0 +1,27 @@
// AnvilStats.txt
// A Readme for the project
/*
AnvilStats
==========
This is a project for measuring various metrics throughout an Anvil world, presumably created by a vanilla MC.
It works by parsing the MCA files in the path specified as its param (or current directory, if no params) and
feeding each decompressed chunk into the statistics-gathering callback function.
Possible usage:
- count the per-chunk density of specific blocks
- count the per-chunk density of dungeons, by measuring the number of zombie/skeleton/regularspider spawners
- count the per-chunk-per-biome density of trees, by measuring the number of dirt-log vertical transitions, correlating to biome data
This project is Windows-only, although it shouldn't be too difficult to make it portable.
*/

284
AnvilStats/AnvilStats.vcproj Normal file
View File

@ -0,0 +1,284 @@
<?xml version="1.0" encoding="windows-1250"?>
<VisualStudioProject
ProjectType="Visual C++"
Version="9,00"
Name="AnvilStats"
ProjectGUID="{CF996A5E-0A86-4004-9710-682B06B5AEBA}"
RootNamespace="AnvilStats"
Keyword="Win32Proj"
TargetFrameworkVersion="196613"
>
<Platforms>
<Platform
Name="Win32"
/>
</Platforms>
<ToolFiles>
</ToolFiles>
<Configurations>
<Configuration
Name="Debug|Win32"
OutputDirectory="$(SolutionDir)$(ConfigurationName)"
IntermediateDirectory="$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="&quot;..\zlib-1.2.7&quot;"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="1"
UsePrecompiledHeader="2"
PrecompiledHeaderThrough="Globals.h"
WarningLevel="3"
DebugInformationFormat="4"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="ws2_32.lib"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
StackReserveSize="16777216"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
<Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)$(ConfigurationName)"
IntermediateDirectory="$(ConfigurationName)"
ConfigurationType="1"
CharacterSet="2"
WholeProgramOptimization="1"
>
<Tool
Name="VCPreBuildEventTool"
/>
<Tool
Name="VCCustomBuildTool"
/>
<Tool
Name="VCXMLDataGeneratorTool"
/>
<Tool
Name="VCWebServiceProxyGeneratorTool"
/>
<Tool
Name="VCMIDLTool"
/>
<Tool
Name="VCCLCompilerTool"
Optimization="2"
EnableIntrinsicFunctions="true"
AdditionalIncludeDirectories="&quot;..\zlib-1.2.7&quot;"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="2"
PrecompiledHeaderThrough="Globals.h"
WarningLevel="3"
DebugInformationFormat="3"
/>
<Tool
Name="VCManagedResourceCompilerTool"
/>
<Tool
Name="VCResourceCompilerTool"
/>
<Tool
Name="VCPreLinkEventTool"
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="ws2_32.lib"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
OptimizeReferences="2"
EnableCOMDATFolding="2"
TargetMachine="1"
/>
<Tool
Name="VCALinkTool"
/>
<Tool
Name="VCManifestTool"
/>
<Tool
Name="VCXDCMakeTool"
/>
<Tool
Name="VCBscMakeTool"
/>
<Tool
Name="VCFxCopTool"
/>
<Tool
Name="VCAppVerifierTool"
/>
<Tool
Name="VCPostBuildEventTool"
/>
</Configuration>
</Configurations>
<References>
</References>
<Files>
<Filter
Name="Source Files"
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath=".\AnvilStats.cpp"
>
</File>
<File
RelativePath=".\Callback.h"
>
</File>
<File
RelativePath=".\Globals.cpp"
>
<FileConfiguration
Name="Debug|Win32"
>
<Tool
Name="VCCLCompilerTool"
UsePrecompiledHeader="1"
/>
</FileConfiguration>
<FileConfiguration
Name="Release|Win32"
>
<Tool
Name="VCCLCompilerTool"
UsePrecompiledHeader="1"
/>
</FileConfiguration>
</File>
<File
RelativePath=".\Globals.h"
>
</File>
<File
RelativePath=".\Processor.cpp"
>
</File>
<File
RelativePath=".\Processor.h"
>
</File>
<File
RelativePath=".\Statistics.cpp"
>
</File>
<File
RelativePath=".\Statistics.h"
>
</File>
</Filter>
<Filter
Name="shared"
>
<File
RelativePath="..\source\OSSupport\CriticalSection.cpp"
>
</File>
<File
RelativePath="..\source\OSSupport\CriticalSection.h"
>
</File>
<File
RelativePath="..\source\Endianness.h"
>
</File>
<File
RelativePath="..\source\WorldStorage\FastNBT.cpp"
>
</File>
<File
RelativePath="..\source\WorldStorage\FastNBT.h"
>
</File>
<File
RelativePath="..\source\OSSupport\File.cpp"
>
</File>
<File
RelativePath="..\source\OSSupport\File.h"
>
</File>
<File
RelativePath="..\source\OSSupport\IsThread.cpp"
>
</File>
<File
RelativePath="..\source\OSSupport\IsThread.h"
>
</File>
<File
RelativePath="..\source\StringUtils.cpp"
>
</File>
<File
RelativePath="..\source\StringUtils.h"
>
</File>
</Filter>
<File
RelativePath=".\AnvilStats.txt"
>
</File>
</Files>
<Globals>
</Globals>
</VisualStudioProject>

108
AnvilStats/Callback.h Normal file
View File

@ -0,0 +1,108 @@
// Callback.h
// Interfaces to the cCallback base class used as the base class for all statistical callbacks
#pragma once
/** The base class for all chunk-processor callbacks, declares the interface.
The processor calls each virtual function in the order they are declared here with the specified args.
If the function returns true, the processor moves on to next chunk and starts calling the callbacks again from start with
the new chunk data.
So if a statistics collector doesn't need data decompression at all, it can stop the processor from doing so early-enough
and still get meaningful data.
A callback is guaranteed to run in a single thread and always the same thread.
A callback is guaranteed to run on all chunks in a region and one region is guaranteed to be handled by only callback.
*/
class cCallback abstract
{
public:
virtual ~cCallback() {} // Force a virtual destructor in each descendant
/// Called to inform the stats module of the chunk coords for newly processing chunk
virtual bool OnNewChunk(int a_ChunkX, int a_ChunkZ) = 0;
/// Called to inform about the chunk's data offset in the file (chunk mini-header), the number of sectors it uses and the timestamp field value
virtual bool OnHeader(int a_FileOffset, unsigned char a_NumSectors, int a_Timestamp) { return true; }
/// Called to inform of the compressed chunk data size and position in the file (offset from file start to the actual data)
virtual bool OnCompressedDataSizePos(int a_CompressedDataSize, int a_DataOffset, char a_CompressionMethod) { return true; }
/// Just in case you wanted to process the NBT yourself ;)
virtual bool OnDecompressedData(const char * a_DecompressedNBT, int a_DataSize) { return true; }
/// The chunk's NBT should specify chunk coords, these are sent here:
virtual bool OnRealCoords(int a_ChunkX, int a_ChunkZ) { return true; }
/// The chunk contains a LastUpdate value specifying the last tick in which it was saved.
virtual bool OnLastUpdate(Int64 a_LastUpdate) { return true; }
virtual bool OnTerrainPopulated(bool a_Populated) { return true; }
virtual bool OnBiomes(const unsigned char * a_BiomeData) { return true; }
virtual bool OnHeightMap(const int * a_HeightMap) { return true; }
virtual bool OnSection(
unsigned char a_Y,
const BLOCKTYPE * a_BlockTypes,
const NIBBLETYPE * a_BlockAdditional,
const NIBBLETYPE * a_BlockMeta,
const NIBBLETYPE * a_BlockLight,
const NIBBLETYPE * a_BlockSkyLight
) { return true; }
// TODO: entities, tile-entities, tile-ticks
} ;
typedef std::vector<cCallback *> cCallbacks;
/** The base class for a factory that creates callback objects for separate threads.
The processor creates a callback for each thread on which it runs using this factory.
The factory is guaranteed to be called from a single thread.
The factory keeps track of all the callbacks that it has created and deletes them when destructed
*/
class cCallbackFactory
{
public:
virtual ~cCallbackFactory()
{
for (cCallbacks::iterator itr = m_Callbacks.begin(), end = m_Callbacks.end(); itr != end; ++itr)
{
delete *itr;
}
}
/// Descendants override this method to return the correct callback type
virtual cCallback * CreateNewCallback(void) = 0;
/// cProcessor uses this method to request a new callback
cCallback * GetNewCallback(void)
{
cCallback * Callback = CreateNewCallback();
if (Callback != NULL)
{
m_Callbacks.push_back(Callback);
}
return Callback;
}
protected:
cCallbacks m_Callbacks;
} ;

10
AnvilStats/Globals.cpp Normal file
View File

@ -0,0 +1,10 @@
// Globals.cpp
// This file is used for precompiled header generation in MSVC environments
#include "Globals.h"

220
AnvilStats/Globals.h Normal file
View File

@ -0,0 +1,220 @@
// Globals.h
// This file gets included from every module in the project, so that global symbols may be introduced easily
// Also used for precompiled header generation in MSVC environments
// Compiler-dependent stuff:
#if defined(_MSC_VER)
// MSVC produces warning C4481 on the override keyword usage, so disable the warning altogether
#pragma warning(disable:4481)
// Disable some warnings that we don't care about:
#pragma warning(disable:4100)
#define OBSOLETE __declspec(deprecated)
// No alignment needed in MSVC
#define ALIGN_8
#define ALIGN_16
#elif defined(__GNUC__)
// TODO: Can GCC explicitly mark classes as abstract (no instances can be created)?
#define abstract
// TODO: Can GCC mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
#define override
#define OBSOLETE __attribute__((deprecated))
#define ALIGN_8 __attribute__((aligned(8)))
#define ALIGN_16 __attribute__((aligned(16)))
// Some portability macros :)
#define stricmp strcasecmp
#else
#error "You are using an unsupported compiler, you might need to #define some stuff here for your compiler"
/*
// Copy and uncomment this into another #elif section based on your compiler identification
// Explicitly mark classes as abstract (no instances can be created)
#define abstract
// Mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
#define override
// Mark functions as obsolete, so that their usage results in a compile-time warning
#define OBSOLETE
// Mark types / variables for alignment. Do the platforms need it?
#define ALIGN_8
#define ALIGN_16
*/
#endif
// Integral types with predefined sizes:
typedef long long Int64;
typedef int Int32;
typedef short Int16;
// A macro to disallow the copy constructor and operator= functions
// This should be used in the private: declarations for any class that shouldn't allow copying itself
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName &); \
void operator=(const TypeName &)
// A macro that is used to mark unused function parameters, to avoid pedantic warnings in gcc
#define UNUSED(X) (void)(X)
// OS-dependent stuff:
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <winsock2.h>
// Windows SDK defines min and max macros, messing up with our std::min and std::max usage
#undef min
#undef max
// Windows SDK defines GetFreeSpace as a constant, probably a Win16 API remnant
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif // GetFreeSpace
#else
#include <sys/types.h>
#include <sys/stat.h> // for mkdir
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <time.h>
#include <dirent.h>
#include <errno.h>
#include <iostream>
#include <cstdio>
#include <cstring>
#include <pthread.h>
#include <semaphore.h>
#include <errno.h>
#include <fcntl.h>
#endif
#define FILE_IO_PREFIX ""
// CRT stuff:
#include <assert.h>
#include <stdio.h>
#include <math.h>
#include <stdarg.h>
// STL stuff:
#include <vector>
#include <list>
#include <deque>
#include <string>
#include <map>
#include <algorithm>
#include <memory>
// Common headers (part 1, without macros):
#include "../source/StringUtils.h"
#include "../source/OSSupport/CriticalSection.h"
#include "../source/OSSupport/Semaphore.h"
#include "../source/OSSupport/Event.h"
#include "../source/OSSupport/IsThread.h"
#include "../source/OSSupport/File.h"
// Common definitions:
#define LOG(x,...) printf(x "\n", __VA_ARGS__)
#define LOGERROR LOG
#define LOGWARNING LOG
#define LOGINFO LOG
/// Evaluates to the number of elements in an array (compile-time!)
#define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X)))
/// Allows arithmetic expressions like "32 KiB" (but consider using parenthesis around it, "(32 KiB)" )
#define KiB * 1024
/// Allows arithmetic expressions like "32 MiB" (but consider using parenthesis around it, "(32 MiB)" )
#define MiB * 1024 * 1024
/// Faster than (int)floorf((float)x / (float)div)
#define FAST_FLOOR_DIV( x, div ) ( (x) < 0 ? (((int)x / div) - 1) : ((int)x / div) )
// Own version of assert() that writes failed assertions to the log for review
#ifdef _DEBUG
#define ASSERT( x ) ( !!(x) || ( LOGERROR("Assertion failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), assert(0), 0 ) )
#else
#define ASSERT(x) ((void)0)
#endif
// Pretty much the same as ASSERT() but stays in Release builds
#define VERIFY( x ) ( !!(x) || ( LOGERROR("Verification failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), exit(1), 0 ) )
/// A generic interface used mainly in ForEach() functions
template <typename Type> class cItemCallback
{
public:
/// Called for each item in the internal list; return true to stop the loop, or false to continue enumerating
virtual bool Item(Type * a_Type) = 0;
} ;
// Common headers (part 2, with macros):
#include "../source/ChunkDef.h"
#include "../source/BlockID.h"

407
AnvilStats/Processor.cpp Normal file
View File

@ -0,0 +1,407 @@
// Processor.cpp
// Implements the cProcessor class representing the overall processor engine that manages threads, calls callbacks etc.
#include "Globals.h"
#include "Processor.h"
#include "Callback.h"
#include "../source/WorldStorage/FastNBT.h"
#include "zlib.h"
const int CHUNK_INFLATE_MAX = 1 MiB;
const int MAX_COMPRESSED_CHUNK_SIZE = 1 MiB;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cProcessor::cThread:
cProcessor::cThread::cThread(cCallback & a_Callback, cProcessor & a_ParentProcessor) :
super("cProcessor::cThread"),
m_Callback(a_Callback),
m_ParentProcessor(a_ParentProcessor)
{
super::Start();
}
void cProcessor::cThread::Execute(void)
{
for (;;)
{
AString FileName = m_ParentProcessor.GetOneFileName();
if (FileName.empty())
{
// All done, terminate the thread
return;
}
ProcessFile(FileName);
} // for-ever
}
void cProcessor::cThread::ProcessFile(const AString & a_FileName)
{
LOG("Processing file \"%s\"", a_FileName.c_str());
size_t idx = a_FileName.rfind("r.");
if (idx == AString::npos)
{
LOG("Cannot parse filename \"%s\", skipping file.", a_FileName.c_str());
return;
}
int RegionX = 0, RegionZ = 0;
if (sscanf_s(a_FileName.c_str() + idx, "r.%d.%d.mca", &RegionX, &RegionZ) != 2)
{
LOG("Cannot parse filename \"%s\" into coords, skipping file.", a_FileName.c_str());
return;
}
cFile f;
if (!f.Open(a_FileName, cFile::fmRead))
{
LOG("Cannot open file \"%s\", skipping file.", a_FileName.c_str());
return;
}
int Header[2048];
if (f.Read(Header, sizeof(Header)) != sizeof(Header))
{
LOG("Cannot read header in file \"%s\", skipping file.", a_FileName.c_str());
return;
}
for (int i = 0; i < ARRAYCOUNT(Header); i++)
{
Header[i] = ntohl(Header[i]);
}
int ChunkBaseX = RegionX * 32;
int ChunkBaseZ = RegionZ * 32;
for (int i = 0; i < 1024; i++)
{
unsigned Location = Header[i];
unsigned Timestamp = Header[i + 1024];
if (
((Location == 0) && (Timestamp == 0)) || // Official docs' "not present"
(Location >> 8 < 2) || // Logical - no chunk can start inside the header
((Location & 0xff) == 0) // Logical - no chunk can be zero bytes
)
{
// Chunk not present in the file
continue;
}
int ChunkX = ChunkBaseX + (i % 32);
int ChunkZ = ChunkBaseZ + (i / 32);
if (m_Callback.OnNewChunk(ChunkX, ChunkZ))
{
continue;
}
ProcessChunk(f, ChunkX, ChunkZ, Location >> 8, Location & 0xff, Timestamp);
} // for i - chunk index
}
void cProcessor::cThread::ProcessChunk(cFile & a_File, int a_ChunkX, int a_ChunkZ, unsigned a_SectorStart, unsigned a_SectorSize, unsigned a_TimeStamp)
{
if (m_Callback.OnHeader(a_SectorStart * 4096, a_SectorSize, a_TimeStamp))
{
return;
}
if (a_File.Seek(a_SectorStart * 4096) < 0)
{
LOG("Seeking to sector %d failed, skipping chunk [%d, %d]", a_SectorStart, a_ChunkX, a_ChunkZ);
return;
}
int ByteSize;
if (a_File.Read(&ByteSize, sizeof(ByteSize)) != sizeof(ByteSize))
{
LOG("Cannot read bytesize at offset %d, skipping chunk [%d, %d].", a_SectorStart * 4096, a_ChunkX, a_ChunkZ);
return;
}
ByteSize = ntohl(ByteSize);
char CompressionMethod;
if (a_File.Read(&CompressionMethod, sizeof(CompressionMethod)) != sizeof(CompressionMethod))
{
LOG("Cannot read CompressionMethod at offset %d, skipping chunk [%d, %d].", a_SectorStart * 4096 + 4, a_ChunkX, a_ChunkZ);
return;
}
if (m_Callback.OnCompressedDataSizePos(ByteSize, a_SectorStart * 4096 + 5, CompressionMethod))
{
return;
}
char CompressedData[MAX_COMPRESSED_CHUNK_SIZE];
if (a_File.Read(CompressedData, ByteSize - 1) != ByteSize - 1)
{
LOG("Cannot read %d bytes of compressed data at offset %d, skipping chunk [%d, %d]", ByteSize - 1, a_SectorStart * 4096 + 5, a_ChunkX, a_ChunkZ);
return;
}
ProcessCompressedChunkData(a_ChunkX, a_ChunkZ, CompressedData, ByteSize);
}
void cProcessor::cThread::ProcessCompressedChunkData(int a_ChunkX, int a_ChunkZ, const char * a_CompressedData, int a_CompressedSize)
{
char Decompressed[CHUNK_INFLATE_MAX];
z_stream strm;
strm.zalloc = (alloc_func)NULL;
strm.zfree = (free_func)NULL;
strm.opaque = NULL;
inflateInit(&strm);
strm.next_out = (Bytef *)Decompressed;
strm.avail_out = sizeof(Decompressed);
strm.next_in = (Bytef *)a_CompressedData;
strm.avail_in = a_CompressedSize;
int res = inflate(&strm, Z_FINISH);
inflateEnd(&strm);
if (res != Z_STREAM_END)
{
LOG("Decompression failed, skipping chunk [%d, %d]", a_ChunkX, a_ChunkZ);
return;
}
if (m_Callback.OnDecompressedData(Decompressed, strm.total_out))
{
return;
}
// Parse the NBT data:
cParsedNBT NBT(Decompressed, strm.total_out);
if (!NBT.IsValid())
{
LOG("NBT Parsing failed, skipping chunk [%d, %d]", a_ChunkX, a_ChunkZ);
return;
}
ProcessParsedChunkData(a_ChunkX, a_ChunkZ, NBT);
}
void cProcessor::cThread::ProcessParsedChunkData(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT)
{
int LevelTag = a_NBT.FindChildByName(0, "Level");
if (LevelTag < 0)
{
LOG("Bad logical structure of the NBT, skipping chunk [%d, %d].", a_ChunkX, a_ChunkZ);
return;
}
int XPosTag = a_NBT.FindChildByName(LevelTag, "xPos");
int ZPosTag = a_NBT.FindChildByName(LevelTag, "zPos");
if ((XPosTag < 0) || (ZPosTag < 0))
{
LOG("Pos tags missing in NTB, skipping chunk [%d, %d].", a_ChunkX, a_ChunkZ);
return;
}
if (m_Callback.OnRealCoords(a_NBT.GetInt(XPosTag), a_NBT.GetInt(ZPosTag)))
{
return;
}
int LastUpdateTag = a_NBT.FindChildByName(LevelTag, "LastUpdate");
if (LastUpdateTag > 0)
{
if (m_Callback.OnLastUpdate(a_NBT.GetLong(LastUpdateTag)))
{
return;
}
}
int TerrainPopulatedTag = a_NBT.FindChildByName(LevelTag, "TerrainPopulated");
bool TerrainPopulated = (TerrainPopulatedTag < 0) ? false : (a_NBT.GetByte(TerrainPopulatedTag) != 0);
if (m_Callback.OnTerrainPopulated(TerrainPopulated))
{
return;
}
int BiomesTag = a_NBT.FindChildByName(LevelTag, "Biomes");
if (BiomesTag > 0)
{
if (m_Callback.OnBiomes((const unsigned char *)(a_NBT.GetData(BiomesTag))))
{
return;
}
}
int HeightMapTag = a_NBT.FindChildByName(LevelTag, "HeightMap");
if (HeightMapTag > 0)
{
if (m_Callback.OnHeightMap((const int *)(a_NBT.GetData(HeightMapTag))))
{
return;
}
}
if (ProcessChunkSections(a_ChunkX, a_ChunkZ, a_NBT, LevelTag))
{
return;
}
// TODO: entities, tile-entities etc.
}
bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT, int a_LevelTag)
{
int Sections = a_NBT.FindChildByName(a_LevelTag, "Sections");
if (Sections < 0)
{
return false;
}
for (int Tag = a_NBT.GetFirstChild(Sections); Tag > 0; Tag = a_NBT.GetNextSibling(Tag))
{
int YTag = a_NBT.FindChildByName(Tag, "Y");
int BlocksTag = a_NBT.FindChildByName(Tag, "Blocks");
int AddTag = a_NBT.FindChildByName(Tag, "Add");
int DataTag = a_NBT.FindChildByName(Tag, "Data");
int BlockLightTag = a_NBT.FindChildByName(Tag, "BlockLightTag");
int SkyLightTag = a_NBT.FindChildByName(Tag, "SkyLight");
if ((YTag < 0) || (BlocksTag < 0) || (DataTag < 0))
{
continue;
}
if (m_Callback.OnSection(
a_NBT.GetByte(YTag),
(const BLOCKTYPE *) (a_NBT.GetData(BlocksTag)),
(AddTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(AddTag)) : NULL,
(const NIBBLETYPE *)(a_NBT.GetData(DataTag)),
(BlockLightTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(BlockLightTag)) : NULL,
(BlockLightTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(BlockLightTag)) : NULL
))
{
return true;
}
} // for Tag - Sections[]
return false;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cProcessor:
cProcessor::cProcessor(void) :
m_IsShuttingDown(false)
{
}
cProcessor::~cProcessor()
{
}
void cProcessor::ProcessWorld(const AString & a_WorldFolder, cCallbackFactory & a_CallbackFactory)
{
PopulateFileQueue(a_WorldFolder);
// Start as many threads as there are cores:
// Get number of cores by querying the system process affinity mask
DWORD Affinity, ProcAffinity;
GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity);
while (Affinity > 0)
{
if ((Affinity & 1) == 1)
{
cCallback * Callback = a_CallbackFactory.GetNewCallback();
m_Threads.push_back(new cThread(*Callback, *this));
}
Affinity >>= 1;
} // while (Affinity > 0)
if (m_Threads.size() == 0)
{
LOG("Zero cores detected - how am I running? Running in a single thread.");
cCallback * Callback = a_CallbackFactory.GetNewCallback();
m_Threads.push_back(new cThread(*Callback, *this));
}
// Wait for all threads to finish
// simply by calling each thread's destructor sequentially
for (cThreads::iterator itr = m_Threads.begin(), end = m_Threads.end(); itr != end; ++itr)
{
delete *itr;
} // for itr - m_Threads[]
}
void cProcessor::PopulateFileQueue(const AString & a_WorldFolder)
{
LOG("Processing world in \"%s\"...", a_WorldFolder.c_str());
AString Path = a_WorldFolder;
Path.push_back(cFile::PathSeparator);
AStringList AllFiles = GetDirectoryContents(Path.c_str());
for (AStringList::iterator itr = AllFiles.begin(), end = AllFiles.end(); itr != end; ++itr)
{
if (itr->rfind(".mca") != itr->length() - 4)
{
// Not a .mca file
continue;
}
m_FileQueue.push_back(Path + *itr);
} // for itr - AllFiles[]
}
AString cProcessor::GetOneFileName(void)
{
cCSLock Lock(m_CS);
if (m_FileQueue.empty())
{
return "";
}
AString res = m_FileQueue.back();
m_FileQueue.pop_back();
return res;
}

70
AnvilStats/Processor.h Normal file
View File

@ -0,0 +1,70 @@
// Processor.h
// Interfaces to the cProcessor class representing the overall processor engine that manages threads, calls callbacks etc.
#pragma once
// fwd:
class cCallback;
class cCallbackFactory;
class cParsedNBT;
class cProcessor
{
class cThread :
public cIsThread
{
typedef cIsThread super;
cCallback & m_Callback;
cProcessor & m_ParentProcessor;
// cIsThread override:
virtual void Execute(void) override;
void ProcessFile(const AString & a_FileName);
void ProcessChunk(cFile & a_File, int a_ChunkX, int a_ChunkZ, unsigned a_SectorStart, unsigned a_SectorSize, unsigned a_TimeStamp);
void ProcessCompressedChunkData(int a_ChunkX, int a_ChunkZ, const char * a_CompressedData, int a_CompressedSize);
void ProcessParsedChunkData(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT);
bool ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT, int a_LevelTag);
public:
cThread(cCallback & a_Callback, cProcessor & a_ParentProcessor);
} ;
typedef std::vector<cThread *> cThreads;
public:
cProcessor(void);
~cProcessor();
void ProcessWorld(const AString & a_WorldFolder, cCallbackFactory & a_CallbackFactory);
protected:
bool m_IsShuttingDown; // If true, the threads should stop ASAP
cCriticalSection m_CS;
AStringList m_FileQueue;
cThreads m_Threads;
void PopulateFileQueue(const AString & a_WorldFolder);
AString GetOneFileName(void);
} ;

211
AnvilStats/Statistics.cpp Normal file
View File

@ -0,0 +1,211 @@
// Statistics.cpp
// Implements the various statistics-collecting classes
#include "Globals.h"
#include "Statistics.h"
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cStatistics:
cStatistics::cStatistics(void) :
m_TotalChunks(0),
m_BiomeNumChunks(0),
m_BlockNumChunks(0)
{
memset(m_BiomeCounts, 0, sizeof(m_BiomeCounts));
memset(m_BlockCounts, 0, sizeof(m_BlockCounts));
}
bool cStatistics::OnNewChunk(int a_ChunkX, int a_ChunkZ)
{
m_TotalChunks++;
m_IsBiomesValid = false;
m_IsFirstSectionInChunk = true;
return false;
}
bool cStatistics::OnBiomes(const unsigned char * a_BiomeData)
{
for (int i = 0; i < 16 * 16; i++)
{
m_BiomeCounts[a_BiomeData[i]] += 1;
}
m_BiomeNumChunks += 1;
memcpy(m_BiomeData, a_BiomeData, sizeof(m_BiomeData));
m_IsBiomesValid = true;
return false;
}
bool cStatistics::OnSection
(
unsigned char a_Y,
const BLOCKTYPE * a_BlockTypes,
const NIBBLETYPE * a_BlockAdditional,
const NIBBLETYPE * a_BlockMeta,
const NIBBLETYPE * a_BlockLight,
const NIBBLETYPE * a_BlockSkyLight
)
{
if (!m_IsBiomesValid)
{
// The current biome data is not valid, we don't have the means for sorting the BlockTypes into per-biome arrays
return true;
}
for (int y = 0; y < 16; y++)
{
for (int z = 0; z < 16; z++)
{
for (int x = 0; x < 16; x++)
{
unsigned char Biome = m_BiomeData[x + 16 * z]; // Cannot use cChunkDef, different data size
unsigned char BlockType = cChunkDef::GetBlock(a_BlockTypes, x, y, z);
if (BlockType == 12)
{
__asm nop;
}
m_BlockCounts[Biome][BlockType] += 1;
}
}
}
m_BlockNumChunks += m_IsFirstSectionInChunk ? 1 : 0;
m_IsFirstSectionInChunk = false;
return true;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cStatisticsFactory:
cStatisticsFactory::~cStatisticsFactory()
{
// TODO: Join the results together and export
LOG("cStatistics:");
LOG(" Joining results...");
JoinResults();
LOG(" Total %d chunks went through", m_TotalChunks);
LOG(" Biomes processed for %d chunks", m_BiomeNumChunks);
LOG(" BlockIDs processed for %d chunks", m_BlockNumChunks);
LOG(" Saving statistics into files:");
LOG(" Biomes.txt");
SaveBiomes();
LOG(" BlockTypes.txt");
SaveBlockTypes();
LOG(" BiomeBlockTypes.txt");
SaveBiomeBlockTypes();
}
void cStatisticsFactory::JoinResults(void)
{
m_BiomeNumChunks = 0;
m_BlockNumChunks = 0;
m_TotalChunks = 0;
memset(m_BiomeCounts, 0, sizeof(m_BiomeCounts));
memset(m_BlockCounts, 0, sizeof(m_BlockCounts));
for (cCallbacks::iterator itr = m_Callbacks.begin(), end = m_Callbacks.end(); itr != end; ++itr)
{
cStatistics * stats = (cStatistics *)(*itr);
for (int i = 0; i <= 255; i++)
{
m_BiomeCounts[i] += stats->m_BiomeCounts[i];
}
for (int i = 0; i <= 255; i++)
{
for (int j = 0; j <= 255; j++)
{
m_BlockCounts[i][j] += stats->m_BlockCounts[i][j];
}
}
m_BiomeNumChunks += stats->m_BiomeNumChunks;
m_BlockNumChunks += stats->m_BlockNumChunks;
m_TotalChunks += stats->m_TotalChunks;
} // for itr - m_Callbacks[]
}
void cStatisticsFactory::SaveBiomes(void)
{
cFile f;
if (!f.Open("Biomes.xls", cFile::fmWrite))
{
LOG("Cannot write to file Biomes.txt. Statistics not written.");
return;
}
for (int i = 0; i <= 255; i++)
{
AString Line;
Printf(Line, "%d\t%d\n", i, m_BiomeCounts[i]);
f.Write(Line.c_str(), Line.length());
}
}
void cStatisticsFactory::SaveBlockTypes(void)
{
cFile f;
if (!f.Open("BlockTypes.xls", cFile::fmWrite))
{
LOG("Cannot write to file Biomes.txt. Statistics not written.");
return;
}
for (int i = 0; i <= 255; i++)
{
int Count = 0;
for (int Biome = 0; Biome <= 255; ++Biome)
{
Count += m_BlockCounts[Biome][i];
}
AString Line;
Printf(Line, "%d\t%d\n", i, Count);
f.Write(Line.c_str(), Line.length());
}
// TODO
}
void cStatisticsFactory::SaveBiomeBlockTypes(void)
{
LOG("Not implemented yet!");
// TODO
}

88
AnvilStats/Statistics.h Normal file
View File

@ -0,0 +1,88 @@
// Statistics.h
// Interfaces to the cStatistics class representing a statistics-collecting callback
#pragma once
#include "Callback.h"
class cStatistics :
public cCallback
{
friend class cStatisticsFactory;
public:
cStatistics(void);
protected:
int m_TotalChunks; // Total number of chunks that go through this callback (OnNewChunk())
int m_BiomeCounts[256];
int m_BlockCounts[256][256]; // First dimension is the biome, second dimension is BlockType
int m_BiomeNumChunks; // Num chunks that have been processed for biome stats
int m_BlockNumChunks; // Num chunks that have been processed for block stats
bool m_IsBiomesValid; // Set to true in OnBiomes(), reset to false in OnNewChunk(); if true, the m_BiomeData is valid for the current chunk
unsigned char m_BiomeData[16 * 16];
bool m_IsFirstSectionInChunk; // True if there was no section in the chunk yet. Set by OnNewChunk(), reset by OnSection()
// cCallback overrides:
virtual bool OnNewChunk(int a_ChunkX, int a_ChunkZ) override;
virtual bool OnHeader(int a_FileOffset, unsigned char a_NumSectors, int a_Timestamp) override { return false; }
virtual bool OnCompressedDataSizePos(int a_CompressedDataSize, int a_DataOffset, char a_CompressionMethod) override { return false; }
virtual bool OnDecompressedData(const char * a_DecompressedNBT, int a_DataSize) override { return false; }
virtual bool OnRealCoords(int a_ChunkX, int a_ChunkZ) override { return false; }
virtual bool OnLastUpdate(Int64 a_LastUpdate) override { return false; }
virtual bool OnTerrainPopulated(bool a_Populated) override { return !a_Populated; } // If not populated, we don't want it!
virtual bool OnBiomes(const unsigned char * a_BiomeData) override;
virtual bool OnHeightMap(const int * a_HeightMap) override { return false; }
virtual bool OnSection(
unsigned char a_Y,
const BLOCKTYPE * a_BlockTypes,
const NIBBLETYPE * a_BlockAdditional,
const NIBBLETYPE * a_BlockMeta,
const NIBBLETYPE * a_BlockLight,
const NIBBLETYPE * a_BlockSkyLight
) override;
} ;
class cStatisticsFactory :
public cCallbackFactory
{
public:
virtual ~cStatisticsFactory();
virtual cCallback * CreateNewCallback(void)
{
return new cStatistics;
}
protected:
// The results, combined, are stored here:
int m_TotalChunks;
int m_BiomeCounts[256];
int m_BlockCounts[256][256]; // First dimension is the biome, second dimension is BlockType
int m_BiomeNumChunks; // Num chunks that have been processed for biome stats
int m_BlockNumChunks; // Num chunks that have been processed for block stats
void JoinResults(void);
void SaveBiomes(void);
void SaveBlockTypes(void);
void SaveBiomeBlockTypes(void);
} ;