From 6ccc4e36743485c50c58ec0f43d08181ba20958a Mon Sep 17 00:00:00 2001 From: "madmaxoft@gmail.com" Date: Thu, 27 Sep 2012 20:02:25 +0000 Subject: [PATCH] AnvilStats: initial project import, can do block and biome statistics git-svn-id: http://mc-server.googlecode.com/svn/trunk@895 0a769ca7-a7f5-676a-18bf-c427514a06d6 --- AnvilStats/AnvilStats.cpp | 53 +++++ AnvilStats/AnvilStats.sln | 34 +++ AnvilStats/AnvilStats.txt | 27 +++ AnvilStats/AnvilStats.vcproj | 284 ++++++++++++++++++++++++ AnvilStats/Callback.h | 108 ++++++++++ AnvilStats/Globals.cpp | 10 + AnvilStats/Globals.h | 220 +++++++++++++++++++ AnvilStats/Processor.cpp | 407 +++++++++++++++++++++++++++++++++++ AnvilStats/Processor.h | 70 ++++++ AnvilStats/Statistics.cpp | 211 ++++++++++++++++++ AnvilStats/Statistics.h | 88 ++++++++ 11 files changed, 1512 insertions(+) create mode 100644 AnvilStats/AnvilStats.cpp create mode 100644 AnvilStats/AnvilStats.sln create mode 100644 AnvilStats/AnvilStats.txt create mode 100644 AnvilStats/AnvilStats.vcproj create mode 100644 AnvilStats/Callback.h create mode 100644 AnvilStats/Globals.cpp create mode 100644 AnvilStats/Globals.h create mode 100644 AnvilStats/Processor.cpp create mode 100644 AnvilStats/Processor.h create mode 100644 AnvilStats/Statistics.cpp create mode 100644 AnvilStats/Statistics.h diff --git a/AnvilStats/AnvilStats.cpp b/AnvilStats/AnvilStats.cpp new file mode 100644 index 000000000..1682f76f0 --- /dev/null +++ b/AnvilStats/AnvilStats.cpp @@ -0,0 +1,53 @@ + +// AnvilStats.cpp + +// Implements the main app entrypoint + +#include "Globals.h" +#include "Statistics.h" +#include "Processor.h" + + + + + +int main(int argc, char * argv[]) +{ + if (argc < 2) + { + LOG("Usage: %s []", argv[0]); + LOG("\nNo method number present, aborting."); + return -1; + } + + AString WorldFolder; + if (argc > 2) + { + WorldFolder = argv[2]; + } + else + { + WorldFolder = "." + cFile::PathSeparator; + } + + cCallbackFactory * Factory = NULL; + switch (atol(argv[1])) + { + case 0: Factory = new cStatisticsFactory; break; + default: + { + LOG("Unknown method \"%s\", aborting.", argv[1]); + return -2; + } + } + cProcessor Processor; + Processor.ProcessWorld(WorldFolder, *Factory); + + delete Factory; + + LOG("Done"); +} + + + + diff --git a/AnvilStats/AnvilStats.sln b/AnvilStats/AnvilStats.sln new file mode 100644 index 000000000..b7f2d8b9e --- /dev/null +++ b/AnvilStats/AnvilStats.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual C++ Express 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "AnvilStats", "AnvilStats.vcproj", "{CF996A5E-0A86-4004-9710-682B06B5AEBA}" + ProjectSection(ProjectDependencies) = postProject + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA} = {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "..\VC2008\zlib.vcproj", "{EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release profiled|Win32 = Release profiled|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Debug|Win32.ActiveCfg = Debug|Win32 + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Debug|Win32.Build.0 = Debug|Win32 + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release profiled|Win32.ActiveCfg = Release|Win32 + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release profiled|Win32.Build.0 = Release|Win32 + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release|Win32.ActiveCfg = Release|Win32 + {CF996A5E-0A86-4004-9710-682B06B5AEBA}.Release|Win32.Build.0 = Release|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Debug|Win32.ActiveCfg = Debug|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Debug|Win32.Build.0 = Debug|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release profiled|Win32.ActiveCfg = Release profiled|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release profiled|Win32.Build.0 = Release profiled|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release|Win32.ActiveCfg = Release|Win32 + {EA9D50FD-937A-4EF5-8C37-5F4175AF4FEA}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/AnvilStats/AnvilStats.txt b/AnvilStats/AnvilStats.txt new file mode 100644 index 000000000..74c12d0af --- /dev/null +++ b/AnvilStats/AnvilStats.txt @@ -0,0 +1,27 @@ + +// AnvilStats.txt + +// A Readme for the project + +/* +AnvilStats +========== + +This is a project for measuring various metrics throughout an Anvil world, presumably created by a vanilla MC. +It works by parsing the MCA files in the path specified as its param (or current directory, if no params) and +feeding each decompressed chunk into the statistics-gathering callback function. + +Possible usage: + - count the per-chunk density of specific blocks + - count the per-chunk density of dungeons, by measuring the number of zombie/skeleton/regularspider spawners + - count the per-chunk-per-biome density of trees, by measuring the number of dirt-log vertical transitions, correlating to biome data + +This project is Windows-only, although it shouldn't be too difficult to make it portable. + + + + +*/ + + + diff --git a/AnvilStats/AnvilStats.vcproj b/AnvilStats/AnvilStats.vcproj new file mode 100644 index 000000000..6e8038a1c --- /dev/null +++ b/AnvilStats/AnvilStats.vcproj @@ -0,0 +1,284 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/AnvilStats/Callback.h b/AnvilStats/Callback.h new file mode 100644 index 000000000..b18f4947a --- /dev/null +++ b/AnvilStats/Callback.h @@ -0,0 +1,108 @@ + +// Callback.h + +// Interfaces to the cCallback base class used as the base class for all statistical callbacks + + + + + +#pragma once + + + + + +/** The base class for all chunk-processor callbacks, declares the interface. +The processor calls each virtual function in the order they are declared here with the specified args. +If the function returns true, the processor moves on to next chunk and starts calling the callbacks again from start with +the new chunk data. +So if a statistics collector doesn't need data decompression at all, it can stop the processor from doing so early-enough +and still get meaningful data. +A callback is guaranteed to run in a single thread and always the same thread. +A callback is guaranteed to run on all chunks in a region and one region is guaranteed to be handled by only callback. +*/ +class cCallback abstract +{ +public: + virtual ~cCallback() {} // Force a virtual destructor in each descendant + + /// Called to inform the stats module of the chunk coords for newly processing chunk + virtual bool OnNewChunk(int a_ChunkX, int a_ChunkZ) = 0; + + /// Called to inform about the chunk's data offset in the file (chunk mini-header), the number of sectors it uses and the timestamp field value + virtual bool OnHeader(int a_FileOffset, unsigned char a_NumSectors, int a_Timestamp) { return true; } + + /// Called to inform of the compressed chunk data size and position in the file (offset from file start to the actual data) + virtual bool OnCompressedDataSizePos(int a_CompressedDataSize, int a_DataOffset, char a_CompressionMethod) { return true; } + + /// Just in case you wanted to process the NBT yourself ;) + virtual bool OnDecompressedData(const char * a_DecompressedNBT, int a_DataSize) { return true; } + + /// The chunk's NBT should specify chunk coords, these are sent here: + virtual bool OnRealCoords(int a_ChunkX, int a_ChunkZ) { return true; } + + /// The chunk contains a LastUpdate value specifying the last tick in which it was saved. + virtual bool OnLastUpdate(Int64 a_LastUpdate) { return true; } + + virtual bool OnTerrainPopulated(bool a_Populated) { return true; } + + virtual bool OnBiomes(const unsigned char * a_BiomeData) { return true; } + + virtual bool OnHeightMap(const int * a_HeightMap) { return true; } + + virtual bool OnSection( + unsigned char a_Y, + const BLOCKTYPE * a_BlockTypes, + const NIBBLETYPE * a_BlockAdditional, + const NIBBLETYPE * a_BlockMeta, + const NIBBLETYPE * a_BlockLight, + const NIBBLETYPE * a_BlockSkyLight + ) { return true; } + + // TODO: entities, tile-entities, tile-ticks +} ; + +typedef std::vector cCallbacks; + + + + + +/** The base class for a factory that creates callback objects for separate threads. +The processor creates a callback for each thread on which it runs using this factory. +The factory is guaranteed to be called from a single thread. +The factory keeps track of all the callbacks that it has created and deletes them when destructed +*/ +class cCallbackFactory +{ +public: + virtual ~cCallbackFactory() + { + for (cCallbacks::iterator itr = m_Callbacks.begin(), end = m_Callbacks.end(); itr != end; ++itr) + { + delete *itr; + } + } + + /// Descendants override this method to return the correct callback type + virtual cCallback * CreateNewCallback(void) = 0; + + /// cProcessor uses this method to request a new callback + cCallback * GetNewCallback(void) + { + cCallback * Callback = CreateNewCallback(); + if (Callback != NULL) + { + m_Callbacks.push_back(Callback); + } + return Callback; + } + +protected: + cCallbacks m_Callbacks; +} ; + + + + diff --git a/AnvilStats/Globals.cpp b/AnvilStats/Globals.cpp new file mode 100644 index 000000000..2c60fd698 --- /dev/null +++ b/AnvilStats/Globals.cpp @@ -0,0 +1,10 @@ + +// Globals.cpp + +// This file is used for precompiled header generation in MSVC environments + +#include "Globals.h" + + + + diff --git a/AnvilStats/Globals.h b/AnvilStats/Globals.h new file mode 100644 index 000000000..2d4d4d20e --- /dev/null +++ b/AnvilStats/Globals.h @@ -0,0 +1,220 @@ + +// Globals.h + +// This file gets included from every module in the project, so that global symbols may be introduced easily +// Also used for precompiled header generation in MSVC environments + + + + + +// Compiler-dependent stuff: +#if defined(_MSC_VER) + // MSVC produces warning C4481 on the override keyword usage, so disable the warning altogether + #pragma warning(disable:4481) + + // Disable some warnings that we don't care about: + #pragma warning(disable:4100) + + #define OBSOLETE __declspec(deprecated) + + // No alignment needed in MSVC + #define ALIGN_8 + #define ALIGN_16 + +#elif defined(__GNUC__) + + // TODO: Can GCC explicitly mark classes as abstract (no instances can be created)? + #define abstract + + // TODO: Can GCC mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class) + #define override + + #define OBSOLETE __attribute__((deprecated)) + + #define ALIGN_8 __attribute__((aligned(8))) + #define ALIGN_16 __attribute__((aligned(16))) + + // Some portability macros :) + #define stricmp strcasecmp + +#else + + #error "You are using an unsupported compiler, you might need to #define some stuff here for your compiler" + + /* + // Copy and uncomment this into another #elif section based on your compiler identification + + // Explicitly mark classes as abstract (no instances can be created) + #define abstract + + // Mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class) + #define override + + // Mark functions as obsolete, so that their usage results in a compile-time warning + #define OBSOLETE + + // Mark types / variables for alignment. Do the platforms need it? + #define ALIGN_8 + #define ALIGN_16 + */ + +#endif + + + + + +// Integral types with predefined sizes: +typedef long long Int64; +typedef int Int32; +typedef short Int16; + + + + + +// A macro to disallow the copy constructor and operator= functions +// This should be used in the private: declarations for any class that shouldn't allow copying itself +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName &); \ + void operator=(const TypeName &) + +// A macro that is used to mark unused function parameters, to avoid pedantic warnings in gcc +#define UNUSED(X) (void)(X) + + + + +// OS-dependent stuff: +#ifdef _WIN32 + #define WIN32_LEAN_AND_MEAN + #include + #include + + // Windows SDK defines min and max macros, messing up with our std::min and std::max usage + #undef min + #undef max + + // Windows SDK defines GetFreeSpace as a constant, probably a Win16 API remnant + #ifdef GetFreeSpace + #undef GetFreeSpace + #endif // GetFreeSpace +#else + #include + #include // for mkdir + #include + #include + #include + #include + #include + #include + #include + #include + #include + + #include + #include + #include + #include + #include + #include +#endif + + + + + +#define FILE_IO_PREFIX "" + + + + + +// CRT stuff: +#include +#include +#include +#include + + + + + +// STL stuff: +#include +#include +#include +#include +#include +#include +#include + + + + + +// Common headers (part 1, without macros): +#include "../source/StringUtils.h" +#include "../source/OSSupport/CriticalSection.h" +#include "../source/OSSupport/Semaphore.h" +#include "../source/OSSupport/Event.h" +#include "../source/OSSupport/IsThread.h" +#include "../source/OSSupport/File.h" + + + + + +// Common definitions: + +#define LOG(x,...) printf(x "\n", __VA_ARGS__) +#define LOGERROR LOG +#define LOGWARNING LOG +#define LOGINFO LOG + +/// Evaluates to the number of elements in an array (compile-time!) +#define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X))) + +/// Allows arithmetic expressions like "32 KiB" (but consider using parenthesis around it, "(32 KiB)" ) +#define KiB * 1024 + +/// Allows arithmetic expressions like "32 MiB" (but consider using parenthesis around it, "(32 MiB)" ) +#define MiB * 1024 * 1024 + +/// Faster than (int)floorf((float)x / (float)div) +#define FAST_FLOOR_DIV( x, div ) ( (x) < 0 ? (((int)x / div) - 1) : ((int)x / div) ) + +// Own version of assert() that writes failed assertions to the log for review +#ifdef _DEBUG + #define ASSERT( x ) ( !!(x) || ( LOGERROR("Assertion failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), assert(0), 0 ) ) +#else + #define ASSERT(x) ((void)0) +#endif + +// Pretty much the same as ASSERT() but stays in Release builds +#define VERIFY( x ) ( !!(x) || ( LOGERROR("Verification failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), exit(1), 0 ) ) + + + + + +/// A generic interface used mainly in ForEach() functions +template class cItemCallback +{ +public: + /// Called for each item in the internal list; return true to stop the loop, or false to continue enumerating + virtual bool Item(Type * a_Type) = 0; +} ; + + + + + +// Common headers (part 2, with macros): +#include "../source/ChunkDef.h" +#include "../source/BlockID.h" + + + + diff --git a/AnvilStats/Processor.cpp b/AnvilStats/Processor.cpp new file mode 100644 index 000000000..0d9bc4694 --- /dev/null +++ b/AnvilStats/Processor.cpp @@ -0,0 +1,407 @@ + +// Processor.cpp + +// Implements the cProcessor class representing the overall processor engine that manages threads, calls callbacks etc. + +#include "Globals.h" +#include "Processor.h" +#include "Callback.h" +#include "../source/WorldStorage/FastNBT.h" +#include "zlib.h" + + + + + +const int CHUNK_INFLATE_MAX = 1 MiB; +const int MAX_COMPRESSED_CHUNK_SIZE = 1 MiB; + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cProcessor::cThread: + +cProcessor::cThread::cThread(cCallback & a_Callback, cProcessor & a_ParentProcessor) : + super("cProcessor::cThread"), + m_Callback(a_Callback), + m_ParentProcessor(a_ParentProcessor) +{ + super::Start(); +} + + + + + +void cProcessor::cThread::Execute(void) +{ + for (;;) + { + AString FileName = m_ParentProcessor.GetOneFileName(); + if (FileName.empty()) + { + // All done, terminate the thread + return; + } + ProcessFile(FileName); + } // for-ever +} + + + + + +void cProcessor::cThread::ProcessFile(const AString & a_FileName) +{ + LOG("Processing file \"%s\"", a_FileName.c_str()); + + size_t idx = a_FileName.rfind("r."); + if (idx == AString::npos) + { + LOG("Cannot parse filename \"%s\", skipping file.", a_FileName.c_str()); + return; + } + int RegionX = 0, RegionZ = 0; + if (sscanf_s(a_FileName.c_str() + idx, "r.%d.%d.mca", &RegionX, &RegionZ) != 2) + { + LOG("Cannot parse filename \"%s\" into coords, skipping file.", a_FileName.c_str()); + return; + } + + cFile f; + if (!f.Open(a_FileName, cFile::fmRead)) + { + LOG("Cannot open file \"%s\", skipping file.", a_FileName.c_str()); + return; + } + + int Header[2048]; + if (f.Read(Header, sizeof(Header)) != sizeof(Header)) + { + LOG("Cannot read header in file \"%s\", skipping file.", a_FileName.c_str()); + return; + } + + for (int i = 0; i < ARRAYCOUNT(Header); i++) + { + Header[i] = ntohl(Header[i]); + } + + int ChunkBaseX = RegionX * 32; + int ChunkBaseZ = RegionZ * 32; + for (int i = 0; i < 1024; i++) + { + unsigned Location = Header[i]; + unsigned Timestamp = Header[i + 1024]; + if ( + ((Location == 0) && (Timestamp == 0)) || // Official docs' "not present" + (Location >> 8 < 2) || // Logical - no chunk can start inside the header + ((Location & 0xff) == 0) // Logical - no chunk can be zero bytes + ) + { + // Chunk not present in the file + continue; + } + int ChunkX = ChunkBaseX + (i % 32); + int ChunkZ = ChunkBaseZ + (i / 32); + if (m_Callback.OnNewChunk(ChunkX, ChunkZ)) + { + continue; + } + ProcessChunk(f, ChunkX, ChunkZ, Location >> 8, Location & 0xff, Timestamp); + } // for i - chunk index +} + + + + + +void cProcessor::cThread::ProcessChunk(cFile & a_File, int a_ChunkX, int a_ChunkZ, unsigned a_SectorStart, unsigned a_SectorSize, unsigned a_TimeStamp) +{ + if (m_Callback.OnHeader(a_SectorStart * 4096, a_SectorSize, a_TimeStamp)) + { + return; + } + + if (a_File.Seek(a_SectorStart * 4096) < 0) + { + LOG("Seeking to sector %d failed, skipping chunk [%d, %d]", a_SectorStart, a_ChunkX, a_ChunkZ); + return; + } + + int ByteSize; + if (a_File.Read(&ByteSize, sizeof(ByteSize)) != sizeof(ByteSize)) + { + LOG("Cannot read bytesize at offset %d, skipping chunk [%d, %d].", a_SectorStart * 4096, a_ChunkX, a_ChunkZ); + return; + } + ByteSize = ntohl(ByteSize); + + char CompressionMethod; + if (a_File.Read(&CompressionMethod, sizeof(CompressionMethod)) != sizeof(CompressionMethod)) + { + LOG("Cannot read CompressionMethod at offset %d, skipping chunk [%d, %d].", a_SectorStart * 4096 + 4, a_ChunkX, a_ChunkZ); + return; + } + + if (m_Callback.OnCompressedDataSizePos(ByteSize, a_SectorStart * 4096 + 5, CompressionMethod)) + { + return; + } + + char CompressedData[MAX_COMPRESSED_CHUNK_SIZE]; + if (a_File.Read(CompressedData, ByteSize - 1) != ByteSize - 1) + { + LOG("Cannot read %d bytes of compressed data at offset %d, skipping chunk [%d, %d]", ByteSize - 1, a_SectorStart * 4096 + 5, a_ChunkX, a_ChunkZ); + return; + } + + ProcessCompressedChunkData(a_ChunkX, a_ChunkZ, CompressedData, ByteSize); +} + + + + + +void cProcessor::cThread::ProcessCompressedChunkData(int a_ChunkX, int a_ChunkZ, const char * a_CompressedData, int a_CompressedSize) +{ + char Decompressed[CHUNK_INFLATE_MAX]; + z_stream strm; + strm.zalloc = (alloc_func)NULL; + strm.zfree = (free_func)NULL; + strm.opaque = NULL; + inflateInit(&strm); + strm.next_out = (Bytef *)Decompressed; + strm.avail_out = sizeof(Decompressed); + strm.next_in = (Bytef *)a_CompressedData; + strm.avail_in = a_CompressedSize; + int res = inflate(&strm, Z_FINISH); + inflateEnd(&strm); + if (res != Z_STREAM_END) + { + LOG("Decompression failed, skipping chunk [%d, %d]", a_ChunkX, a_ChunkZ); + return; + } + + if (m_Callback.OnDecompressedData(Decompressed, strm.total_out)) + { + return; + } + + // Parse the NBT data: + cParsedNBT NBT(Decompressed, strm.total_out); + if (!NBT.IsValid()) + { + LOG("NBT Parsing failed, skipping chunk [%d, %d]", a_ChunkX, a_ChunkZ); + return; + } + + ProcessParsedChunkData(a_ChunkX, a_ChunkZ, NBT); +} + + + + + +void cProcessor::cThread::ProcessParsedChunkData(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT) +{ + int LevelTag = a_NBT.FindChildByName(0, "Level"); + if (LevelTag < 0) + { + LOG("Bad logical structure of the NBT, skipping chunk [%d, %d].", a_ChunkX, a_ChunkZ); + return; + } + int XPosTag = a_NBT.FindChildByName(LevelTag, "xPos"); + int ZPosTag = a_NBT.FindChildByName(LevelTag, "zPos"); + if ((XPosTag < 0) || (ZPosTag < 0)) + { + LOG("Pos tags missing in NTB, skipping chunk [%d, %d].", a_ChunkX, a_ChunkZ); + return; + } + if (m_Callback.OnRealCoords(a_NBT.GetInt(XPosTag), a_NBT.GetInt(ZPosTag))) + { + return; + } + + int LastUpdateTag = a_NBT.FindChildByName(LevelTag, "LastUpdate"); + if (LastUpdateTag > 0) + { + if (m_Callback.OnLastUpdate(a_NBT.GetLong(LastUpdateTag))) + { + return; + } + } + + int TerrainPopulatedTag = a_NBT.FindChildByName(LevelTag, "TerrainPopulated"); + bool TerrainPopulated = (TerrainPopulatedTag < 0) ? false : (a_NBT.GetByte(TerrainPopulatedTag) != 0); + if (m_Callback.OnTerrainPopulated(TerrainPopulated)) + { + return; + } + + int BiomesTag = a_NBT.FindChildByName(LevelTag, "Biomes"); + if (BiomesTag > 0) + { + if (m_Callback.OnBiomes((const unsigned char *)(a_NBT.GetData(BiomesTag)))) + { + return; + } + } + + int HeightMapTag = a_NBT.FindChildByName(LevelTag, "HeightMap"); + if (HeightMapTag > 0) + { + if (m_Callback.OnHeightMap((const int *)(a_NBT.GetData(HeightMapTag)))) + { + return; + } + } + + if (ProcessChunkSections(a_ChunkX, a_ChunkZ, a_NBT, LevelTag)) + { + return; + } + // TODO: entities, tile-entities etc. +} + + + + + +bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT, int a_LevelTag) +{ + int Sections = a_NBT.FindChildByName(a_LevelTag, "Sections"); + if (Sections < 0) + { + return false; + } + + for (int Tag = a_NBT.GetFirstChild(Sections); Tag > 0; Tag = a_NBT.GetNextSibling(Tag)) + { + int YTag = a_NBT.FindChildByName(Tag, "Y"); + int BlocksTag = a_NBT.FindChildByName(Tag, "Blocks"); + int AddTag = a_NBT.FindChildByName(Tag, "Add"); + int DataTag = a_NBT.FindChildByName(Tag, "Data"); + int BlockLightTag = a_NBT.FindChildByName(Tag, "BlockLightTag"); + int SkyLightTag = a_NBT.FindChildByName(Tag, "SkyLight"); + + if ((YTag < 0) || (BlocksTag < 0) || (DataTag < 0)) + { + continue; + } + + if (m_Callback.OnSection( + a_NBT.GetByte(YTag), + (const BLOCKTYPE *) (a_NBT.GetData(BlocksTag)), + (AddTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(AddTag)) : NULL, + (const NIBBLETYPE *)(a_NBT.GetData(DataTag)), + (BlockLightTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(BlockLightTag)) : NULL, + (BlockLightTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(BlockLightTag)) : NULL + )) + { + return true; + } + } // for Tag - Sections[] + + return false; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cProcessor: + +cProcessor::cProcessor(void) : + m_IsShuttingDown(false) +{ +} + + + + + +cProcessor::~cProcessor() +{ +} + + + + + +void cProcessor::ProcessWorld(const AString & a_WorldFolder, cCallbackFactory & a_CallbackFactory) +{ + PopulateFileQueue(a_WorldFolder); + + // Start as many threads as there are cores: + // Get number of cores by querying the system process affinity mask + DWORD Affinity, ProcAffinity; + GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity); + while (Affinity > 0) + { + if ((Affinity & 1) == 1) + { + cCallback * Callback = a_CallbackFactory.GetNewCallback(); + m_Threads.push_back(new cThread(*Callback, *this)); + } + Affinity >>= 1; + } // while (Affinity > 0) + if (m_Threads.size() == 0) + { + LOG("Zero cores detected - how am I running? Running in a single thread."); + cCallback * Callback = a_CallbackFactory.GetNewCallback(); + m_Threads.push_back(new cThread(*Callback, *this)); + } + + // Wait for all threads to finish + // simply by calling each thread's destructor sequentially + for (cThreads::iterator itr = m_Threads.begin(), end = m_Threads.end(); itr != end; ++itr) + { + delete *itr; + } // for itr - m_Threads[] +} + + + + + +void cProcessor::PopulateFileQueue(const AString & a_WorldFolder) +{ + LOG("Processing world in \"%s\"...", a_WorldFolder.c_str()); + + AString Path = a_WorldFolder; + Path.push_back(cFile::PathSeparator); + AStringList AllFiles = GetDirectoryContents(Path.c_str()); + for (AStringList::iterator itr = AllFiles.begin(), end = AllFiles.end(); itr != end; ++itr) + { + if (itr->rfind(".mca") != itr->length() - 4) + { + // Not a .mca file + continue; + } + m_FileQueue.push_back(Path + *itr); + } // for itr - AllFiles[] +} + + + + + +AString cProcessor::GetOneFileName(void) +{ + cCSLock Lock(m_CS); + if (m_FileQueue.empty()) + { + return ""; + } + AString res = m_FileQueue.back(); + m_FileQueue.pop_back(); + return res; +} + + + + diff --git a/AnvilStats/Processor.h b/AnvilStats/Processor.h new file mode 100644 index 000000000..300185e67 --- /dev/null +++ b/AnvilStats/Processor.h @@ -0,0 +1,70 @@ + +// Processor.h + +// Interfaces to the cProcessor class representing the overall processor engine that manages threads, calls callbacks etc. + + + + +#pragma once + + + + + +// fwd: +class cCallback; +class cCallbackFactory; +class cParsedNBT; + + + + + +class cProcessor +{ + class cThread : + public cIsThread + { + typedef cIsThread super; + + cCallback & m_Callback; + cProcessor & m_ParentProcessor; + + // cIsThread override: + virtual void Execute(void) override; + + void ProcessFile(const AString & a_FileName); + void ProcessChunk(cFile & a_File, int a_ChunkX, int a_ChunkZ, unsigned a_SectorStart, unsigned a_SectorSize, unsigned a_TimeStamp); + void ProcessCompressedChunkData(int a_ChunkX, int a_ChunkZ, const char * a_CompressedData, int a_CompressedSize); + void ProcessParsedChunkData(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT); + bool ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cParsedNBT & a_NBT, int a_LevelTag); + + public: + cThread(cCallback & a_Callback, cProcessor & a_ParentProcessor); + } ; + + typedef std::vector cThreads; + +public: + cProcessor(void); + ~cProcessor(); + + void ProcessWorld(const AString & a_WorldFolder, cCallbackFactory & a_CallbackFactory); + +protected: + bool m_IsShuttingDown; // If true, the threads should stop ASAP + + cCriticalSection m_CS; + AStringList m_FileQueue; + + cThreads m_Threads; + + void PopulateFileQueue(const AString & a_WorldFolder); + + AString GetOneFileName(void); +} ; + + + + diff --git a/AnvilStats/Statistics.cpp b/AnvilStats/Statistics.cpp new file mode 100644 index 000000000..6c3b61778 --- /dev/null +++ b/AnvilStats/Statistics.cpp @@ -0,0 +1,211 @@ + +// Statistics.cpp + +// Implements the various statistics-collecting classes + +#include "Globals.h" +#include "Statistics.h" + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cStatistics: + +cStatistics::cStatistics(void) : + m_TotalChunks(0), + m_BiomeNumChunks(0), + m_BlockNumChunks(0) +{ + memset(m_BiomeCounts, 0, sizeof(m_BiomeCounts)); + memset(m_BlockCounts, 0, sizeof(m_BlockCounts)); +} + + + + + +bool cStatistics::OnNewChunk(int a_ChunkX, int a_ChunkZ) +{ + m_TotalChunks++; + m_IsBiomesValid = false; + m_IsFirstSectionInChunk = true; + return false; +} + + + + + +bool cStatistics::OnBiomes(const unsigned char * a_BiomeData) +{ + for (int i = 0; i < 16 * 16; i++) + { + m_BiomeCounts[a_BiomeData[i]] += 1; + } + m_BiomeNumChunks += 1; + memcpy(m_BiomeData, a_BiomeData, sizeof(m_BiomeData)); + m_IsBiomesValid = true; + return false; +} + + + + + + +bool cStatistics::OnSection +( + unsigned char a_Y, + const BLOCKTYPE * a_BlockTypes, + const NIBBLETYPE * a_BlockAdditional, + const NIBBLETYPE * a_BlockMeta, + const NIBBLETYPE * a_BlockLight, + const NIBBLETYPE * a_BlockSkyLight +) +{ + if (!m_IsBiomesValid) + { + // The current biome data is not valid, we don't have the means for sorting the BlockTypes into per-biome arrays + return true; + } + + for (int y = 0; y < 16; y++) + { + for (int z = 0; z < 16; z++) + { + for (int x = 0; x < 16; x++) + { + unsigned char Biome = m_BiomeData[x + 16 * z]; // Cannot use cChunkDef, different data size + unsigned char BlockType = cChunkDef::GetBlock(a_BlockTypes, x, y, z); + if (BlockType == 12) + { + __asm nop; + } + m_BlockCounts[Biome][BlockType] += 1; + } + } + } + m_BlockNumChunks += m_IsFirstSectionInChunk ? 1 : 0; + m_IsFirstSectionInChunk = false; + + return true; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cStatisticsFactory: + +cStatisticsFactory::~cStatisticsFactory() +{ + // TODO: Join the results together and export + LOG("cStatistics:"); + LOG(" Joining results..."); + JoinResults(); + LOG(" Total %d chunks went through", m_TotalChunks); + LOG(" Biomes processed for %d chunks", m_BiomeNumChunks); + LOG(" BlockIDs processed for %d chunks", m_BlockNumChunks); + LOG(" Saving statistics into files:"); + LOG(" Biomes.txt"); + SaveBiomes(); + LOG(" BlockTypes.txt"); + SaveBlockTypes(); + LOG(" BiomeBlockTypes.txt"); + SaveBiomeBlockTypes(); +} + + + + + +void cStatisticsFactory::JoinResults(void) +{ + m_BiomeNumChunks = 0; + m_BlockNumChunks = 0; + m_TotalChunks = 0; + memset(m_BiomeCounts, 0, sizeof(m_BiomeCounts)); + memset(m_BlockCounts, 0, sizeof(m_BlockCounts)); + for (cCallbacks::iterator itr = m_Callbacks.begin(), end = m_Callbacks.end(); itr != end; ++itr) + { + cStatistics * stats = (cStatistics *)(*itr); + for (int i = 0; i <= 255; i++) + { + m_BiomeCounts[i] += stats->m_BiomeCounts[i]; + } + for (int i = 0; i <= 255; i++) + { + for (int j = 0; j <= 255; j++) + { + m_BlockCounts[i][j] += stats->m_BlockCounts[i][j]; + } + } + m_BiomeNumChunks += stats->m_BiomeNumChunks; + m_BlockNumChunks += stats->m_BlockNumChunks; + m_TotalChunks += stats->m_TotalChunks; + } // for itr - m_Callbacks[] +} + + + + + +void cStatisticsFactory::SaveBiomes(void) +{ + cFile f; + if (!f.Open("Biomes.xls", cFile::fmWrite)) + { + LOG("Cannot write to file Biomes.txt. Statistics not written."); + return; + } + for (int i = 0; i <= 255; i++) + { + AString Line; + Printf(Line, "%d\t%d\n", i, m_BiomeCounts[i]); + f.Write(Line.c_str(), Line.length()); + } +} + + + + + +void cStatisticsFactory::SaveBlockTypes(void) +{ + cFile f; + if (!f.Open("BlockTypes.xls", cFile::fmWrite)) + { + LOG("Cannot write to file Biomes.txt. Statistics not written."); + return; + } + for (int i = 0; i <= 255; i++) + { + int Count = 0; + for (int Biome = 0; Biome <= 255; ++Biome) + { + Count += m_BlockCounts[Biome][i]; + } + AString Line; + Printf(Line, "%d\t%d\n", i, Count); + f.Write(Line.c_str(), Line.length()); + } + // TODO +} + + + + + +void cStatisticsFactory::SaveBiomeBlockTypes(void) +{ + LOG("Not implemented yet!"); + // TODO +} + + + + + diff --git a/AnvilStats/Statistics.h b/AnvilStats/Statistics.h new file mode 100644 index 000000000..980241ae6 --- /dev/null +++ b/AnvilStats/Statistics.h @@ -0,0 +1,88 @@ + +// Statistics.h + +// Interfaces to the cStatistics class representing a statistics-collecting callback + + + + + +#pragma once + +#include "Callback.h" + + + + + +class cStatistics : + public cCallback +{ + friend class cStatisticsFactory; + +public: + cStatistics(void); + +protected: + int m_TotalChunks; // Total number of chunks that go through this callback (OnNewChunk()) + int m_BiomeCounts[256]; + int m_BlockCounts[256][256]; // First dimension is the biome, second dimension is BlockType + int m_BiomeNumChunks; // Num chunks that have been processed for biome stats + int m_BlockNumChunks; // Num chunks that have been processed for block stats + bool m_IsBiomesValid; // Set to true in OnBiomes(), reset to false in OnNewChunk(); if true, the m_BiomeData is valid for the current chunk + unsigned char m_BiomeData[16 * 16]; + bool m_IsFirstSectionInChunk; // True if there was no section in the chunk yet. Set by OnNewChunk(), reset by OnSection() + + // cCallback overrides: + virtual bool OnNewChunk(int a_ChunkX, int a_ChunkZ) override; + virtual bool OnHeader(int a_FileOffset, unsigned char a_NumSectors, int a_Timestamp) override { return false; } + virtual bool OnCompressedDataSizePos(int a_CompressedDataSize, int a_DataOffset, char a_CompressionMethod) override { return false; } + virtual bool OnDecompressedData(const char * a_DecompressedNBT, int a_DataSize) override { return false; } + virtual bool OnRealCoords(int a_ChunkX, int a_ChunkZ) override { return false; } + virtual bool OnLastUpdate(Int64 a_LastUpdate) override { return false; } + virtual bool OnTerrainPopulated(bool a_Populated) override { return !a_Populated; } // If not populated, we don't want it! + virtual bool OnBiomes(const unsigned char * a_BiomeData) override; + virtual bool OnHeightMap(const int * a_HeightMap) override { return false; } + virtual bool OnSection( + unsigned char a_Y, + const BLOCKTYPE * a_BlockTypes, + const NIBBLETYPE * a_BlockAdditional, + const NIBBLETYPE * a_BlockMeta, + const NIBBLETYPE * a_BlockLight, + const NIBBLETYPE * a_BlockSkyLight + ) override; +} ; + + + + + +class cStatisticsFactory : + public cCallbackFactory +{ +public: + virtual ~cStatisticsFactory(); + + virtual cCallback * CreateNewCallback(void) + { + return new cStatistics; + } + +protected: + // The results, combined, are stored here: + int m_TotalChunks; + int m_BiomeCounts[256]; + int m_BlockCounts[256][256]; // First dimension is the biome, second dimension is BlockType + int m_BiomeNumChunks; // Num chunks that have been processed for biome stats + int m_BlockNumChunks; // Num chunks that have been processed for block stats + + void JoinResults(void); + void SaveBiomes(void); + void SaveBlockTypes(void); + void SaveBiomeBlockTypes(void); + +} ; + + + +