From 3165458c5993a447556efbbefc5aeb4352353347 Mon Sep 17 00:00:00 2001 From: "madmaxoft@gmail.com" Date: Fri, 28 Sep 2012 20:17:29 +0000 Subject: [PATCH] AnvilStats: complete per-biome blocktype statistics git-svn-id: http://mc-server.googlecode.com/svn/trunk@897 0a769ca7-a7f5-676a-18bf-c427514a06d6 --- AnvilStats/AnvilStats.vcproj | 16 +++ AnvilStats/Callback.h | 8 ++ AnvilStats/Globals.h | 1 + AnvilStats/Processor.cpp | 56 ++++++--- AnvilStats/Processor.h | 1 + AnvilStats/Statistics.cpp | 140 +++++++++++++++++++--- AnvilStats/Statistics.h | 1 + AnvilStats/Utils.cpp | 225 +++++++++++++++++++++++++++++++++++ AnvilStats/Utils.h | 15 +++ 9 files changed, 431 insertions(+), 32 deletions(-) create mode 100644 AnvilStats/Utils.cpp create mode 100644 AnvilStats/Utils.h diff --git a/AnvilStats/AnvilStats.vcproj b/AnvilStats/AnvilStats.vcproj index eaf861376..4bc494bf7 100644 --- a/AnvilStats/AnvilStats.vcproj +++ b/AnvilStats/AnvilStats.vcproj @@ -226,6 +226,14 @@ RelativePath=".\Statistics.h" > + + + + + + + + diff --git a/AnvilStats/Callback.h b/AnvilStats/Callback.h index b18f4947a..66d6bcfae 100644 --- a/AnvilStats/Callback.h +++ b/AnvilStats/Callback.h @@ -51,6 +51,9 @@ public: virtual bool OnHeightMap(const int * a_HeightMap) { return true; } + /** If there is data for the section, this callback is called; otherwise OnEmptySection() is called instead. + All OnSection() callbacks are called first, and only then all the remaining sections are reported in OnEmptySection(). + */ virtual bool OnSection( unsigned char a_Y, const BLOCKTYPE * a_BlockTypes, @@ -60,6 +63,11 @@ public: const NIBBLETYPE * a_BlockSkyLight ) { return true; } + /** If there is no data for a section, this callback is called; otherwise OnSection() is called instead. + OnEmptySection() callbacks are called after all OnSection() callbacks. + */ + virtual bool OnEmptySection(unsigned char a_Y) { return false; } + // TODO: entities, tile-entities, tile-ticks } ; diff --git a/AnvilStats/Globals.h b/AnvilStats/Globals.h index 2d4d4d20e..1c7387b75 100644 --- a/AnvilStats/Globals.h +++ b/AnvilStats/Globals.h @@ -172,6 +172,7 @@ typedef short Int16; #define LOGERROR LOG #define LOGWARNING LOG #define LOGINFO LOG +#define LOGWARN LOG /// Evaluates to the number of elements in an array (compile-time!) #define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X))) diff --git a/AnvilStats/Processor.cpp b/AnvilStats/Processor.cpp index 7a2668ea4..58f11e650 100644 --- a/AnvilStats/Processor.cpp +++ b/AnvilStats/Processor.cpp @@ -8,6 +8,7 @@ #include "Callback.h" #include "../source/WorldStorage/FastNBT.h" #include "zlib.h" +#include "Utils.h" @@ -36,16 +37,22 @@ cProcessor::cThread::cThread(cCallback & a_Callback, cProcessor & a_ParentProces void cProcessor::cThread::Execute(void) { + LOG("Started a new thread: %d", cIsThread::GetCurrentID()); + + m_ParentProcessor.m_ThreadsHaveStarted.Set(); + for (;;) { AString FileName = m_ParentProcessor.GetOneFileName(); if (FileName.empty()) { // All done, terminate the thread - return; + break; } ProcessFile(FileName); } // for-ever + + LOG("Thread %d terminated", cIsThread::GetCurrentID()); } @@ -264,6 +271,8 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars return false; } + bool SectionProcessed[16]; + memset(SectionProcessed, 0, sizeof(SectionProcessed)); for (int Tag = a_NBT.GetFirstChild(Sections); Tag > 0; Tag = a_NBT.GetNextSibling(Tag)) { int YTag = a_NBT.FindChildByName(Tag, "Y"); @@ -278,8 +287,14 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars continue; } + unsigned char SectionY = a_NBT.GetByte(YTag); + if (SectionY >= 16) + { + LOG("WARNING: Section Y >= 16 (%d), high world, wtf? Skipping section!", SectionY); + continue; + } if (m_Callback.OnSection( - a_NBT.GetByte(YTag), + SectionY, (const BLOCKTYPE *) (a_NBT.GetData(BlocksTag)), (AddTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(AddTag)) : NULL, (const NIBBLETYPE *)(a_NBT.GetData(DataTag)), @@ -289,8 +304,18 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars { return true; } + SectionProcessed[SectionY] = true; } // for Tag - Sections[] + // Call the callback for empty sections: + for (unsigned char y = 0; y < 16; y++) + { + if (!SectionProcessed[y]) + { + m_Callback.OnEmptySection(y); + } + } + return false; } @@ -322,26 +347,18 @@ void cProcessor::ProcessWorld(const AString & a_WorldFolder, cCallbackFactory & { PopulateFileQueue(a_WorldFolder); - // Start as many threads as there are cores: - // Get number of cores by querying the system process affinity mask - DWORD Affinity, ProcAffinity; - GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity); - while (Affinity > 0) + // Start as many threads as there are cores, plus one: + // (One more thread can be in the file-read IO block while all other threads crunch the numbers) + int NumThreads = GetNumCores() + 1; + for (int i = 0; i < NumThreads; i++) { - if ((Affinity & 1) == 1) - { - cCallback * Callback = a_CallbackFactory.GetNewCallback(); - m_Threads.push_back(new cThread(*Callback, *this)); - } - Affinity >>= 1; - } // while (Affinity > 0) - if (m_Threads.size() == 0) - { - LOG("Zero cores detected - how am I running? Running in a single thread."); cCallback * Callback = a_CallbackFactory.GetNewCallback(); m_Threads.push_back(new cThread(*Callback, *this)); } + // Wait for the first thread to start processing: + m_ThreadsHaveStarted.Wait(); + // Wait for all threads to finish // simply by calling each thread's destructor sequentially for (cThreads::iterator itr = m_Threads.begin(), end = m_Threads.end(); itr != end; ++itr) @@ -359,7 +376,10 @@ void cProcessor::PopulateFileQueue(const AString & a_WorldFolder) LOG("Processing world in \"%s\"...", a_WorldFolder.c_str()); AString Path = a_WorldFolder; - Path.push_back(cFile::PathSeparator); + if (!Path.empty() && (Path[Path.length() - 1] != cFile::PathSeparator)) + { + Path.push_back(cFile::PathSeparator); + } AStringList AllFiles = GetDirectoryContents(Path.c_str()); for (AStringList::iterator itr = AllFiles.begin(), end = AllFiles.end(); itr != end; ++itr) { diff --git a/AnvilStats/Processor.h b/AnvilStats/Processor.h index 09dc980ec..4f5496bc6 100644 --- a/AnvilStats/Processor.h +++ b/AnvilStats/Processor.h @@ -60,6 +60,7 @@ protected: AStringList m_FileQueue; cThreads m_Threads; + cEvent m_ThreadsHaveStarted; // This is signalled by each thread to notify the parent thread that it can start waiting for those threads void PopulateFileQueue(const AString & a_WorldFolder); diff --git a/AnvilStats/Statistics.cpp b/AnvilStats/Statistics.cpp index 6c3b61778..222b3796b 100644 --- a/AnvilStats/Statistics.cpp +++ b/AnvilStats/Statistics.cpp @@ -5,6 +5,7 @@ #include "Globals.h" #include "Statistics.h" +#include "Utils.h" @@ -77,20 +78,42 @@ bool cStatistics::OnSection { for (int x = 0; x < 16; x++) { - unsigned char Biome = m_BiomeData[x + 16 * z]; // Cannot use cChunkDef, different data size + unsigned char Biome = m_BiomeData[x + 16 * z]; // Cannot use cChunkDef, different datatype unsigned char BlockType = cChunkDef::GetBlock(a_BlockTypes, x, y, z); - if (BlockType == 12) - { - __asm nop; - } m_BlockCounts[Biome][BlockType] += 1; } } } + m_BlockNumChunks += m_IsFirstSectionInChunk ? 1 : 0; m_IsFirstSectionInChunk = false; - return true; + return false; +} + + + + + +bool cStatistics::OnEmptySection(unsigned char a_Y) +{ + if (!m_IsBiomesValid) + { + // The current biome data is not valid, we don't have the means for sorting the BlockTypes into per-biome arrays + return true; + } + + // Add air to all columns: + for (int z = 0; z < 16; z++) + { + for (int x = 0; x < 16; x++) + { + unsigned char Biome = m_BiomeData[x + 16 * z]; // Cannot use cChunkDef, different datatype + m_BlockCounts[Biome][0] += 16; // 16 blocks in a column, all air + } + } + + return false; } @@ -102,13 +125,26 @@ bool cStatistics::OnSection cStatisticsFactory::~cStatisticsFactory() { - // TODO: Join the results together and export + // Join the results together: LOG("cStatistics:"); LOG(" Joining results..."); JoinResults(); LOG(" Total %d chunks went through", m_TotalChunks); LOG(" Biomes processed for %d chunks", m_BiomeNumChunks); - LOG(" BlockIDs processed for %d chunks", m_BlockNumChunks); + + // Check the number of blocks processed + Int64 TotalBlocks = 0; + for (int i = 0; i <= 255; i++) + { + for (int j = 0; j < 255; j++) + { + TotalBlocks += m_BlockCounts[i][j]; + } + } + Int64 ExpTotalBlocks = (Int64)m_BlockNumChunks * 16LL * 16LL * 256LL; + LOG(" BlockIDs processed for %d chunks, %lld blocks (exp %lld; %s)", m_BlockNumChunks, TotalBlocks, ExpTotalBlocks, (TotalBlocks == ExpTotalBlocks) ? "match" : "failed"); + + // Save statistics: LOG(" Saving statistics into files:"); LOG(" Biomes.txt"); SaveBiomes(); @@ -158,13 +194,19 @@ void cStatisticsFactory::SaveBiomes(void) cFile f; if (!f.Open("Biomes.xls", cFile::fmWrite)) { - LOG("Cannot write to file Biomes.txt. Statistics not written."); + LOG("Cannot write to file Biomes.xls. Statistics not written."); return; } + double TotalColumns = (double)m_BiomeNumChunks * 16 * 16; // Total number of columns processed + if (TotalColumns < 1) + { + // Avoid division by zero + TotalColumns = 1; + } for (int i = 0; i <= 255; i++) { AString Line; - Printf(Line, "%d\t%d\n", i, m_BiomeCounts[i]); + Printf(Line, "%s\t%d\t%.05f\n", GetBiomeString(i), i, m_BiomeCounts[i], ((double)m_BiomeCounts[i]) / TotalColumns); f.Write(Line.c_str(), Line.length()); } } @@ -178,9 +220,15 @@ void cStatisticsFactory::SaveBlockTypes(void) cFile f; if (!f.Open("BlockTypes.xls", cFile::fmWrite)) { - LOG("Cannot write to file Biomes.txt. Statistics not written."); + LOG("Cannot write to file Biomes.xls. Statistics not written."); return; } + double TotalBlocks = ((double)m_BlockNumChunks) * 16 * 16 * 256 / 100; // Total number of blocks processed + if (TotalBlocks < 1) + { + // Avoid division by zero + TotalBlocks = 1; + } for (int i = 0; i <= 255; i++) { int Count = 0; @@ -189,7 +237,7 @@ void cStatisticsFactory::SaveBlockTypes(void) Count += m_BlockCounts[Biome][i]; } AString Line; - Printf(Line, "%d\t%d\n", i, Count); + Printf(Line, "%s\t%d\t%d\t%.08f\n", GetBlockTypeString(i), i, Count, ((double)Count) / TotalBlocks); f.Write(Line.c_str(), Line.length()); } // TODO @@ -201,8 +249,72 @@ void cStatisticsFactory::SaveBlockTypes(void) void cStatisticsFactory::SaveBiomeBlockTypes(void) { - LOG("Not implemented yet!"); - // TODO + // Export as two tables: biomes 0-127 and 128-255, because OpenOffice doesn't support more than 256 columns + cFile f; + if (!f.Open("BiomeBlockTypes.xls", cFile::fmWrite)) + { + LOG("Cannot write to file BiomeBlockTypes.xls. Statistics not written."); + return; + } + + AString FileHeader("Biomes 0-127:\n"); + f.Write(FileHeader.c_str(), FileHeader.length()); + + AString Header("BlockType\tBlockType"); + for (int Biome = 0; Biome <= 127; Biome++) + { + const char * BiomeName = GetBiomeString(Biome); + if ((BiomeName != NULL) && (BiomeName[0] != 0)) + { + AppendPrintf(Header, "\t%s (%d)", BiomeName, Biome); + } + else + { + AppendPrintf(Header, "\t%d", Biome); + } + } + Header.append("\n"); + f.Write(Header.c_str(), Header.length()); + + for (int BlockType = 0; BlockType <= 255; BlockType++) + { + AString Line; + Printf(Line, "%s\t%d", GetBlockTypeString(BlockType), BlockType); + for (int Biome = 0; Biome <= 127; Biome++) + { + AppendPrintf(Line, "\t%d", m_BlockCounts[Biome][BlockType]); + } + Line.append("\n"); + f.Write(Line.c_str(), Line.length()); + } + + Header.assign("\n\nBiomes 127-255:\nBlockType\tBlockType"); + for (int Biome = 0; Biome <= 127; Biome++) + { + const char * BiomeName = GetBiomeString(Biome); + if ((BiomeName != NULL) && (BiomeName[0] != 0)) + { + AppendPrintf(Header, "\t%s (%d)", BiomeName, Biome); + } + else + { + AppendPrintf(Header, "\t%d", Biome); + } + } + Header.append("\n"); + f.Write(Header.c_str(), Header.length()); + + for (int BlockType = 0; BlockType <= 255; BlockType++) + { + AString Line; + Printf(Line, "%s\t%d", GetBlockTypeString(BlockType), BlockType); + for (int Biome = 128; Biome <= 255; Biome++) + { + AppendPrintf(Line, "\t%d", m_BlockCounts[Biome][BlockType]); + } + Line.append("\n"); + f.Write(Line.c_str(), Line.length()); + } } diff --git a/AnvilStats/Statistics.h b/AnvilStats/Statistics.h index 980241ae6..38f69615c 100644 --- a/AnvilStats/Statistics.h +++ b/AnvilStats/Statistics.h @@ -51,6 +51,7 @@ protected: const NIBBLETYPE * a_BlockLight, const NIBBLETYPE * a_BlockSkyLight ) override; + virtual bool OnEmptySection(unsigned char a_Y) override; } ; diff --git a/AnvilStats/Utils.cpp b/AnvilStats/Utils.cpp new file mode 100644 index 000000000..e6fe296e0 --- /dev/null +++ b/AnvilStats/Utils.cpp @@ -0,0 +1,225 @@ + +// Utils.cpp + +// Implements utility functions + +#include "Globals.h" +#include "Utils.h" + + + + + +const char * GetBiomeString(unsigned char a_Biome) +{ + static const char * BiomeNames[] = // Biome names, as equivalent to their index + { + "Ocean", + "Plains", + "Desert", + "Extreme Hills", + "Forest", + "Taiga", + "Swampland", + "River", + "Hell", + "Sky", + "Frozen Ocean", + "Frozen River", + "Ice Plains", + "Ice Mountains", + "Mushroom Island", + "Mushroom Island Shore", + "Beach", + "Desert Hills", + "Forest Hills", + "Taiga Hills", + "Extreme Hills Edge", + "Jungle", + "Jungle Hills", + } ; + return (a_Biome < ARRAYCOUNT(BiomeNames)) ? BiomeNames[a_Biome] : ""; +} + + + + + +const char * GetBlockTypeString(unsigned char a_BlockType) +{ + static const char * BlockTypeNames[] = // Block type names, as equivalent to their index + { + "air", + "stone", + "grass", + "dirt", + "cobblestone", + "planks", + "sapling", + "bedrock", + "water", + "stillwater", + "lava", + "stilllava", + "sand", + "gravel", + "goldore", + "ironore", + "coalore", + "log", + "leaves", + "sponge", + "glass", + "lapisore", + "lapisblock", + "dispenser", + "sandstone", + "noteblock", + "bedblock", + "poweredrail", + "detectorrail", + "stickypiston", + "cobweb", + "tallgrass", + "deadbush", + "piston", + "pistonhead", + "wool", + "pistonmovedblock", + "flower", + "rose", + "brownmushroom", + "redmushroom", + "goldblock", + "ironblock", + "doubleslab", + "slab", + "brickblock", + "tnt", + "bookcase", + "mossycobblestone", + "obsidian", + "torch", + "fire", + "mobspawner", + "woodstairs", + "chest", + "redstonedust", + "diamondore", + "diamondblock", + "workbench", + "crops", + "soil", + "furnace", + "litfurnace", + "signblock", + "wooddoorblock", + "ladder", + "tracks", + "cobblestonestairs", + "wallsign", + "lever", + "stoneplate", + "irondoorblock", + "woodplate", + "redstoneore", + "redstoneorealt", + "redstonetorchoff", + "redstonetorchon", + "button", + "snow", + "ice", + "snowblock", + "cactus", + "clayblock", + "reedblock", + "jukebox", + "fence", + "pumpkin", + "netherrack", + "soulsand", + "glowstone", + "portal", + "jack-o-lantern", + "cakeblock", + "repeateroff", + "repeateron", + "lockedchest", + "trapdoor", + "silverfishblock", + "stonebricks", + "hugebrownmushroom", + "hugeredmushroom", + "ironbars", + "glasspane", + "melon", + "pumpkinstem", + "melonstem", + "vines", + "fencegate", + "brickstairs", + "stonebrickstairs", + "mycelium", + "lilypad", + "netherbrick", + "netherbrickfence", + "netherbrickstairs", + "netherwartblock", + "enchantmenttable", + "brewingstandblock", + "cauldronblock", + "endportal", + "endportalframe", + "endstone", + "dragonegg", + "redstonelampoff", + "redstonelampon", + "woodendoubleslab", + "woodenslab", + "cocoapod", + "sandstonestairs", /* 128 */ + "Emerald Ore", + "Ender Chest", + "Tripwire Hook", + "Tripwire", + "Block of Emerald", + "Spruce Wood Stairs", + "Birch Wood Stairs", + "Jungle Wood Stairs", + "Command Block", + "Beacon", + "Cobblestone Wall", + "Flower Pot", + "Carrots", + "Potatoes", + "Wooden Button", + "Head", + } ; + + return (a_BlockType < ARRAYCOUNT(BlockTypeNames)) ? BlockTypeNames[a_BlockType] : ""; +} + + + + + +int GetNumCores(void) +{ + // Get number of cores by querying the system process affinity mask (Windows-specific) + DWORD Affinity, ProcAffinity; + GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity); + int NumCores = 0; + while (Affinity > 0) + { + if ((Affinity & 1) == 1) + { + ++NumCores; + } + Affinity >>= 1; + } // while (Affinity > 0) + return NumCores; +} + + + + diff --git a/AnvilStats/Utils.h b/AnvilStats/Utils.h new file mode 100644 index 000000000..c22a9ab58 --- /dev/null +++ b/AnvilStats/Utils.h @@ -0,0 +1,15 @@ + +// Utils.h + +// Interfaces to utility functions + + + + + +extern const char * GetBiomeString(unsigned char a_Biome); +extern const char * GetBlockTypeString(unsigned char a_BlockType); +extern int GetNumCores(void); + + +