AnvilStats: complete per-biome blocktype statistics

git-svn-id: http://mc-server.googlecode.com/svn/trunk@897 0a769ca7-a7f5-676a-18bf-c427514a06d6

AnvilStats/AnvilStats.vcproj

@@ -226,6 +226,14 @@
 				RelativePath=".\Statistics.h"
 				>
 			</File>
+			<File
+				RelativePath=".\Utils.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\Utils.h"
+				>
+			</File>
 		</Filter>
 		<Filter
 			Name="shared"
@@ -242,6 +250,14 @@
 				RelativePath="..\source\Endianness.h"
 				>
 			</File>
+			<File
+				RelativePath="..\source\OSSupport\Event.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\source\OSSupport\Event.h"
+				>
+			</File>
 			<File
 				RelativePath="..\source\WorldStorage\FastNBT.cpp"
 				>

AnvilStats/Callback.h

@@ -51,6 +51,9 @@ public:
 	
 	virtual bool OnHeightMap(const int * a_HeightMap) { return true; }
 	
+	/** If there is data for the section, this callback is called; otherwise OnEmptySection() is called instead.
+	All OnSection() callbacks are called first, and only then all the remaining sections are reported in OnEmptySection().
+	*/
 	virtual bool OnSection(
 		unsigned char a_Y,
 		const BLOCKTYPE * a_BlockTypes,
@@ -60,6 +63,11 @@ public:
 		const NIBBLETYPE * a_BlockSkyLight
 	) { return true; }
 	
+	/** If there is no data for a section, this callback is called; otherwise OnSection() is called instead. 
+	OnEmptySection() callbacks are called after all OnSection() callbacks.
+	*/
+	virtual bool OnEmptySection(unsigned char a_Y) { return false; }
+	
 	// TODO: entities, tile-entities, tile-ticks
 } ;
 

AnvilStats/Globals.h

@@ -172,6 +172,7 @@ typedef short     Int16;
 #define LOGERROR LOG
 #define LOGWARNING LOG
 #define LOGINFO LOG
+#define LOGWARN LOG
 
 /// Evaluates to the number of elements in an array (compile-time!)
 #define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X)))

AnvilStats/Processor.cpp

@@ -8,6 +8,7 @@
 #include "Callback.h"
 #include "../source/WorldStorage/FastNBT.h"
 #include "zlib.h"
+#include "Utils.h"
 
 
 
@@ -36,16 +37,22 @@ cProcessor::cThread::cThread(cCallback & a_Callback, cProcessor & a_ParentProces
 
 void cProcessor::cThread::Execute(void)
 {
+	LOG("Started a new thread: %d", cIsThread::GetCurrentID());
+	
+	m_ParentProcessor.m_ThreadsHaveStarted.Set();
+	
 	for (;;)
 	{
 		AString FileName = m_ParentProcessor.GetOneFileName();
 		if (FileName.empty())
 		{
 			// All done, terminate the thread
-			return;
+			break;
 		}
 		ProcessFile(FileName);
 	}  // for-ever
+	
+	LOG("Thread %d terminated", cIsThread::GetCurrentID());
 }
 
 
@@ -264,6 +271,8 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars
 		return false;
 	}
 	
+	bool SectionProcessed[16];
+	memset(SectionProcessed, 0, sizeof(SectionProcessed));
 	for (int Tag = a_NBT.GetFirstChild(Sections); Tag > 0; Tag = a_NBT.GetNextSibling(Tag))
 	{
 		int YTag          = a_NBT.FindChildByName(Tag, "Y");
@@ -278,8 +287,14 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars
 			continue;
 		}
 		
+		unsigned char SectionY = a_NBT.GetByte(YTag);
+		if (SectionY >= 16)
+		{
+			LOG("WARNING: Section Y >= 16 (%d), high world, wtf? Skipping section!", SectionY);
+			continue;
+		}
 		if (m_Callback.OnSection(
-			a_NBT.GetByte(YTag),
+			SectionY,
 			(const BLOCKTYPE *) (a_NBT.GetData(BlocksTag)),
 			(AddTag > 0) ? (const NIBBLETYPE *)(a_NBT.GetData(AddTag)) : NULL,
 			(const NIBBLETYPE *)(a_NBT.GetData(DataTag)),
@@ -289,8 +304,18 @@ bool cProcessor::cThread::ProcessChunkSections(int a_ChunkX, int a_ChunkZ, cPars
 		{
 			return true;
 		}
+		SectionProcessed[SectionY] = true;
 	}  // for Tag - Sections[]
 	
+	// Call the callback for empty sections:
+	for (unsigned char y = 0; y < 16; y++)
+	{
+		if (!SectionProcessed[y])
+		{
+			m_Callback.OnEmptySection(y);
+		}
+	}
+	
 	return false;
 }
 
@@ -322,26 +347,18 @@ void cProcessor::ProcessWorld(const AString & a_WorldFolder, cCallbackFactory &
 {
 	PopulateFileQueue(a_WorldFolder);
 	
-	// Start as many threads as there are cores:
+	// Start as many threads as there are cores, plus one:
-	// Get number of cores by querying the system process affinity mask
+	// (One more thread can be in the file-read IO block while all other threads crunch the numbers)
-	DWORD Affinity, ProcAffinity;
+	int NumThreads = GetNumCores() + 1;
-	GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity);
+	for (int i = 0; i < NumThreads; i++)
-	while (Affinity > 0)
 	{
-		if ((Affinity & 1) == 1)
-		{
-			cCallback * Callback = a_CallbackFactory.GetNewCallback();
-			m_Threads.push_back(new cThread(*Callback, *this));
-		}
-		Affinity >>= 1;
-	}  // while (Affinity > 0)
-	if (m_Threads.size() == 0)
-	{
-		LOG("Zero cores detected - how am I running? Running in a single thread.");
 		cCallback * Callback = a_CallbackFactory.GetNewCallback();
 		m_Threads.push_back(new cThread(*Callback, *this));
 	}
 
+	// Wait for the first thread to start processing:
+	m_ThreadsHaveStarted.Wait();
+	
 	// Wait for all threads to finish
 	// simply by calling each thread's destructor sequentially
 	for (cThreads::iterator itr = m_Threads.begin(), end = m_Threads.end(); itr != end; ++itr)
@@ -359,7 +376,10 @@ void cProcessor::PopulateFileQueue(const AString & a_WorldFolder)
 	LOG("Processing world in \"%s\"...", a_WorldFolder.c_str());
 
 	AString Path = a_WorldFolder;
-	Path.push_back(cFile::PathSeparator);
+	if (!Path.empty() && (Path[Path.length() - 1] != cFile::PathSeparator))
+	{
+		Path.push_back(cFile::PathSeparator);
+	}
 	AStringList AllFiles = GetDirectoryContents(Path.c_str());
 	for (AStringList::iterator itr = AllFiles.begin(), end = AllFiles.end(); itr != end; ++itr)
 	{

AnvilStats/Processor.h

@@ -60,6 +60,7 @@ protected:
 	AStringList      m_FileQueue;
 	
 	cThreads m_Threads;
+	cEvent   m_ThreadsHaveStarted;  // This is signalled by each thread to notify the parent thread that it can start waiting for those threads
 	
 	void PopulateFileQueue(const AString & a_WorldFolder);
 	

AnvilStats/Statistics.cpp

@@ -5,6 +5,7 @@
 
 #include "Globals.h"
 #include "Statistics.h"
+#include "Utils.h"
 
 
 
@@ -77,20 +78,42 @@ bool cStatistics::OnSection
 		{
 			for (int x = 0; x < 16; x++)
 			{
-				unsigned char Biome = m_BiomeData[x + 16 * z];  // Cannot use cChunkDef, different data size
+				unsigned char Biome = m_BiomeData[x + 16 * z];  // Cannot use cChunkDef, different datatype
 				unsigned char BlockType = cChunkDef::GetBlock(a_BlockTypes, x, y, z);
-				if (BlockType == 12)
-				{
-					__asm nop;
-				}
 				m_BlockCounts[Biome][BlockType] += 1;
 			}
 		}
 	}
+	
 	m_BlockNumChunks += m_IsFirstSectionInChunk ? 1 : 0;
 	m_IsFirstSectionInChunk = false;
 	
-	return true;
+	return false;
+}
+
+
+
+
+
+bool cStatistics::OnEmptySection(unsigned char a_Y)
+{
+	if (!m_IsBiomesValid)
+	{
+		// The current biome data is not valid, we don't have the means for sorting the BlockTypes into per-biome arrays
+		return true;
+	}
+
+	// Add air to all columns:	
+	for (int z = 0; z < 16; z++)
+	{
+		for (int x = 0; x < 16; x++)
+		{
+			unsigned char Biome = m_BiomeData[x + 16 * z];  // Cannot use cChunkDef, different datatype
+			m_BlockCounts[Biome][0] += 16;  // 16 blocks in a column, all air
+		}
+	}
+	
+	return false;
 }
 
 
@@ -102,13 +125,26 @@ bool cStatistics::OnSection
 
 cStatisticsFactory::~cStatisticsFactory()
 {
-	// TODO: Join the results together and export
+	// Join the results together:
 	LOG("cStatistics:");
 	LOG("  Joining results...");
 	JoinResults();
 	LOG("  Total %d chunks went through", m_TotalChunks);
 	LOG("  Biomes processed for %d chunks", m_BiomeNumChunks);
-	LOG("  BlockIDs processed for %d chunks", m_BlockNumChunks);
+	
+	// Check the number of blocks processed
+	Int64 TotalBlocks = 0;
+	for (int i = 0; i <= 255; i++)
+	{
+		for (int j = 0; j < 255; j++)
+		{
+			TotalBlocks += m_BlockCounts[i][j];
+		}
+	}
+	Int64 ExpTotalBlocks = (Int64)m_BlockNumChunks * 16LL * 16LL * 256LL;
+	LOG("  BlockIDs processed for %d chunks, %lld blocks (exp %lld; %s)", m_BlockNumChunks, TotalBlocks, ExpTotalBlocks, (TotalBlocks == ExpTotalBlocks) ? "match" : "failed");
+	
+	// Save statistics:
 	LOG("  Saving statistics into files:");
 	LOG("    Biomes.txt");
 	SaveBiomes();
@@ -158,13 +194,19 @@ void cStatisticsFactory::SaveBiomes(void)
 	cFile f;
 	if (!f.Open("Biomes.xls", cFile::fmWrite))
 	{
-		LOG("Cannot write to file Biomes.txt. Statistics not written.");
+		LOG("Cannot write to file Biomes.xls. Statistics not written.");
 		return;
 	}
+	double TotalColumns = (double)m_BiomeNumChunks * 16 * 16;  // Total number of columns processed
+	if (TotalColumns < 1)
+	{
+		// Avoid division by zero
+		TotalColumns = 1;
+	}
 	for (int i = 0; i <= 255; i++)
 	{
 		AString Line;
-		Printf(Line, "%d\t%d\n", i, m_BiomeCounts[i]);
+		Printf(Line, "%s\t%d\t%.05f\n", GetBiomeString(i), i, m_BiomeCounts[i], ((double)m_BiomeCounts[i]) / TotalColumns);
 		f.Write(Line.c_str(), Line.length());
 	}
 }
@@ -178,9 +220,15 @@ void cStatisticsFactory::SaveBlockTypes(void)
 	cFile f;
 	if (!f.Open("BlockTypes.xls", cFile::fmWrite))
 	{
-		LOG("Cannot write to file Biomes.txt. Statistics not written.");
+		LOG("Cannot write to file Biomes.xls. Statistics not written.");
 		return;
 	}
+	double TotalBlocks = ((double)m_BlockNumChunks) * 16 * 16 * 256 / 100;  // Total number of blocks processed
+	if (TotalBlocks < 1)
+	{
+		// Avoid division by zero
+		TotalBlocks = 1;
+	}
 	for (int i = 0; i <= 255; i++)
 	{
 		int Count = 0;
@@ -189,7 +237,7 @@ void cStatisticsFactory::SaveBlockTypes(void)
 			Count += m_BlockCounts[Biome][i];
 		}
 		AString Line;
-		Printf(Line, "%d\t%d\n", i, Count);
+		Printf(Line, "%s\t%d\t%d\t%.08f\n", GetBlockTypeString(i), i, Count, ((double)Count) / TotalBlocks);
 		f.Write(Line.c_str(), Line.length());
 	}
 	// TODO
@@ -201,8 +249,72 @@ void cStatisticsFactory::SaveBlockTypes(void)
 
 void cStatisticsFactory::SaveBiomeBlockTypes(void)
 {
-	LOG("Not implemented yet!");
+	// Export as two tables: biomes 0-127 and 128-255, because OpenOffice doesn't support more than 256 columns
-	// TODO
+	cFile f;
+	if (!f.Open("BiomeBlockTypes.xls", cFile::fmWrite))
+	{
+		LOG("Cannot write to file BiomeBlockTypes.xls. Statistics not written.");
+		return;
+	}
+	
+	AString FileHeader("Biomes 0-127:\n");
+	f.Write(FileHeader.c_str(), FileHeader.length());
+	
+	AString Header("BlockType\tBlockType");
+	for (int Biome = 0; Biome <= 127; Biome++)
+	{
+		const char * BiomeName = GetBiomeString(Biome);
+		if ((BiomeName != NULL) && (BiomeName[0] != 0))
+		{
+			AppendPrintf(Header, "\t%s (%d)", BiomeName, Biome);
+		}
+		else
+		{
+			AppendPrintf(Header, "\t%d", Biome);
+		}
+	}
+	Header.append("\n");
+	f.Write(Header.c_str(), Header.length());
+	
+	for (int BlockType = 0; BlockType <= 255; BlockType++)
+	{
+		AString Line;
+		Printf(Line, "%s\t%d", GetBlockTypeString(BlockType), BlockType);
+		for (int Biome = 0; Biome <= 127; Biome++)
+		{
+			AppendPrintf(Line, "\t%d", m_BlockCounts[Biome][BlockType]);
+		}
+		Line.append("\n");
+		f.Write(Line.c_str(), Line.length());
+	}
+
+	Header.assign("\n\nBiomes 127-255:\nBlockType\tBlockType");
+	for (int Biome = 0; Biome <= 127; Biome++)
+	{
+		const char * BiomeName = GetBiomeString(Biome);
+		if ((BiomeName != NULL) && (BiomeName[0] != 0))
+		{
+			AppendPrintf(Header, "\t%s (%d)", BiomeName, Biome);
+		}
+		else
+		{
+			AppendPrintf(Header, "\t%d", Biome);
+		}
+	}
+	Header.append("\n");
+	f.Write(Header.c_str(), Header.length());
+	
+	for (int BlockType = 0; BlockType <= 255; BlockType++)
+	{
+		AString Line;
+		Printf(Line, "%s\t%d", GetBlockTypeString(BlockType), BlockType);
+		for (int Biome = 128; Biome <= 255; Biome++)
+		{
+			AppendPrintf(Line, "\t%d", m_BlockCounts[Biome][BlockType]);
+		}
+		Line.append("\n");
+		f.Write(Line.c_str(), Line.length());
+	}
 }
 
 

AnvilStats/Statistics.h

@@ -51,6 +51,7 @@ protected:
 		const NIBBLETYPE * a_BlockLight,
 		const NIBBLETYPE * a_BlockSkyLight
 	) override;
+	virtual bool OnEmptySection(unsigned char a_Y) override;
 } ;
 
 

AnvilStats/Utils.cpp

@@ -0,0 +1,225 @@
+
+// Utils.cpp
+
+// Implements utility functions
+
+#include "Globals.h"
+#include "Utils.h"
+
+
+
+
+
+const char * GetBiomeString(unsigned char a_Biome)
+{
+	static const char * BiomeNames[] =  // Biome names, as equivalent to their index
+	{
+		"Ocean",
+		"Plains",
+		"Desert",
+		"Extreme Hills",
+		"Forest",
+		"Taiga",
+		"Swampland",
+		"River",
+		"Hell",
+		"Sky",
+		"Frozen Ocean",
+		"Frozen River",
+		"Ice Plains",
+		"Ice Mountains",
+		"Mushroom Island",
+		"Mushroom Island Shore",
+		"Beach",
+		"Desert Hills",
+		"Forest Hills",
+		"Taiga Hills",
+		"Extreme Hills Edge",
+		"Jungle",
+		"Jungle Hills",
+	} ;
+	return (a_Biome < ARRAYCOUNT(BiomeNames)) ? BiomeNames[a_Biome] : "";
+}
+
+
+
+
+
+const char * GetBlockTypeString(unsigned char a_BlockType)
+{
+	static const char * BlockTypeNames[] =  // Block type names, as equivalent to their index
+	{
+		"air",
+		"stone",
+		"grass",
+		"dirt",
+		"cobblestone",
+		"planks",
+		"sapling",
+		"bedrock",
+		"water",
+		"stillwater",
+		"lava",
+		"stilllava",
+		"sand",
+		"gravel",
+		"goldore",
+		"ironore",
+		"coalore",
+		"log",
+		"leaves",
+		"sponge",
+		"glass",
+		"lapisore",
+		"lapisblock",
+		"dispenser",
+		"sandstone",
+		"noteblock",
+		"bedblock",
+		"poweredrail",
+		"detectorrail",
+		"stickypiston",
+		"cobweb",
+		"tallgrass",
+		"deadbush",
+		"piston",
+		"pistonhead",
+		"wool",
+		"pistonmovedblock",
+		"flower",
+		"rose",
+		"brownmushroom",
+		"redmushroom",
+		"goldblock",
+		"ironblock",
+		"doubleslab",
+		"slab",
+		"brickblock",
+		"tnt",
+		"bookcase",
+		"mossycobblestone",
+		"obsidian",
+		"torch",
+		"fire",
+		"mobspawner",
+		"woodstairs",
+		"chest",
+		"redstonedust",
+		"diamondore",
+		"diamondblock",
+		"workbench",
+		"crops",
+		"soil",
+		"furnace",
+		"litfurnace",
+		"signblock",
+		"wooddoorblock",
+		"ladder",
+		"tracks",
+		"cobblestonestairs",
+		"wallsign",
+		"lever",
+		"stoneplate",
+		"irondoorblock",
+		"woodplate",
+		"redstoneore",
+		"redstoneorealt",
+		"redstonetorchoff",
+		"redstonetorchon",
+		"button",
+		"snow",
+		"ice",
+		"snowblock",
+		"cactus",
+		"clayblock",
+		"reedblock",
+		"jukebox",
+		"fence",
+		"pumpkin",
+		"netherrack",
+		"soulsand",
+		"glowstone",
+		"portal",
+		"jack-o-lantern",
+		"cakeblock",
+		"repeateroff",
+		"repeateron",
+		"lockedchest",
+		"trapdoor",
+		"silverfishblock",
+		"stonebricks",
+		"hugebrownmushroom",
+		"hugeredmushroom",
+		"ironbars",
+		"glasspane",
+		"melon",
+		"pumpkinstem",
+		"melonstem",
+		"vines",
+		"fencegate",
+		"brickstairs",
+		"stonebrickstairs",
+		"mycelium",
+		"lilypad",
+		"netherbrick",
+		"netherbrickfence",
+		"netherbrickstairs",
+		"netherwartblock",
+		"enchantmenttable",
+		"brewingstandblock",
+		"cauldronblock",
+		"endportal",
+		"endportalframe",
+		"endstone",
+		"dragonegg",
+		"redstonelampoff",
+		"redstonelampon",
+		"woodendoubleslab",
+		"woodenslab",
+		"cocoapod",
+		"sandstonestairs",  /* 128 */
+		"Emerald Ore",
+		"Ender Chest",
+		"Tripwire Hook",
+		"Tripwire",
+		"Block of Emerald",
+		"Spruce Wood Stairs",
+		"Birch Wood Stairs",
+		"Jungle Wood Stairs",
+		"Command Block",
+		"Beacon",
+		"Cobblestone Wall",
+		"Flower Pot",
+		"Carrots",
+		"Potatoes",
+		"Wooden Button",
+		"Head",
+	} ;
+	
+	return (a_BlockType < ARRAYCOUNT(BlockTypeNames)) ? BlockTypeNames[a_BlockType] : "";
+}
+
+
+
+
+
+int GetNumCores(void)
+{
+	// Get number of cores by querying the system process affinity mask (Windows-specific)
+	DWORD Affinity, ProcAffinity;
+	GetProcessAffinityMask(GetCurrentProcess(), &ProcAffinity, &Affinity);
+	int NumCores = 0;
+	while (Affinity > 0)
+	{
+		if ((Affinity & 1) == 1)
+		{
+			++NumCores;
+		}
+		Affinity >>= 1;
+	}  // while (Affinity > 0)
+	return NumCores;
+}
+
+
+
+

AnvilStats/Utils.h

@@ -0,0 +1,15 @@
+
+// Utils.h
+
+// Interfaces to utility functions
+
+
+
+
+
+extern const char * GetBiomeString(unsigned char a_Biome);
+extern const char * GetBlockTypeString(unsigned char a_BlockType);
+extern int GetNumCores(void);
+
+
+