flewkey/cuberite-2a
1
0
Fork 0
Code Releases Activity

MCADefrag: Initial implementation.

Browse Source 
Partially implements #639.
This only defragments the chunks, without recompressing them.
This commit is contained in:
madmaxoft 2014-02-13 11:47:20 +01:00
parent cc72b656c9
commit 05590fb91d
6 changed files with 799 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Tools/MCADefrag/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.mca

144
Tools/MCADefrag/CMakeLists.txt Normal file
View File

@ -0,0 +1,144 @@
cmake_minimum_required (VERSION 2.6)
project (MCADefrag)
macro(add_flags_cxx FLAGS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAGS}")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${FLAGS}")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${FLAGS}")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${FLAGS}")
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${FLAGS}")
endmacro()
# Add the preprocessor macros used for distinguishing between debug and release builds (CMake does this automatically for MSVC):
if (NOT MSVC)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG")
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG")
endif()
if(MSVC)
# Make build use multiple threads under MSVC:
add_flags_cxx("/MP")
# Make release builds use link-time code generation:
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL")
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /GL")
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG")
set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG")
set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} /LTCG")
elseif(APPLE)
#on os x clang adds pthread for us but we need to add it for gcc
if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
add_flags_cxx("-pthread")
endif()
else()
# Let gcc / clang know that we're compiling a multi-threaded app:
add_flags_cxx("-pthread")
endif()
# Use static CRT in MSVC builds:
if (MSVC)
string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
string(REPLACE "/MD" "/MT" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
string(REPLACE "/MDd" "/MTd" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
string(REPLACE "/MDd" "/MTd" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
endif()
# Set include paths to the used libraries:
include_directories("../../lib")
include_directories("../../src")
function(flatten_files arg1)
set(res "")
foreach(f ${${arg1}})
get_filename_component(f ${f} ABSOLUTE)
list(APPEND res ${f})
endforeach()
set(${arg1} "${res}" PARENT_SCOPE)
endfunction()
# Include the libraries:
file(GLOB ZLIB_SRC "../../lib/zlib/*.c")
file(GLOB ZLIB_HDR "../../lib/zlib/*.h")
flatten_files(ZLIB_SRC)
flatten_files(ZLIB_HDR)
source_group("ZLib" FILES ${ZLIB_SRC} ${ZLIB_HDR})
# Include the shared files:
set(SHARED_SRC
../../src/StringCompression.cpp
../../src/StringUtils.cpp
../../src/Log.cpp
../../src/MCLogger.cpp
)
set(SHARED_HDR
../../src/ByteBuffer.h
../../src/StringUtils.h
../../src/Log.h
../../src/MCLogger.h
)
set(SHARED_OSS_SRC
../../src/OSSupport/CriticalSection.cpp
../../src/OSSupport/File.cpp
../../src/OSSupport/IsThread.cpp
../../src/OSSupport/Timer.cpp
)
set(SHARED_OSS_HDR
../../src/OSSupport/CriticalSection.h
../../src/OSSupport/File.h
../../src/OSSupport/IsThread.h
../../src/OSSupport/Timer.h
)
flatten_files(SHARED_SRC)
flatten_files(SHARED_HDR)
flatten_files(SHARED_OSS_SRC)
flatten_files(SHARED_OSS_HDR)
source_group("Shared" FILES ${SHARED_SRC} ${SHARED_HDR})
source_group("Shared\\OSSupport" FILES ${SHARED_OSS_SRC} ${SHARED_OSS_HDR})
# Include the main source files:
set(SOURCES
MCADefrag.cpp
Globals.cpp
)
set(HEADERS
MCADefrag.h
Globals.h
)
source_group("" FILES ${SOURCES} ${HEADERS})
add_executable(MCADefrag
${SOURCES}
${HEADERS}
${SHARED_SRC}
${SHARED_HDR}
${SHARED_OSS_SRC}
${SHARED_OSS_HDR}
${ZLIB_SRC}
${ZLIB_HDR}
)

10
Tools/MCADefrag/Globals.cpp Normal file
View File

@ -0,0 +1,10 @@
// Globals.cpp
// This file is used for precompiled header generation in MSVC environments
#include "Globals.h"

229
Tools/MCADefrag/Globals.h Normal file
View File

@ -0,0 +1,229 @@
// Globals.h
// This file gets included from every module in the project, so that global symbols may be introduced easily
// Also used for precompiled header generation in MSVC environments
// Compiler-dependent stuff:
#if defined(_MSC_VER)
// MSVC produces warning C4481 on the override keyword usage, so disable the warning altogether
#pragma warning(disable:4481)
// Disable some warnings that we don't care about:
#pragma warning(disable:4100)
#define OBSOLETE __declspec(deprecated)
// No alignment needed in MSVC
#define ALIGN_8
#define ALIGN_16
#elif defined(__GNUC__)
// TODO: Can GCC explicitly mark classes as abstract (no instances can be created)?
#define abstract
// TODO: Can GCC mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
#define override
#define OBSOLETE __attribute__((deprecated))
#define ALIGN_8 __attribute__((aligned(8)))
#define ALIGN_16 __attribute__((aligned(16)))
// Some portability macros :)
#define stricmp strcasecmp
#else
#error "You are using an unsupported compiler, you might need to #define some stuff here for your compiler"
/*
// Copy and uncomment this into another #elif section based on your compiler identification
// Explicitly mark classes as abstract (no instances can be created)
#define abstract
// Mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class)
#define override
// Mark functions as obsolete, so that their usage results in a compile-time warning
#define OBSOLETE
// Mark types / variables for alignment. Do the platforms need it?
#define ALIGN_8
#define ALIGN_16
*/
#endif
// Integral types with predefined sizes:
typedef long long Int64;
typedef int Int32;
typedef short Int16;
typedef unsigned long long UInt64;
typedef unsigned int UInt32;
typedef unsigned short UInt16;
typedef unsigned char Byte;
// A macro to disallow the copy constructor and operator= functions
// This should be used in the private: declarations for any class that shouldn't allow copying itself
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName &); \
void operator=(const TypeName &)
// A macro that is used to mark unused function parameters, to avoid pedantic warnings in gcc
#define UNUSED(X) (void)(X)
// OS-dependent stuff:
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
// Windows SDK defines min and max macros, messing up with our std::min and std::max usage
#undef min
#undef max
// Windows SDK defines GetFreeSpace as a constant, probably a Win16 API remnant
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif // GetFreeSpace
#define SocketError WSAGetLastError()
#else
#include <sys/types.h>
#include <sys/stat.h> // for mkdir
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <time.h>
#include <dirent.h>
#include <errno.h>
#include <iostream>
#include <unistd.h>
#include <cstdio>
#include <cstring>
#include <pthread.h>
#include <semaphore.h>
#include <errno.h>
#include <fcntl.h>
typedef int SOCKET;
enum
{
INVALID_SOCKET = -1,
};
#define closesocket close
#define SocketError errno
#if !defined(ANDROID_NDK)
#include <tr1/memory>
#endif
#endif
#if !defined(ANDROID_NDK)
#define USE_SQUIRREL
#endif
#if defined(ANDROID_NDK)
#define FILE_IO_PREFIX "/sdcard/mcserver/"
#else
#define FILE_IO_PREFIX ""
#endif
// CRT stuff:
#include <assert.h>
#include <stdio.h>
#include <math.h>
#include <stdarg.h>
#include <time.h>
// STL stuff:
#include <vector>
#include <list>
#include <deque>
#include <string>
#include <map>
#include <algorithm>
#include <memory>
// Common headers (without macros):
#include "StringUtils.h"
#include "OSSupport/CriticalSection.h"
#include "OSSupport/IsThread.h"
#include "OSSupport/File.h"
// Common definitions:
/// Evaluates to the number of elements in an array (compile-time!)
#define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X)))
/// Allows arithmetic expressions like "32 KiB" (but consider using parenthesis around it, "(32 KiB)" )
#define KiB * 1024
#define MiB * 1024 * 1024
/// Faster than (int)floorf((float)x / (float)div)
#define FAST_FLOOR_DIV( x, div ) ( (x) < 0 ? (((int)x / div) - 1) : ((int)x / div) )
// Own version of assert() that writes failed assertions to the log for review
#ifdef NDEBUG
#define ASSERT(x) ((void)0)
#else
#define ASSERT assert
#endif
// Pretty much the same as ASSERT() but stays in Release builds
#define VERIFY( x ) ( !!(x) || ( LOGERROR("Verification failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), exit(1), 0 ) )
/// A generic interface used mainly in ForEach() functions
template <typename Type> class cItemCallback
{
public:
/// Called for each item in the internal list; return true to stop the loop, or false to continue enumerating
virtual bool Item(Type * a_Type) = 0;
} ;

303
Tools/MCADefrag/MCADefrag.cpp Normal file
View File

@ -0,0 +1,303 @@
// MCADefrag.cpp
// Implements the main app entrypoint and the cMCADefrag class representing the entire app
#include "Globals.h"
#include "MCADefrag.h"
#include "MCLogger.h"
int main(int argc, char ** argv)
{
new cMCLogger(Printf("Defrag_%08x.log", time(NULL)));
cMCADefrag Defrag;
if (!Defrag.Init(argc, argv))
{
return 1;
}
Defrag.Run();
return 0;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cMCADefrag:
cMCADefrag::cMCADefrag(void) :
m_NumThreads(1)
{
}
bool cMCADefrag::Init(int argc, char ** argv)
{
// Nothing needed yet
return true;
}
void cMCADefrag::Run(void)
{
// Fill the queue with MCA files
m_Queue = cFile::GetFolderContents(".");
// Start the processing threads:
for (int i = 0; i < m_NumThreads; i++)
{
StartThread();
}
// Wait for all the threads to finish:
while (!m_Threads.empty())
{
m_Threads.front()->Wait();
delete m_Threads.front();
m_Threads.pop_front();
}
}
void cMCADefrag::StartThread(void)
{
cThread * Thread = new cThread(*this);
m_Threads.push_back(Thread);
Thread->Start();
}
AString cMCADefrag::GetNextFileName(void)
{
cCSLock Lock(m_CS);
if (m_Queue.empty())
{
return AString();
}
AString res = m_Queue.back();
m_Queue.pop_back();
return res;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// cMCADefrag::cThread:
cMCADefrag::cThread::cThread(cMCADefrag & a_Parent) :
super("MCADefrag thread"),
m_Parent(a_Parent)
{
}
void cMCADefrag::cThread::Execute(void)
{
for (;;)
{
AString FileName = m_Parent.GetNextFileName();
if (FileName.empty())
{
return;
}
ProcessFile(FileName);
}
}
void cMCADefrag::cThread::ProcessFile(const AString & a_FileName)
{
// Filter out non-MCA files:
if ((a_FileName.length() < 4) || (a_FileName.substr(a_FileName.length() - 4, 4) != ".mca"))
{
return;
}
LOGINFO("%s", a_FileName.c_str());
// Open input and output files:
AString OutFileName = a_FileName + ".new";
cFile In, Out;
if (!In.Open(a_FileName, cFile::fmRead))
{
LOGWARNING("Cannot open file %s for reading, skipping file.", a_FileName.c_str());
return;
}
if (!Out.Open(OutFileName.c_str(), cFile::fmWrite))
{
LOGWARNING("Cannot open file %s for writing, skipping file.", OutFileName.c_str());
return;
}
// Read the Locations and Timestamps from the input file:
Byte Locations[4096];
UInt32 Timestamps[1024];
if (In.Read(Locations, sizeof(Locations)) != sizeof(Locations))
{
LOGWARNING("Cannot read Locations in file %s, skipping file.", a_FileName.c_str());
return;
}
if (In.Read(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps))
{
LOGWARNING("Cannot read Timestamps in file %s, skipping file.", a_FileName.c_str());
return;
}
// Write dummy Locations to the Out file (will be overwritten once the correct ones are known)
if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations))
{
LOGWARNING("Cannot write Locations to file %s, skipping file.", OutFileName.c_str());
return;
}
m_CurrentSectorOut = 2;
// Write a copy of the Timestamps into the Out file:
if (Out.Write(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps))
{
LOGWARNING("Cannot write Timestamps to file %s, skipping file.", OutFileName.c_str());
return;
}
// Process each chunk:
for (size_t i = 0; i < 1024; i++)
{
size_t idx = i * 4;
if (
(Locations[idx] == 0) &&
(Locations[idx + 1] == 0) &&
(Locations[idx + 2] == 0) &&
(Locations[idx + 3] == 0)
)
{
// Chunk not present
continue;
}
if (!ReadChunk(In, Locations + idx))
{
LOGWARNING("Cannot read chunk #%d from file %s. Skipping file.", i, a_FileName.c_str());
return;
}
if (!WriteChunk(Out, Locations + idx))
{
LOGWARNING("Cannot write chunk #%d to file %s. Skipping file.", i, OutFileName.c_str());
return;
}
}
// Write the new Locations into the MCA header:
Out.Seek(0);
if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations))
{
LOGWARNING("Cannot write updated Locations to file %s, skipping file.", OutFileName.c_str());
return;
}
// Close the files, delete orig, rename new:
In.Close();
Out.Close();
cFile::Delete(a_FileName);
cFile::Rename(OutFileName, a_FileName);
}
bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw)
{
int SectorNum = (a_LocationRaw[0] << 16) | (a_LocationRaw[1] << 8) | a_LocationRaw[2];
int SizeInSectors = a_LocationRaw[3] * (4 KiB);
if (a_File.Seek(SectorNum * (4 KiB)) < 0)
{
LOGWARNING("Failed to seek to chunk data - file pos %llu (%d KiB, %.02f MiB)!", (Int64)SectorNum * (4 KiB), SectorNum * 4, ((double)SectorNum) / 256);
return false;
}
// Read the exact size:
Byte Buf[4];
if (a_File.Read(Buf, 4) != 4)
{
LOGWARNING("Failed to read chunk data length");
return false;
}
m_CompressedChunkDataSize = (Buf[0] << 24) | (Buf[1] << 16) | (Buf[2] << 8) | Buf[3];
if (m_CompressedChunkDataSize > SizeInSectors)
{
LOGWARNING("Invalid chunk data - SizeInSectors (%d) smaller that RealSize (%d)", SizeInSectors, m_CompressedChunkDataSize);
return false;
}
// Read the data:
if (a_File.Read(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize)
{
LOGWARNING("Failed to read chunk data!");
return false;
}
// TODO: Uncompress the data if recompression is active
return true;
}
bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw)
{
// TODO: Recompress the data if recompression is active
a_LocationRaw[0] = m_CurrentSectorOut >> 16;
a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff;
a_LocationRaw[2] = m_CurrentSectorOut & 0xff;
a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) - 1) / (4 KiB);
// Write the data length:
Byte Buf[4];
Buf[0] = m_CompressedChunkDataSize >> 24;
Buf[1] = (m_CompressedChunkDataSize >> 16) & 0xff;
Buf[2] = (m_CompressedChunkDataSize >> 8) & 0xff;
Buf[3] = m_CompressedChunkDataSize & 0xff;
if (a_File.Write(Buf, 4) != 4)
{
LOGWARNING("Failed to write chunk length!");
return false;
}
// Write the data:
if (a_File.Write(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize)
{
LOGWARNING("Failed to write chunk data!");
return false;
}
return true;
}

112
Tools/MCADefrag/MCADefrag.h Normal file
View File

@ -0,0 +1,112 @@
// MCADefrag.h
// Interfaces to the cMCADefrag class encapsulating the entire app
#pragma once
class cMCADefrag
{
public:
enum
{
MAX_COMPRESSED_CHUNK_DATA_SIZE = (1 MiB),
MAX_RAW_CHUNK_DATA_SIZE = (100 MiB),
} ;
cMCADefrag(void);
/** Reads the cmdline params and initializes the app.
Returns true if the app should continue, false if not. */
bool Init(int argc, char ** argv);
/** Runs the entire app. */
void Run(void);
protected:
/** A single thread processing MCA files from the queue */
class cThread :
public cIsThread
{
typedef cIsThread super;
public:
cThread(cMCADefrag & a_Parent);
protected:
cMCADefrag & m_Parent;
/** The current compressed chunk data. Valid after a successful ReadChunk().
This contains only the compression method byte and the compressed data,
but not the length preceding the data in the MCA file. */
unsigned char m_CompressedChunkData[MAX_COMPRESSED_CHUNK_DATA_SIZE];
/** Size of the actual current compressed chunk data. */
int m_CompressedChunkDataSize;
/** The current raw chunk data. Valid after a successful ReadChunk(), if recompression is active. */
unsigned char m_RawChunkData[MAX_RAW_CHUNK_DATA_SIZE];
/** Size of the actual current raw chunk data. */
int m_RawChunkDataSize;
/** Number of the sector where the next chunk will be written by WriteChunk(). */
int m_CurrentSectorOut;
/** Processes the specified file. */
void ProcessFile(const AString & a_FileName);
/** Reads the chunk data into m_CompressedChunkData.
Calls DecompressChunkData() if recompression is active.
a_LocationRaw is the pointer to the first byte of the Location data in the MCA header.
Returns true if successful. */
bool ReadChunk(cFile & a_File, const Byte * a_LocationRaw);
/** Writes the chunk data from m_CompressedData or m_RawChunkData (performing recompression) into file.
Calls CompressChunkData() for the actual compression, if recompression is active.
a_LocationRaw is the pointer to the first byte of the Location data to be put into the MCA header,
the chunk's location is stored in that memory area. Updates m_CurrentSectorOut.
Returns true if successful. */
bool WriteChunk(cFile & a_File, Byte * a_LocationRaw);
// cIsThread overrides:
virtual void Execute(void) override;
} ;
typedef std::list<cThread *> cThreads;
/** The mutex protecting m_Files agains multithreaded access. */
cCriticalSection m_CS;
/** The queue of MCA files to be processed by the threads. Protected by m_CS. */
AStringVector m_Queue;
/** List of threads that the server has running. */
cThreads m_Threads;
/** The number of threads that should be started. Configurable on the command line. */
int m_NumThreads;
/** Starts a new processing thread and adds it to cThreads. */
void StartThread(void);
/** Retrieves one file from the queue (and removes it from the queue).
Returns an empty string when queue empty. */
AString GetNextFileName(void);
} ;