Fixed some performance issues with binary reads. Moved config to its own spot. (#112)

common/Configuration.go

@@ -0,0 +1,55 @@
+package common
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+)
+
+// Configuration defines the configuration for the engine, loaded from config.json
+type Configuration struct {
+	Language        string
+	FullScreen      bool
+	Scale           float64
+	RunInBackground bool
+	TicksPerSecond  int
+	FpsCap          int
+	VsyncEnabled    bool
+	MpqPath         string
+	MpqLoadOrder    []string
+	SfxVolume       float64
+	BgmVolume       float64
+}
+
+// ConfigBasePath is used for tests to find the base config json file
+var ConfigBasePath = "./"
+
+func LoadConfiguration() *Configuration {
+	configJSON, err := ioutil.ReadFile(path.Join(ConfigBasePath, "config.json"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	var config Configuration
+	err = json.Unmarshal(configJSON, &config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Path fixup for wine-installed diablo 2 in linux
+	if config.MpqPath[0] != '/' {
+		if _, err := os.Stat(config.MpqPath); os.IsNotExist(err) {
+			homeDir, _ := homedir.Dir()
+			newPath := strings.ReplaceAll(config.MpqPath, `C:\`, homeDir+"/.wine/drive_c/")
+			newPath = strings.ReplaceAll(newPath, "C:/", homeDir+"/.wine/drive_c/")
+			newPath = strings.ReplaceAll(newPath, `\`, "/")
+			if _, err := os.Stat(newPath); !os.IsNotExist(err) {
+				config.MpqPath = newPath
+			}
+		}
+	}
+	return &config
+}

common/Dcc.go

@@ -230,6 +230,7 @@ func CreateDCCDirection(bm *BitMuncher, file *DCC) *DCCDirection {
 	result.FillPixelBuffer(pixelCodeandDisplacement, equalCellsBitstream, pixelMaskBitstream, encodingTypeBitsream, rawPixelCodesBitstream)
 	// Generate the actual frame pixel data
 	result.GenerateFrames(pixelCodeandDisplacement)
+	result.PixelBuffer = nil
 	// Verify that everything we expected to read was actually read (sanity check)...
 	if equalCellsBitstream.BitsRead != result.EqualCellsBitstreamSize {
 		log.Panic("Did not read the correct number of bits!")
@@ -440,6 +441,7 @@ func (v *DCCDirection) FillPixelBuffer(pcd, ec, pm, et, rp *BitMuncher) {
 			}
 		}
 	}
+	cellBuffer = nil
 	// Convert the palette entry index into actual palette entries
 	for i := 0; i <= pbIndex; i++ {
 		for x := 0; x < 4; x++ {

common/StreamReader.go

@@ -1,10 +1,7 @@
 package common
 
 import (
-	"bytes"
-	"encoding/binary"
 	"io"
-	"log"
 )
 
 // StreamReader allows you to read data from a byte array in various formats
@@ -49,11 +46,7 @@ func (v *StreamReader) GetUInt16() uint16 {
 
 // GetInt16 returns a int16 word from the stream
 func (v *StreamReader) GetInt16() int16 {
-	var result int16
+	result := (int16(v.data[v.position+1]) << uint(8)) + int16(v.data[v.position])
-	err := binary.Read(bytes.NewReader([]byte{v.data[v.position], v.data[v.position+1]}), binary.LittleEndian, &result)
-	if err != nil {
-		log.Panic(err)
-	}
 	v.position += 2
 	return result
 }
@@ -64,34 +57,14 @@ func (v *StreamReader) SetPosition(newPosition uint64) {
 
 // GetUInt32 returns a uint32 word from the stream
 func (v *StreamReader) GetUInt32() uint32 {
-	var result uint32
+	result := (uint32(v.data[v.position+3]) << uint(24)) + (uint32(v.data[v.position+2]) << uint(16)) + (uint32(v.data[v.position+1]) << uint(8)) + uint32(v.data[v.position])
-	err := binary.Read(bytes.NewReader(
-		[]byte{
-			v.data[v.position],
-			v.data[v.position+1],
-			v.data[v.position+2],
-			v.data[v.position+3],
-		}), binary.LittleEndian, &result)
-	if err != nil {
-		log.Panic(err)
-	}
 	v.position += 4
 	return result
 }
 
 // GetInt32 returns an int32 word from the stream
 func (v *StreamReader) GetInt32() int32 {
-	var result int32
+	result := (int32(v.data[v.position+3]) << uint(24)) + (int32(v.data[v.position+2]) << uint(16)) + (int32(v.data[v.position+1]) << uint(8)) + int32(v.data[v.position])
-	err := binary.Read(bytes.NewReader(
-		[]byte{
-			v.data[v.position],
-			v.data[v.position+1],
-			v.data[v.position+2],
-			v.data[v.position+3],
-		}), binary.LittleEndian, &result)
-	if err != nil {
-		log.Panic(err)
-	}
 	v.position += 4
 	return result
 }

common/StreamReader_test.go

@@ -27,14 +27,14 @@ func TestStreamReaderByte(t *testing.T) {
 func TestStreamReaderWord(t *testing.T) {
 	data := []byte{0x78, 0x56, 0x34, 0x12}
 	sr := CreateStreamReader(data)
-	ret := sr.GetWord()
+	ret := sr.GetUInt16()
 	if ret != 0x5678 {
 		t.Fatalf("StreamReader.GetDword() was expected to return %X, but returned %X instead", 0x5678, ret)
 	}
 	if pos := sr.GetPosition(); pos != 2 {
 		t.Fatalf("StreamReader.GetPosition() should be at %d, but was at %d instead", 2, pos)
 	}
-	ret = sr.GetWord()
+	ret = sr.GetUInt16()
 	if ret != 0x1234 {
 		t.Fatalf("StreamReader.GetDword() was expected to return %X, but returned %X instead", 0x1234, ret)
 	}
@@ -46,7 +46,7 @@ func TestStreamReaderWord(t *testing.T) {
 func TestStreamReaderDword(t *testing.T) {
 	data := []byte{0x78, 0x56, 0x34, 0x12}
 	sr := CreateStreamReader(data)
-	ret := sr.GetDword()
+	ret := sr.GetUInt32()
 	if ret != 0x12345678 {
 		t.Fatalf("StreamReader.GetDword() was expected to return %X, but returned %X instead", 0x12345678, ret)
 	}

common/TextDictionary.go

@@ -21,7 +21,7 @@ var lookupTable map[string]string
 func TranslateString(key string) string {
 	result, ok := lookupTable[key]
 	if !ok {
-		log.Panic("Could not find a string for the key '%s'", key)
+		log.Panicf("Could not find a string for the key '%s'", key)
 	}
 	return result
 }

core/Engine.go

@@ -1,11 +1,8 @@
 package core
 
 import (
-	"encoding/json"
-	"io/ioutil"
 	"log"
 	"math"
-	"os"
 	"path"
 	"strings"
 	"sync"
@@ -22,37 +19,21 @@ import (
 	"github.com/OpenDiablo2/OpenDiablo2/ui"
 
 	"github.com/hajimehoshi/ebiten"
-	"github.com/mitchellh/go-homedir"
 )
 
-// EngineConfig defines the configuration for the engine, loaded from config.json
-type EngineConfig struct {
-	Language        string
-	FullScreen      bool
-	Scale           float64
-	RunInBackground bool
-	TicksPerSecond  int
-	FpsCap          int
-	VsyncEnabled    bool
-	MpqPath         string
-	MpqLoadOrder    []string
-	SfxVolume       float64
-	BgmVolume       float64
-}
-
 // Engine is the core OpenDiablo2 engine
 type Engine struct {
-	Settings        *EngineConfig     // Engine configuration settings from json file
+	Settings        *common.Configuration // Engine configuration settings from json file
-	Files           map[string]string // Map that defines which files are in which MPQs
+	Files           map[string]string     // Map that defines which files are in which MPQs
-	CheckedPatch    map[string]bool   // First time we check a file, we'll check if it's in the patch. This notes that we've already checked that.
+	CheckedPatch    map[string]bool       // First time we check a file, we'll check if it's in the patch. This notes that we've already checked that.
-	LoadingSprite   *common.Sprite    // The sprite shown when loading stuff
+	LoadingSprite   *common.Sprite        // The sprite shown when loading stuff
-	loadingProgress float64           // LoadingProcess is a range between 0.0 and 1.0. If set, loading screen displays.
+	loadingProgress float64               // LoadingProcess is a range between 0.0 and 1.0. If set, loading screen displays.
-	stepLoadingSize float64           // The size for each loading step
+	stepLoadingSize float64               // The size for each loading step
-	CurrentScene    scenes.Scene      // The current scene being rendered
+	CurrentScene    scenes.Scene          // The current scene being rendered
-	UIManager       *ui.Manager       // The UI manager
+	UIManager       *ui.Manager           // The UI manager
-	SoundManager    *sound.Manager    // The sound manager
+	SoundManager    *sound.Manager        // The sound manager
-	nextScene       scenes.Scene      // The next scene to be loaded at the end of the game loop
+	nextScene       scenes.Scene          // The next scene to be loaded at the end of the game loop
-	fullscreenKey   bool              // When true, the fullscreen toggle is still being pressed
+	fullscreenKey   bool                  // When true, the fullscreen toggle is still being pressed
 }
 
 // CreateEngine creates and instance of the OpenDiablo2 engine
@@ -92,152 +73,13 @@ func CreateEngine() *Engine {
 
 func (v *Engine) loadConfigurationFile() {
 	log.Println("Loading configuration file")
-	configJSON, err := ioutil.ReadFile("config.json")
+	v.Settings = common.LoadConfiguration()
-	if err != nil {
-		log.Fatal(err)
-	}
-	var config EngineConfig
-
-	err = json.Unmarshal(configJSON, &config)
-	if err != nil {
-		log.Fatal(err)
-	}
-	v.Settings = &config
-	// Path fixup for wine-installed diablo 2 in linux
-	if v.Settings.MpqPath[0] != '/' {
-		if _, err := os.Stat(v.Settings.MpqPath); os.IsNotExist(err) {
-			homeDir, _ := homedir.Dir()
-			newPath := strings.ReplaceAll(v.Settings.MpqPath, `C:\`, homeDir+"/.wine/drive_c/")
-			newPath = strings.ReplaceAll(newPath, "C:/", homeDir+"/.wine/drive_c/")
-			newPath = strings.ReplaceAll(newPath, `\`, "/")
-			if _, err := os.Stat(newPath); !os.IsNotExist(err) {
-				log.Printf("Detected linux wine installation, path updated to wine prefix path.")
-				v.Settings.MpqPath = newPath
-			}
-		}
-	}
 }
 
 func (v *Engine) mapMpqFiles() {
 	v.Files = make(map[string]string)
 }
 
-/*
-func (v *Engine) mapMpqFiles() {
-	log.Println("mapping mpq file structure")
-	v.Files = make(map[string]*common.MpqFileRecord)
-	v.CheckedPatch = make(map[string]bool)
-	for _, mpqFileName := range v.Settings.MpqLoadOrder {
-		mpqPath := path.Join(v.Settings.MpqPath, mpqFileName)
-		archive, err := mpq.Load(mpqPath)
-
-		if err != nil {
-			log.Fatal(err)
-		}
-		fileListText, err := archive.ReadFile("(listfile)")
-		if err != nil || fileListText == nil {
-			// Super secret patch file activate!
-			continue
-		}
-		fileList := strings.Split(string(fileListText), "\r\n")
-
-		for _, filePath := range fileList {
-			transFilePath := `/` + strings.ReplaceAll(strings.ToLower(filePath), `\`, `/`)
-			if _, exists := v.Files[transFilePath]; exists {
-				if v.Files[transFilePath].IsPatch {
-					v.Files[transFilePath].UnpatchedMpqFile = mpqPath
-				}
-				continue
-			}
-			v.Files[transFilePath] = &common.MpqFileRecord{
-				mpqPath, false, ""}
-			v.CheckedPatch[transFilePath] = false
-		}
-	}
-}
-
-var mutex sync.Mutex
-
-// LoadFile loads a file from the specified mpq and returns the data as a byte array
-func (v *Engine) LoadFile(fileName string) []byte {
-	fileName = strings.ReplaceAll(fileName, "{LANG}", ResourcePaths.LanguageCode)
-	fileName = strings.ReplaceAll(fileName, `\`, `/`)
-	var mpqLookupFileName string
-	if strings.HasPrefix(fileName, "/") || strings.HasPrefix(fileName, "\\") {
-		mpqLookupFileName = strings.ReplaceAll(fileName, `/`, `\`)[1:]
-	} else {
-		mpqLookupFileName = strings.ReplaceAll(fileName, `/`, `\`)
-	}
-
-	mutex.Lock()
-	// TODO: May want to cache some things if performance becomes an issue
-	mpqFile := v.Files[strings.ToLower(fileName)]
-	var archive mpq.MPQ
-	var err error
-
-	// always try to load from patch first
-	checked, checkok := v.CheckedPatch[strings.ToLower(fileName)]
-	patchLoaded := false
-	if !checked || !checkok {
-		patchMpqFilePath := path.Join(v.Settings.MpqPath, v.Settings.MpqLoadOrder[0])
-		archive, err = mpq.Load(patchMpqFilePath)
-		if err == nil {
-			// loaded patch mpq. check if this file exists in it
-			fileInPatch := archive.FileExists(mpqLookupFileName)
-			if fileInPatch {
-				patchLoaded = true
-				// set the path to the patch so it will be loaded there in the future
-				mpqFile = &common.MpqFileRecord{patchMpqFilePath, false, ""}
-				v.Files[strings.ToLower(fileName)] = mpqFile
-			}
-		}
-		v.CheckedPatch[strings.ToLower(fileName)] = true
-	}
-
-	if patchLoaded {
-		// if we already loaded the correct mpq from the patch check, don't bother reloading it
-	} else if mpqFile == nil {
-		// Super secret non-listed file?
-		found := false
-		for _, mpqFile := range v.Settings.MpqLoadOrder {
-			mpqFilePath := path.Join(v.Settings.MpqPath, mpqFile)
-			archive, err = mpq.Load(mpqFilePath)
-			if err != nil {
-				continue
-			}
-			if !archive.FileExists(strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(fileName, "/data", "data"), "/", `\`))) {
-				continue
-			}
-			// We found the super-secret file!
-			found = true
-			v.Files[strings.ToLower(fileName)] = &common.MpqFileRecord{mpqFilePath, false, ""}
-			break
-		}
-		if !found {
-			log.Fatal(fmt.Sprintf("File '%s' not found during preload of listfiles, and could not be located in any MPQ checking manually.", fileName))
-		}
-	} else if mpqFile.IsPatch {
-		log.Fatal("Tried to load a patchfile")
-	} else {
-		archive, err = mpq.Load(mpqFile.MpqFile)
-		if err != nil {
-			log.Printf("Error loading file '%s'", fileName)
-			log.Fatal(err)
-		}
-	}
-
-	blockTableEntry, err := archive.GetFileBlockData(mpqLookupFileName)
-	if err != nil {
-		log.Printf("Error locating block data entry for '%s' in mpq file '%s'", mpqLookupFileName, archive.FileName)
-		log.Fatal(err)
-	}
-	mpqStream := mpq.CreateStream(mpq, blockTableEntry, mpqLookupFileName)
-	result := make([]byte, blockTableEntry.UncompressedFileSize)
-	mpqStream.Read(result, 0, blockTableEntry.UncompressedFileSize)
-	mutex.Unlock()
-	return result
-}
-*/
 var mutex sync.Mutex
 
 func (v *Engine) LoadFile(fileName string) []byte {

mpq/MPQStream.go

@@ -1,6 +1,7 @@
 package mpq
 
 import (
+	"bufio"
 	"bytes"
 	"compress/zlib"
 	"encoding/binary"
@@ -54,7 +55,14 @@ func (v *Stream) loadBlockOffsets() {
 	blockPositionCount := ((v.BlockTableEntry.UncompressedFileSize + v.BlockSize - 1) / v.BlockSize) + 1
 	v.BlockPositions = make([]uint32, blockPositionCount)
 	v.MPQData.File.Seek(int64(v.BlockTableEntry.FilePosition), 0)
-	binary.Read(v.MPQData.File, binary.LittleEndian, &v.BlockPositions)
+	reader := bufio.NewReader(v.MPQData.File)
+	bytes := make([]byte, blockPositionCount*4)
+	reader.Read(bytes)
+	for i := range v.BlockPositions {
+		idx := i * 4
+		v.BlockPositions[i] = binary.LittleEndian.Uint32(bytes[idx : idx+4])
+	}
+	//binary.Read(v.MPQData.File, binary.LittleEndian, &v.BlockPositions)
 	blockPosSize := blockPositionCount << 2
 	if v.BlockTableEntry.HasFlag(FileEncrypted) {
 		decrypt(v.BlockPositions, v.EncryptionSeed-1)
@@ -121,7 +129,9 @@ func (v *Stream) bufferData() {
 func (v *Stream) loadSingleUnit() {
 	fileData := make([]byte, v.BlockSize)
 	v.MPQData.File.Seek(int64(v.MPQData.Data.HeaderSize), 0)
-	binary.Read(v.MPQData.File, binary.LittleEndian, &fileData)
+	//binary.Read(v.MPQData.File, binary.LittleEndian, &fileData)
+	reader := bufio.NewReader(v.MPQData.File)
+	reader.Read(fileData)
 	if v.BlockSize == v.BlockTableEntry.UncompressedFileSize {
 		v.CurrentData = fileData
 		return
@@ -144,7 +154,9 @@ func (v *Stream) loadBlock(blockIndex, expectedLength uint32) []byte {
 	offset += v.BlockTableEntry.FilePosition
 	data := make([]byte, toRead)
 	v.MPQData.File.Seek(int64(offset), 0)
-	binary.Read(v.MPQData.File, binary.LittleEndian, &data)
+	//binary.Read(v.MPQData.File, binary.LittleEndian, &data)
+	reader := bufio.NewReader(v.MPQData.File)
+	reader.Read(data)
 	if v.BlockTableEntry.HasFlag(FileEncrypted) && v.BlockTableEntry.UncompressedFileSize > 3 {
 		if v.EncryptionSeed == 0 {
 			panic("Unable to determine encryption key")

tests/MPQ_test.go

@@ -0,0 +1,32 @@
+package tests
+
+import (
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/OpenDiablo2/OpenDiablo2/mpq"
+
+	"github.com/OpenDiablo2/OpenDiablo2/common"
+)
+
+func TestMPQScanPerformance(t *testing.T) {
+	mpq.InitializeCryptoBuffer()
+	common.ConfigBasePath = "../"
+	config := common.LoadConfiguration()
+	for _, fileName := range config.MpqLoadOrder {
+		mpqFile := path.Join(config.MpqPath, fileName)
+		archive, _ := mpq.Load(mpqFile)
+		files, err := archive.GetFileList()
+		if err != nil {
+			continue
+		}
+		for _, archiveFile := range files {
+			// Temporary until all audio formats are supported
+			if strings.Contains(archiveFile, ".wav") || strings.Contains(archiveFile, ".pif") {
+				continue
+			}
+			_, _ = archive.ReadFile(archiveFile)
+		}
+	}
+}

tests/MapLoad_test.go

@@ -0,0 +1,28 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/hajimehoshi/ebiten"
+
+	_map "github.com/OpenDiablo2/OpenDiablo2/map"
+
+	"github.com/OpenDiablo2/OpenDiablo2/common"
+	"github.com/OpenDiablo2/OpenDiablo2/core"
+	"github.com/OpenDiablo2/OpenDiablo2/mpq"
+)
+
+func TestMapGenerationPerformance(t *testing.T) {
+	mpq.InitializeCryptoBuffer()
+	common.ConfigBasePath = "../"
+	engine := core.CreateEngine()
+	gameState := common.CreateGameState()
+	mapEngine := _map.CreateMapEngine(gameState, engine.SoundManager, engine)
+	mapEngine.GenerateAct1Overworld()
+	surface, _ := ebiten.NewImage(800, 600, ebiten.FilterNearest)
+	for y := 0; y < 1000; y++ {
+		mapEngine.Render(surface)
+		mapEngine.OffsetY = float64(-y)
+	}
+
+}