Add asset manager (#253)

* Add asset manager

* Fix rebase

* Update asset manager to support mpq hash caching

* Update vendoring

d2core/asset_manager.go

@@ -0,0 +1,143 @@
+package d2core
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/OpenDiablo2/D2Shared/d2common/d2resource"
+	"github.com/OpenDiablo2/D2Shared/d2data/d2mpq"
+	"github.com/OpenDiablo2/OpenDiablo2/d2corecommon"
+)
+
+type archiveEntry struct {
+	archivePath  string
+	hashEntryMap d2mpq.HashEntryMap
+}
+
+type assetManager struct {
+	fileCache      *cache
+	archiveCache   *cache
+	archiveEntries []archiveEntry
+	config         *d2corecommon.Configuration
+	mutex          sync.Mutex
+}
+
+func createAssetManager(config *d2corecommon.Configuration) *assetManager {
+	return &assetManager{
+		fileCache:    createCache(1024 * 1024 * 32),
+		archiveCache: createCache(1024 * 1024 * 128),
+		config:       config,
+	}
+}
+
+func (am *assetManager) LoadFile(filePath string) []byte {
+	data, err := am.loadFile(am.fixupFilePath(filePath))
+	if err != nil {
+		log.Println(err)
+	}
+
+	return data
+}
+
+func (am *assetManager) loadFile(filePath string) ([]byte, error) {
+	if value, found := am.fileCache.retrieve(filePath); found {
+		return value.([]byte), nil
+	}
+
+	archive, err := am.loadArchiveForFilePath(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := archive.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := am.fileCache.insert(filePath, data, len(data)); err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+func (am *assetManager) loadArchiveForFilePath(filePath string) (*d2mpq.MPQ, error) {
+	am.mutex.Lock()
+	defer am.mutex.Unlock()
+
+	if err := am.cacheArchiveEntries(); err != nil {
+		return nil, err
+	}
+
+	for _, archiveEntry := range am.archiveEntries {
+		if archiveEntry.hashEntryMap.Contains(filePath) {
+			return am.loadArchive(archiveEntry.archivePath)
+		}
+	}
+
+	return nil, fmt.Errorf("file not found: %s", filePath)
+}
+
+func (am *assetManager) loadArchive(archivePath string) (*d2mpq.MPQ, error) {
+	if archive, found := am.archiveCache.retrieve(archivePath); found {
+		return archive.(*d2mpq.MPQ), nil
+	}
+
+	archive, err := d2mpq.Load(archivePath)
+	if err != nil {
+		return nil, err
+	}
+
+	stat, err := os.Stat(archivePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := am.archiveCache.insert(archivePath, archive, int(stat.Size())); err != nil {
+		return nil, err
+	}
+
+	return archive, nil
+}
+
+func (am *assetManager) cacheArchiveEntries() error {
+	if len(am.archiveEntries) == len(am.config.MpqLoadOrder) {
+		return nil
+	}
+
+	am.archiveEntries = nil
+
+	for _, archiveName := range am.config.MpqLoadOrder {
+		archivePath := path.Join(am.config.MpqPath, archiveName)
+		archive, err := am.loadArchive(archivePath)
+		if err != nil {
+			return err
+		}
+
+		am.archiveEntries = append(
+			am.archiveEntries,
+			archiveEntry{archivePath, archive.HashEntryMap},
+		)
+	}
+
+	return nil
+}
+
+func (am *assetManager) fixupFilePath(filePath string) string {
+	filePath = strings.ReplaceAll(filePath, "{LANG}", am.config.Language)
+	if strings.ToUpper(d2resource.LanguageCode) == "CHI" {
+		filePath = strings.ReplaceAll(filePath, "{LANG_FONT}", am.config.Language)
+	} else {
+		filePath = strings.ReplaceAll(filePath, "{LANG_FONT}", "latin")
+	}
+
+	filePath = strings.ToLower(filePath)
+	filePath = strings.ReplaceAll(filePath, `/`, "\\")
+	filePath = strings.TrimPrefix(filePath, "\\")
+
+	return filePath
+}

d2core/cache.go

@@ -0,0 +1,97 @@
+package d2core
+
+import (
+	"errors"
+	"sync"
+)
+
+type cacheNode struct {
+	next   *cacheNode
+	prev   *cacheNode
+	key    string
+	value  interface{}
+	weight int
+}
+
+type cache struct {
+	head   *cacheNode
+	tail   *cacheNode
+	lookup map[string]*cacheNode
+	weight int
+	budget int
+	mutex  sync.Mutex
+}
+
+func createCache(budget int) *cache {
+	return &cache{lookup: make(map[string]*cacheNode), budget: budget}
+}
+
+func (c *cache) insert(key string, value interface{}, weight int) error {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	if _, found := c.lookup[key]; found {
+		return errors.New("key already exists in cache")
+	}
+
+	node := &cacheNode{
+		key:    key,
+		value:  value,
+		weight: weight,
+		next:   c.head,
+	}
+
+	if c.head != nil {
+		c.head.prev = node
+	}
+
+	c.head = node
+	if c.tail == nil {
+		c.tail = node
+	}
+
+	c.lookup[key] = node
+	c.weight += node.weight
+
+	for ; c.tail != nil && c.tail != c.head && c.weight > c.budget; c.tail = c.tail.prev {
+		c.weight -= c.tail.weight
+		c.tail.prev.next = nil
+		delete(c.lookup, c.tail.key)
+	}
+
+	return nil
+}
+
+func (c *cache) retrieve(key string) (interface{}, bool) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	node, found := c.lookup[key]
+	if !found {
+		return nil, false
+	}
+
+	if node != c.head {
+		if node.next != nil {
+			node.next.prev = node.prev
+		}
+		if node.prev != nil {
+			node.prev.next = node.next
+		}
+
+		if node == c.tail {
+			c.tail = c.tail.prev
+		}
+
+		node.next = c.head
+		node.prev = nil
+
+		if c.head != nil {
+			c.head.prev = node
+		}
+
+		c.head = node
+	}
+
+	return node.value, true
+}

d2core/engine.go

@@ -1,16 +1,14 @@
 package d2core
 
 import (
-	"github.com/OpenDiablo2/D2Shared/d2data/d2dc6"
 	"log"
 	"math"
-	"path"
 	"runtime"
 	"strconv"
-	"strings"
-	"sync"
 	"time"
 
+	"github.com/OpenDiablo2/D2Shared/d2data/d2dc6"
+
 	"github.com/OpenDiablo2/D2Shared/d2common/d2resource"
 
 	"github.com/OpenDiablo2/D2Shared/d2helper"
@@ -25,8 +23,6 @@ import (
 
 	"github.com/OpenDiablo2/D2Shared/d2data/d2datadict"
 
-	"github.com/OpenDiablo2/D2Shared/d2data/d2mpq"
-
 	"github.com/OpenDiablo2/OpenDiablo2/d2audio"
 
 	"github.com/OpenDiablo2/D2Shared/d2common"
@@ -41,7 +37,6 @@ import (
 // Engine is the core OpenDiablo2 engine
 type Engine struct {
 	Settings        *d2corecommon.Configuration // Engine configuration settings from json file
-	Files           map[string]string           // Map that defines which files are in which MPQs
 	CheckedPatch    map[string]bool             // First time we check a file, we'll check if it's in the patch. This notes that we've already checked that.
 	LoadingSprite   d2render.Sprite             // The sprite shown when loading stuff
 	loadingProgress float64                     // LoadingProcess is a range between 0.0 and 1.0. If set, loading screen displays.
@@ -55,18 +50,16 @@ type Engine struct {
 	fullscreenKey   bool                        // When true, the fullscreen toggle is still being pressed
 	lastTime        float64                     // Last time we updated the scene
 	showFPS         bool
+	assetManager    *assetManager
 }
 
 // CreateEngine creates and instance of the OpenDiablo2 engine
 func CreateEngine() Engine {
-	result := Engine{
-		CurrentScene: nil,
-		nextScene:    nil,
-	}
+	var result Engine
 	result.loadConfigurationFile()
+	result.assetManager = createAssetManager(result.Settings)
 	d2resource.LanguageCode = result.Settings.Language
-	result.mapMpqFiles()
-	d2datadict.LoadPalettes(result.Files, &result)
+	d2datadict.LoadPalettes(nil, &result)
 	d2common.LoadTextDictionary(&result)
 	d2datadict.LoadLevelTypes(&result)
 	d2datadict.LoadLevelPresets(&result)
@@ -88,7 +81,6 @@ func CreateEngine() Engine {
 	result.LoadingSprite = result.LoadSprite(d2resource.LoadingScreen, d2enum.Loading)
 	loadingSpriteSizeX, loadingSpriteSizeY := result.LoadingSprite.GetSize()
 	result.LoadingSprite.MoveTo(int(400-(loadingSpriteSizeX/2)), int(300+(loadingSpriteSizeY/2)))
-	//result.SetNextScene(Scenes.CreateBlizzardIntro(result, result))
 	return result
 }
 
@@ -97,53 +89,8 @@ func (v *Engine) loadConfigurationFile() {
 	v.Settings = d2corecommon.LoadConfiguration()
 }
 
-func (v *Engine) mapMpqFiles() {
-	v.Files = make(map[string]string)
-}
-
-var mutex sync.Mutex
-
 func (v *Engine) LoadFile(fileName string) []byte {
-	fileName = strings.ReplaceAll(fileName, "{LANG}", d2resource.LanguageCode)
-	// todo: separate CJK and latin characters from LanguageCode
-	if "CHI" == strings.ToUpper(d2resource.LanguageCode) {
-		fileName = strings.ReplaceAll(fileName, "{LANG_FONT}", d2resource.LanguageCode)
-	} else {
-		fileName = strings.ReplaceAll(fileName, "{LANG_FONT}", "latin")
-	}
-	fileName = strings.ToLower(fileName)
-	fileName = strings.ReplaceAll(fileName, `/`, "\\")
-	if fileName[0] == '\\' {
-		fileName = fileName[1:]
-	}
-
-	mutex.Lock()
-	defer mutex.Unlock()
-	// TODO: May want to cache some things if performance becomes an issue
-	cachedMpqFile, cacheExists := v.Files[fileName]
-	if cacheExists {
-		archive, _ := d2mpq.Load(cachedMpqFile)
-		result, _ := archive.ReadFile(fileName)
-		return result
-	}
-	for _, mpqFile := range v.Settings.MpqLoadOrder {
-		archive, _ := d2mpq.Load(path.Join(v.Settings.MpqPath, mpqFile))
-		if archive == nil {
-			log.Fatalf("Failed to load specified MPQ file: %s", mpqFile)
-		}
-		if !archive.FileExists(fileName) {
-			continue
-		}
-		result, _ := archive.ReadFile(fileName)
-		if len(result) == 0 {
-			continue
-		}
-		v.Files[fileName] = path.Join(v.Settings.MpqPath, mpqFile)
-		// log.Printf("%v in %v", fileName, mpqFile)
-		return result
-	}
-	log.Printf("Could not load %s from MPQs\n", fileName)
-	return []byte{}
+	return v.assetManager.LoadFile(fileName)
 }
 
 // IsLoading returns true if the engine is currently in a loading state

go.mod

@@ -3,7 +3,7 @@ module github.com/OpenDiablo2/OpenDiablo2
 go 1.12
 
 require (
-	github.com/OpenDiablo2/D2Shared v0.0.0-20191215185943-ef5f17453dfd
+	github.com/OpenDiablo2/D2Shared v0.0.0-20191217031331-027f7bbcb9fc
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/hajimehoshi/ebiten v1.11.0-alpha.0.20191121152720-3df198f68eea

go.sum

@@ -4,6 +4,8 @@ github.com/JoshVarga/blast v0.0.0-20180421040937-681c804fb9f0 h1:tDnuU0igiBiQFjs
 github.com/JoshVarga/blast v0.0.0-20180421040937-681c804fb9f0/go.mod h1:h/5OEGj4G+fpYxluLjSMZbFY011ZxAntO98nCl8mrCs=
 github.com/OpenDiablo2/D2Shared v0.0.0-20191215185943-ef5f17453dfd h1:f7THoOT1W1BTaSm/hwz9S/THw0ZLkuYQI9BLV1Pa+Iw=
 github.com/OpenDiablo2/D2Shared v0.0.0-20191215185943-ef5f17453dfd/go.mod h1:mY8Ll5/iLRAQsaHvIdqSZiHX3aFCys/Q4Sot+xYpero=
+github.com/OpenDiablo2/D2Shared v0.0.0-20191217031331-027f7bbcb9fc h1:f+PIG0HmaNYzBZuWuJAVau8SnllRLbzv6O8ttADXKrY=
+github.com/OpenDiablo2/D2Shared v0.0.0-20191217031331-027f7bbcb9fc/go.mod h1:mY8Ll5/iLRAQsaHvIdqSZiHX3aFCys/Q4Sot+xYpero=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=