D2loader (#714)

* adding logger implementation to d2common

* Adding file loader implementation

The file loader works in terms of `Sources` and `Assets`. A `Source` is
something like a filesystem that has a cache. An `Asset` is something
that implements `io.ReadSeeker` and has a few methods of its own.

There are currently `Source` implementations for MPQ archives and for the
host filesystem, meaning that one can specify a directory on the host fs to
load files from.

`Sources` are added to a loader with `loader.AddSource(path)`, where `path`
resolves somewhere on disk. In the case that the path points to an MPQ,
then an MPQ `Source` is created and added to the loader. If `path` resolves
to a directory, then a filesystem source is added.

Files are loaded with `loader.Load("data/global/excel/monstats.txt")`, and the
sources are searched in the order that they were added.

* adding tests for d2common/logger_test.go

* adding tests and testdata for d2loader

* logger lint fixes, fixed missing test case

* minor edits, lint fixes, changes some comments, embedded Logger into Loader

* moved d2loader into d2common (I dont think it belonged in d2core)

* removed my simple cache implementation in favor of our existing cache in d2common

d2common/d2loader/asset/asset.go

@@ -0,0 +1,16 @@
+package asset
+
+import (
+	"io"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// Asset represents a game asset. It has a type, an asset source, a sub-path (within the
+// asset source), and it can read data and seek within the data
+type Asset interface {
+	io.ReadSeeker
+	Type() types.AssetType
+	Source() Source
+	Path() string
+}

d2common/d2loader/asset/doc.go

@@ -0,0 +1,2 @@
+// Package asset provides interfaces for Asset and Source
+package asset

d2common/d2loader/asset/source.go

@@ -0,0 +1,12 @@
+package asset
+
+import (
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// Source is an abstraction for something that can load and list assets
+type Source interface {
+	Type() types.SourceType
+	Open(name string) (Asset, error)
+	String() string
+}

d2common/d2loader/asset/types/asset_types.go

@@ -0,0 +1,50 @@
+package types
+
+import "strings"
+
+// AssetType represents the type of an asset
+type AssetType int
+
+// Asset types
+const (
+	AssetTypeUnknown AssetType = iota
+	AssetTypeJSON
+	AssetTypeStringTable
+	AssetTypeDataDictionary
+	AssetTypePalette
+	AssetTypePaletteTransform
+	AssetTypeCOF
+	AssetTypeDC6
+	AssetTypeDCC
+	AssetTypeDS1
+	AssetTypeDT1
+	AssetTypeWAV
+	AssetTypeD2
+)
+
+// Ext2AssetType determines the AssetType with the given file extension
+func Ext2AssetType(ext string) AssetType {
+	ext = strings.ToLower(ext)
+	ext = strings.ReplaceAll(ext, ".", "")
+
+	lookup := map[string]AssetType{
+		"json": AssetTypeJSON,
+		"tbl":  AssetTypeStringTable,
+		"txt":  AssetTypeDataDictionary,
+		"dat":  AssetTypePalette,
+		"pl2":  AssetTypePaletteTransform,
+		"cof":  AssetTypeCOF,
+		"dc6":  AssetTypeDC6,
+		"dcc":  AssetTypeDCC,
+		"ds1":  AssetTypeDS1,
+		"dt1":  AssetTypeDT1,
+		"wav":  AssetTypeWAV,
+		"d2":   AssetTypeD2,
+	}
+
+	if knownType, found := lookup[ext]; found {
+		return knownType
+	}
+
+	return AssetTypeUnknown
+}

d2common/d2loader/asset/types/doc.go

@@ -0,0 +1,3 @@
+// Package types provides an enumeration of Asset and Source types, as well as some utility
+// functions
+package types

d2common/d2loader/asset/types/source_types.go

@@ -0,0 +1,29 @@
+package types
+
+import "strings"
+
+// SourceType represents the type of the asset source
+type SourceType int
+
+// Asset sources
+const (
+	AssetSourceUnknown SourceType = iota
+	AssetSourceFileSystem
+	AssetSourceMPQ
+)
+
+// Ext2SourceType returns the SourceType from the given file extension
+func Ext2SourceType(ext string) SourceType {
+	ext = strings.ToLower(ext)
+	ext = strings.ReplaceAll(ext, ".", "")
+
+	lookup := map[string]SourceType{
+		"mpq": AssetSourceMPQ,
+	}
+
+	if knownType, found := lookup[ext]; found {
+		return knownType
+	}
+
+	return AssetSourceUnknown
+}

d2common/d2loader/doc.go

@@ -0,0 +1,4 @@
+// Package d2loader provides a file loader which works in terms of `Source`s and `Asset`s.
+// A `Source` is something that resembles a filesystem, and an `Asset` is something that
+// implements `io.ReadSeeker`.
+package d2loader

d2common/d2loader/filesystem/asset.go

@@ -0,0 +1,44 @@
+package filesystem
+
+import (
+	"os"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// static check that Asset implements Asset
+var _ asset.Asset = &Asset{}
+
+// Asset represents an asset that is in the host filesystem
+type Asset struct {
+	assetType types.AssetType
+	source    *Source
+	path      string
+	file      *os.File
+}
+
+// Type returns the asset type
+func (fsa *Asset) Type() types.AssetType {
+	return fsa.assetType
+}
+
+// Source returns the asset source that this asset was loaded from
+func (fsa *Asset) Source() asset.Source {
+	return fsa.source
+}
+
+// Path returns the sub-path (within the asset source Root) for this asset
+func (fsa *Asset) Path() string {
+	return fsa.path
+}
+
+// Read reads bytes into the given byte buffer
+func (fsa *Asset) Read(p []byte) (n int, err error) {
+	return fsa.file.Read(p)
+}
+
+// Seek seeks within the file
+func (fsa *Asset) Seek(offset int64, whence int) (int64, error) {
+	return fsa.file.Seek(offset, whence)
+}

d2common/d2loader/filesystem/doc.go

@@ -0,0 +1,2 @@
+// Package filesystem provides a filesystem Asset and Source implementation for d2loader
+package filesystem

d2common/d2loader/filesystem/source.go

@@ -0,0 +1,49 @@
+package filesystem
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// static check that Source implements AssetSource
+var _ asset.Source = &Source{}
+
+// Source represents an asset source which is a normal directory on the host file system
+type Source struct {
+	Root string
+}
+
+// Type returns the type of this asset source
+func (s *Source) Type() types.SourceType {
+	return types.AssetSourceFileSystem
+}
+
+// Open opens a file with the given sub-path within the Root dir of the file system source
+func (s *Source) Open(subPath string) (asset.Asset, error) {
+	file, err := os.Open(s.fullPath(subPath))
+
+	if err == nil {
+		a := &Asset{
+			assetType: types.Ext2AssetType(filepath.Ext(subPath)),
+			source:    s,
+			path:      subPath,
+			file:      file,
+		}
+
+		return a, nil
+	}
+
+	return nil, err
+}
+
+func (s *Source) fullPath(subPath string) string {
+	return filepath.Clean(filepath.Join(s.Root, subPath))
+}
+
+// String returns the Root dir of this file system source
+func (s *Source) String() string {
+	return s.Root
+}

d2common/d2loader/loader.go

@@ -0,0 +1,112 @@
+package d2loader
+
+import (
+	"errors"
+	"fmt"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
+	"os"
+	"path/filepath"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/filesystem"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/mpq"
+)
+
+const (
+	defaultCacheBudget = 1024 * 1024 * 512
+	defaultCacheEntryWeight = 1
+	errFileNotFound = "file not found"
+)
+
+// NewLoader creates a new loader
+func NewLoader() *Loader {
+	loader := &Loader{}
+	loader.Cache = d2common.CreateCache(defaultCacheBudget)
+
+	return loader
+}
+
+// Loader represents the manager that handles loading and caching assets with the asset sources
+// that have been added
+type Loader struct {
+	d2interface.Cache
+	*d2common.Logger
+	sources []asset.Source
+}
+
+// Load attempts to load an asset with the given sub-path. The sub-path is relative to the root
+// of each asset source root (regardless of the type of asset source)
+func (l *Loader) Load(subPath string) (asset.Asset, error) {
+	subPath = filepath.Clean(subPath)
+
+	// first, we check the cache for an existing entry
+	if cached, found := l.Retrieve(subPath); found {
+		l.Debug(fmt.Sprintf("file `%s` exists in loader cache", subPath))
+		return cached.(asset.Asset), nil
+	}
+
+	// if it isn't in the cache, we check if each source can open the file
+	for idx := range l.sources {
+		source := l.sources[idx]
+
+		// if the source can open the file, then we cache it and return it
+		if loadedAsset, err := source.Open(subPath); err == nil {
+			l.Insert(subPath, loadedAsset, defaultCacheEntryWeight)
+			return loadedAsset, nil
+		}
+	}
+
+	return nil, errors.New(errFileNotFound)
+}
+
+// AddSource adds an asset source with the given path. The path will either resolve to a directory
+// or a file on the host filesystem. In the case that it is a file, the file extension is used
+// to determine the type of asset source. In the case that the path points to a directory, a
+// FileSystemSource will be added.
+func (l *Loader) AddSource(path string) {
+	if l.sources == nil {
+		l.sources = make([]asset.Source, 0)
+	}
+
+	cleanPath := filepath.Clean(path)
+
+	info, err := os.Lstat(cleanPath)
+	if err != nil {
+		l.Warning(err.Error())
+		return
+	}
+
+	mode := info.Mode()
+
+	if mode.IsDir() {
+		source := &filesystem.Source{
+			Root: cleanPath,
+		}
+
+		l.Debug(fmt.Sprintf("adding filesystem source `%s`", cleanPath))
+		l.sources = append(l.sources, source)
+	}
+
+	if !mode.IsRegular() {
+		return
+	}
+
+	ext := filepath.Ext(cleanPath)
+	sourceType := types.Ext2SourceType(ext)
+
+	switch sourceType {
+	case types.AssetSourceMPQ:
+		source, err := mpq.NewSource(cleanPath)
+		if err == nil {
+			l.Debug(fmt.Sprintf("adding MPQ source `%s`", cleanPath))
+			l.sources = append(l.sources, source)
+		}
+	case types.AssetSourceUnknown:
+		l.Warning(fmt.Sprintf("unknown asset source `%s`", cleanPath))
+		fallthrough
+	default:
+		return
+	}
+}

d2common/d2loader/loader_test.go

@@ -0,0 +1,124 @@
+package d2loader
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+)
+
+const (
+	sourceA     = "testdata/A"
+	sourceB     = "testdata/B"
+	sourceC     = "testdata/C"
+	sourceD     = "testdata/D.mpq"
+	commonFile  = "common.txt"
+	exclusiveA  = "exclusive_a.txt"
+	exclusiveB  = "exclusive_b.txt"
+	exclusiveC  = "exclusive_c.txt"
+	exclusiveD  = "exclusive_d.txt"
+	badFilePath = "a/bad/file/path.txt"
+)
+
+func TestLoader_NewLoader(t *testing.T) {
+	loader := NewLoader()
+
+	if loader.Cache == nil {
+		t.Error("loader should not be nil")
+	}
+}
+
+func TestLoader_AddSource(t *testing.T) {
+	loader := NewLoader()
+
+	loader.AddSource(sourceA)
+	loader.AddSource(sourceB)
+	loader.AddSource(sourceC)
+	loader.AddSource(sourceD)
+	loader.AddSource("bad/path")
+
+	if loader.sources[0].String() != sourceA {
+		t.Error("source path not the same as what we added")
+	}
+
+	if loader.sources[1].String() != sourceB {
+		t.Error("source path not the same as what we added")
+	}
+
+	if loader.sources[2].String() != sourceC {
+		t.Error("source path not the same as what we added")
+	}
+
+	if loader.sources[3].String() != sourceD {
+		t.Error("source path not the same as what we added")
+	}
+}
+
+func TestLoader_Load(t *testing.T) {
+	loader := NewLoader()
+
+	loader.AddSource(sourceB) // we expect files common to any source to come from here
+	loader.AddSource(sourceD)
+	loader.AddSource(sourceA)
+	loader.AddSource(sourceC)
+
+	entryCommon, errCommon := loader.Load(commonFile) // common file exists in all three sources
+
+	entryA, errA := loader.Load(exclusiveA) // each source has a file exclusive to itself
+	entryB, errB := loader.Load(exclusiveB)
+	entryC, errC := loader.Load(exclusiveC)
+	entryD, errD := loader.Load(exclusiveD)
+
+	_, expectedError := loader.Load(badFilePath) // we expect an Error for this bad file path
+
+	if entryCommon == nil || errCommon != nil {
+		t.Error("common entry should exist")
+	} else if entryCommon.Source() != loader.sources[0] {
+		t.Error("common entry should come from the first loader source")
+	}
+
+	if errA != nil || errB != nil || errC != nil || errD != nil {
+		t.Error("files exclusive to each source don't exist")
+	}
+
+	if expectedError == nil {
+		t.Error("expected Error for nonexistant file path")
+	}
+
+	var result []byte
+
+	buffer := make([]byte, 1)
+
+	tests := []struct {
+		entry asset.Asset
+		data  string
+	}{
+		{entryCommon, "b"}, // sourceB is loaded first, we expect a "b"
+		{entryA, "a"},
+		{entryB, "b"},
+		{entryC, "c"},
+		{entryD, "d"},
+	}
+
+	for idx := range tests {
+		entry, expected := tests[idx].entry, tests[idx].data
+
+		result = make([]byte, 0)
+
+		for {
+			if bytesRead, err := entry.Read(buffer); err != nil || bytesRead == 0 {
+				break
+			}
+
+			result = append(result, buffer...)
+		}
+
+		got := string(result[0])
+
+		if got != expected {
+			fmtStr := "unexpected data in file %s, loaded from source `%s`: expected `%s`, got `%s`"
+			msg := fmt.Sprintf(fmtStr, entry.Path(), entry.Source(), expected, got)
+			t.Error(msg)
+		}
+	}
+}

d2common/d2loader/mpq/asset.go

@@ -0,0 +1,44 @@
+package mpq
+
+import (
+	"path/filepath"
+
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// static check that Asset implements Asset
+var _ asset.Asset = &Asset{}
+
+// Asset represents a file record within an MPQ archive
+type Asset struct {
+	stream d2interface.ArchiveDataStream
+	name   string
+	source *Source
+}
+
+// Type returns the asset type
+func (a *Asset) Type() types.AssetType {
+	return types.Ext2AssetType(filepath.Ext(a.Path()))
+}
+
+// Source returns the source of this asset
+func (a *Asset) Source() asset.Source {
+	return a.source
+}
+
+// Path returns the sub-path (within the source) of this asset
+func (a *Asset) Path() string {
+	return a.name
+}
+
+// Read will read asset data into the given buffer
+func (a *Asset) Read(buf []byte) (n int, err error) {
+	return a.stream.Read(buf)
+}
+
+// Seek will seek the read position for the next read operation
+func (a *Asset) Seek(offset int64, whence int) (n int64, err error) {
+	return a.stream.Seek(offset, whence)
+}

d2common/d2loader/mpq/doc.go

@@ -0,0 +1,2 @@
+// Package mpq provides an MPQ Asset and Source implementation for d2loader
+package mpq

d2common/d2loader/mpq/source.go

@@ -0,0 +1,52 @@
+package mpq
+
+import (
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2mpq"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset"
+	"github.com/OpenDiablo2/OpenDiablo2/d2common/d2loader/asset/types"
+)
+
+// static check that Source implements AssetSource
+var _ asset.Source = &Source{}
+
+// NewSource creates a new MPQ Source
+func NewSource(sourcePath string) (asset.Source, error) {
+	loaded, err := d2mpq.Load(sourcePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Source{loaded}, nil
+}
+
+// Source is an implementation of an asset source for MPQ archives
+type Source struct {
+	MPQ d2interface.Archive
+}
+
+// Type returns the asset type, for MPQ's it always returns the MPQ asset source type
+func (v *Source) Type() types.SourceType {
+	return types.AssetSourceMPQ
+}
+
+// Open attempts to open a file within the MPQ archive
+func (v *Source) Open(name string) (a asset.Asset, err error) {
+	stream, err := v.MPQ.ReadFileStream(name)
+
+	if err != nil {
+		return nil, err
+	}
+
+	a = &Asset{
+		source: v,
+		stream: stream,
+	}
+
+	return a, nil
+}
+
+// String returns the path of the MPQ on the host filesystem
+func (v *Source) String() string {
+	return v.MPQ.Path()
+}

d2common/d2loader/testdata/A/common.txt

@@ -0,0 +1 @@
+a

d2common/d2loader/testdata/A/exclusive_a.txt

@@ -0,0 +1 @@
+a

d2common/d2loader/testdata/B/common.txt

@@ -0,0 +1 @@
+b

d2common/d2loader/testdata/B/exclusive_b.txt

@@ -0,0 +1 @@
+b

d2common/d2loader/testdata/C/common.txt

@@ -0,0 +1 @@
+c

d2common/d2loader/testdata/C/exclusive_c.txt

@@ -0,0 +1 @@
+c

d2common/logger.go

@@ -0,0 +1,101 @@
+package d2common
+
+import (
+	"fmt"
+	"io"
+)
+
+// LogLevel determines how verbose the logging is (higher is more verbose)
+type LogLevel int
+
+// Log levels
+const (
+	LogLevelNone LogLevel = iota
+	LogLevelError
+	LogLevelWarning
+	LogLevelInfo
+	LogLevelDebug
+)
+
+// Log format strings for log levels
+const (
+	LogFmtDebug   = "[DEBUG] %s\n\r"
+	LogFmtInfo    = "[INFO] %s\n\r"
+	LogFmtWarning = "[WARNING] %s\n\r"
+	LogFmtError   = "[ERROR] %s\n\r"
+)
+
+// Logger is used to write log messages, and can have a log level to determine verbosity
+type Logger struct {
+	io.Writer
+	level LogLevel
+}
+
+// SetLevel sets the log level
+func (l *Logger) SetLevel(level LogLevel) {
+	l.level = level
+}
+
+// Debug logs a debug message
+func (l *Logger) Debug(msg string) {
+	if l == nil {
+		return
+	}
+
+	l.print(LogLevelDebug, msg)
+}
+
+// Info logs an info message
+func (l *Logger) Info(msg string) {
+	if l == nil {
+		return
+	}
+
+	l.print(LogLevelInfo, msg)
+}
+
+// Warning logs a warning message
+func (l *Logger) Warning(msg string) {
+	if l == nil {
+		return
+	}
+
+	l.print(LogLevelWarning, msg)
+}
+
+// Error logs an error message
+func (l *Logger) Error(msg string) {
+	if l == nil {
+		return
+	}
+
+	l.print(LogLevelError, msg)
+}
+
+func (l *Logger) print(level LogLevel, msg string) {
+	if l == nil || l.level < level {
+		return
+	}
+
+	fmtString := ""
+
+	switch level {
+	case LogLevelDebug:
+		fmtString = LogFmtDebug
+	case LogLevelInfo:
+		fmtString = LogFmtInfo
+	case LogLevelWarning:
+		fmtString = LogFmtWarning
+	case LogLevelError:
+		fmtString = LogFmtError
+	case LogLevelNone:
+	default:
+		return
+	}
+
+	_, _ = l.Write(format(fmtString, []byte(msg)))
+}
+
+func format(fmtStr string, fmtInput []byte) []byte {
+	return []byte(fmt.Sprintf(fmtStr, string(fmtInput)))
+}

d2common/logger_test.go

@@ -0,0 +1,115 @@
+package d2common
+
+import (
+	"fmt"
+	"testing"
+)
+
+type testWriter struct {
+	data []byte
+}
+
+func (tw *testWriter) Write(msg []byte) (int, error) {
+	tw.data = msg
+
+	return len(msg), nil
+}
+
+func Test_logger_SetLevel(t *testing.T) {
+	l := &Logger{Writer: &testWriter{}}
+
+	tests := []struct {
+		level LogLevel
+	}{
+		{LogLevelNone},
+		{LogLevelError},
+		{LogLevelWarning},
+		{LogLevelInfo},
+		{LogLevelDebug},
+	}
+
+	for idx := range tests {
+		targetLevel := tests[idx].level
+		l.SetLevel(targetLevel)
+
+		if l.level != targetLevel {
+			t.Error("unexpected log level")
+		}
+	}
+}
+
+func Test_logger_LogLevels(t *testing.T) {
+	w := &testWriter{}
+	l := &Logger{Writer: w}
+
+	noMessage := ""
+	message := "test"
+	expectedError := fmt.Sprintf(LogFmtError, message)
+	expectedWarning := fmt.Sprintf(LogFmtWarning, message)
+	expectedInfo := fmt.Sprintf(LogFmtInfo, message)
+	expectedDebug := fmt.Sprintf(LogFmtDebug, message)
+
+	// for each log level we set, we will use different log methods (info, warning, etc) and check
+	// what the output in the writer is (clearing the writer data before each test)
+	tests := []struct {
+		logLevel LogLevel
+		expect   map[LogLevel]string
+	}{
+		{LogLevelDebug, map[LogLevel]string{
+			LogLevelError:   expectedError,
+			LogLevelWarning: expectedWarning,
+			LogLevelInfo:    expectedInfo,
+			LogLevelDebug:   expectedDebug,
+		}},
+		{LogLevelInfo, map[LogLevel]string{
+			LogLevelError:   expectedError,
+			LogLevelWarning: expectedWarning,
+			LogLevelInfo:    expectedInfo,
+			LogLevelDebug:   noMessage,
+		}},
+		{LogLevelWarning, map[LogLevel]string{
+			LogLevelError:   expectedError,
+			LogLevelWarning: expectedWarning,
+			LogLevelInfo:    noMessage,
+			LogLevelDebug:   noMessage,
+		}},
+		{LogLevelError, map[LogLevel]string{
+			LogLevelError:   expectedError,
+			LogLevelWarning: noMessage,
+			LogLevelInfo:    noMessage,
+			LogLevelDebug:   noMessage,
+		}},
+		{LogLevelNone, map[LogLevel]string{
+			LogLevelError:   noMessage,
+			LogLevelWarning: noMessage,
+			LogLevelInfo:    noMessage,
+			LogLevelDebug:   noMessage,
+		}},
+	}
+
+	for idx := range tests {
+		level := tests[idx].logLevel
+		l.SetLevel(level)
+
+		for levelTry, msgExpect := range tests[idx].expect {
+			w.data = make([]byte, 0)
+
+			switch levelTry {
+			case LogLevelError:
+				l.Error(message)
+			case LogLevelWarning:
+				l.Warning(message)
+			case LogLevelInfo:
+				l.Info(message)
+			case LogLevelDebug:
+				l.Debug(message)
+			}
+
+			msgGot := string(w.data)
+
+			if msgGot != msgExpect {
+				t.Errorf("unexpected log message: expected `%s` but got `%s`", msgExpect, msgGot)
+			}
+		}
+	}
+}