gravestench
/
OpenDiablo2
mirror of https://github.com/OpenDiablo2/OpenDiablo2
1
1
Fork 0
Code Issues Releases Wiki Activity

Abstraction for archives and archive manager (#541) 

* archive abstraction

* archive manager abstraction

* fixinglint errors

* archive abstraction

* archive manager abstraction

* fixinglint errors
This commit is contained in:
dk 2020-07-04 19:37:13 -07:00 committed by GitHub
parent 07d90e9681
commit 0a72ccaf16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 142 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
d2common/d2fileformats/d2mpq/hash_entry_map.go
View File

@ -1,6 +1,6 @@
package d2mpq
// HashEntryMap represents a hash entry map
// hashEntryMap represents a hash entry map
type HashEntryMap struct {
entries map[uint64]HashTableEntry
}

67
d2common/d2fileformats/d2mpq/mpq.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"encoding/binary"
"errors"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
"io/ioutil"
"log"
"os"
@ -15,11 +16,11 @@ import (
// MPQ represents an MPQ archive
type MPQ struct {
FileName string
File *os.File
HashEntryMap HashEntryMap
BlockTableEntries []BlockTableEntry
Data Data
filePath string
file *os.File
hashEntryMap HashEntryMap
blockTableEntries []BlockTableEntry
data Data
}
// Data Represents a MPQ file
@ -95,14 +96,14 @@ func (v BlockTableEntry) HasFlag(flag FileFlag) bool {
}
// Load loads an MPQ file and returns a MPQ structure
func Load(fileName string) (*MPQ, error) {
result := &MPQ{FileName: fileName}
func Load(fileName string) (d2interface.Archive, error) {
result := &MPQ{filePath: fileName}
var err error
if runtime.GOOS == "linux" {
result.File, err = openIgnoreCase(fileName)
result.file, err = openIgnoreCase(fileName)
} else {
result.File, err = os.Open(fileName) //nolint:gosec Will fix later
result.file, err = os.Open(fileName) //nolint:gosec Will fix later
}
if err != nil {
@ -144,13 +145,13 @@ func openIgnoreCase(mpqPath string) (*os.File, error) {
}
func (v *MPQ) readHeader() error {
err := binary.Read(v.File, binary.LittleEndian, &v.Data)
err := binary.Read(v.file, binary.LittleEndian, &v.data)
if err != nil {
return err
}
if string(v.Data.Magic[:]) != "MPQ\x1A" {
if string(v.data.Magic[:]) != "MPQ\x1A" {
return errors.New("invalid mpq header")
}
@ -161,23 +162,23 @@ func (v *MPQ) readHeader() error {
}
func (v *MPQ) loadHashTable() {
_, err := v.File.Seek(int64(v.Data.HashTableOffset), 0)
_, err := v.file.Seek(int64(v.data.HashTableOffset), 0)
if err != nil {
log.Panic(err)
}
hashData := make([]uint32, v.Data.HashTableEntries*4) //nolint:gomnd Decryption magic
hashData := make([]uint32, v.data.HashTableEntries*4) //nolint:gomnd Decryption magic
hash := make([]byte, 4)
for i := range hashData {
_, _ = v.File.Read(hash)
_, _ = v.file.Read(hash)
hashData[i] = binary.LittleEndian.Uint32(hash)
}
decrypt(hashData, hashString("(hash table)", 3))
for i := uint32(0); i < v.Data.HashTableEntries; i++ {
v.HashEntryMap.Insert(&HashTableEntry{
for i := uint32(0); i < v.data.HashTableEntries; i++ {
v.hashEntryMap.Insert(&HashTableEntry{
NamePartA: hashData[i*4],
NamePartB: hashData[(i*4)+1],
//nolint:godox // TODO: Verify that we're grabbing the right high/lo word for the vars below
@ -189,23 +190,23 @@ func (v *MPQ) loadHashTable() {
}
func (v *MPQ) loadBlockTable() {
_, err := v.File.Seek(int64(v.Data.BlockTableOffset), 0)
_, err := v.file.Seek(int64(v.data.BlockTableOffset), 0)
if err != nil {
log.Panic(err)
}
blockData := make([]uint32, v.Data.BlockTableEntries*4) //nolint:gomnd binary data
blockData := make([]uint32, v.data.BlockTableEntries*4) //nolint:gomnd binary data
hash := make([]byte, 4)
for i := range blockData {
_, _ = v.File.Read(hash[:]) //nolint:errcheck Will fix later
_, _ = v.file.Read(hash[:]) //nolint:errcheck Will fix later
blockData[i] = binary.LittleEndian.Uint32(hash)
}
decrypt(blockData, hashString("(block table)", 3))
for i := uint32(0); i < v.Data.BlockTableEntries; i++ {
v.BlockTableEntries = append(v.BlockTableEntries, BlockTableEntry{
for i := uint32(0); i < v.data.BlockTableEntries; i++ {
v.blockTableEntries = append(v.blockTableEntries, BlockTableEntry{
FilePosition: blockData[(i * 4)],
CompressedFileSize: blockData[(i*4)+1],
UncompressedFileSize: blockData[(i*4)+2],
@ -259,18 +260,18 @@ func hashString(key string, hashType uint32) uint32 {
// GetFileBlockData gets a block table entry
func (v *MPQ) getFileBlockData(fileName string) (BlockTableEntry, error) {
fileEntry, found := v.HashEntryMap.Find(fileName)
fileEntry, found := v.hashEntryMap.Find(fileName)
if !found || fileEntry.BlockIndex >= uint32(len(v.BlockTableEntries)) {
if !found || fileEntry.BlockIndex >= uint32(len(v.blockTableEntries)) {
return BlockTableEntry{}, errors.New("file not found")
}
return v.BlockTableEntries[fileEntry.BlockIndex], nil
return v.blockTableEntries[fileEntry.BlockIndex], nil
}
// Close closes the MPQ file
func (v *MPQ) Close() {
err := v.File.Close()
err := v.file.Close()
if err != nil {
log.Panic(err)
}
@ -278,7 +279,7 @@ func (v *MPQ) Close() {
// FileExists checks the mpq to see if the file exists
func (v *MPQ) FileExists(fileName string) bool {
return v.HashEntryMap.Contains(fileName)
return v.hashEntryMap.Contains(fileName)
}
// ReadFile reads a file from the MPQ and returns a memory stream
@ -304,7 +305,7 @@ func (v *MPQ) ReadFile(fileName string) ([]byte, error) {
}
// ReadFileStream reads the mpq file data and returns a stream
func (v *MPQ) ReadFileStream(fileName string) (*MpqDataStream, error) {
func (v *MPQ) ReadFileStream(fileName string) (d2interface.ArchiveDataStream, error) {
fileBlockData, err := v.getFileBlockData(fileName)
if err != nil {
@ -364,3 +365,15 @@ func (v *MPQ) GetFileList() ([]string, error) {
return filePaths, nil
}
func (v *MPQ) Path() string {
return v.filePath
}
func (v *MPQ) Contains(filename string) bool {
return v.hashEntryMap.Contains(filename)
}
func (v *MPQ) Size() uint32 {
return v.data.ArchiveSize
}

14
d2common/d2fileformats/d2mpq/mpq_stream.go
View File

@ -42,7 +42,7 @@ func CreateStream(mpq *MPQ, blockTableEntry BlockTableEntry, fileName string) (*
result.EncryptionSeed = (result.EncryptionSeed + result.BlockTableEntry.FilePosition) ^ result.BlockTableEntry.UncompressedFileSize
}
result.BlockSize = 0x200 << result.MPQData.Data.BlockSize //nolint:gomnd MPQ magic
result.BlockSize = 0x200 << result.MPQData.data.BlockSize //nolint:gomnd MPQ magic
if result.BlockTableEntry.HasFlag(FilePatchFile) {
log.Fatal("Patching is not supported")
@ -62,11 +62,11 @@ func (v *Stream) loadBlockOffsets() error {
blockPositionCount := ((v.BlockTableEntry.UncompressedFileSize + v.BlockSize - 1) / v.BlockSize) + 1
v.BlockPositions = make([]uint32, blockPositionCount)
_, _ = v.MPQData.File.Seek(int64(v.BlockTableEntry.FilePosition), 0)
_, _ = v.MPQData.file.Seek(int64(v.BlockTableEntry.FilePosition), 0)
mpqBytes := make([]byte, blockPositionCount*4) //nolint:gomnd MPQ magic
_, _ = v.MPQData.File.Read(mpqBytes)
_, _ = v.MPQData.file.Read(mpqBytes)
for i := range v.BlockPositions {
idx := i * 4 //nolint:gomnd MPQ magic
@ -160,8 +160,8 @@ func (v *Stream) bufferData() {
func (v *Stream) loadSingleUnit() {
fileData := make([]byte, v.BlockSize)
_, _ = v.MPQData.File.Seek(int64(v.MPQData.Data.HeaderSize), 0)
_, _ = v.MPQData.File.Read(fileData)
_, _ = v.MPQData.file.Seek(int64(v.MPQData.data.HeaderSize), 0)
_, _ = v.MPQData.file.Read(fileData)
if v.BlockSize == v.BlockTableEntry.UncompressedFileSize {
v.CurrentData = fileData
@ -188,8 +188,8 @@ func (v *Stream) loadBlock(blockIndex, expectedLength uint32) []byte {
offset += v.BlockTableEntry.FilePosition
data := make([]byte, toRead)
_, _ = v.MPQData.File.Seek(int64(offset), 0)
_, _ = v.MPQData.File.Read(data)
_, _ = v.MPQData.file.Seek(int64(offset), 0)
_, _ = v.MPQData.file.Read(data)
if v.BlockTableEntry.HasFlag(FileEncrypted) && v.BlockTableEntry.UncompressedFileSize > 3 {
if v.EncryptionSeed == 0 {

17
d2common/d2interface/archive.go Normal file
View File

@ -0,0 +1,17 @@
package d2interface
// Archive is an abstract representation of a game archive file
// For the original Diablo II, archives are always MPQ's, but
// OpenDiablo2 can handle any kind of archive file as long as it
// implements this interface
type Archive interface {
Path() string
Contains(string) bool
Size() uint32
Close()
FileExists(fileName string) bool
ReadFile(fileName string) ([]byte, error)
ReadFileStream(fileName string) (ArchiveDataStream, error)
ReadTextFile(fileName string) (string, error)
GetFileList() ([]string, error)
}

8
d2common/d2interface/archive_data_stream.go Normal file
View File

@ -0,0 +1,8 @@
package d2interface
// ArchiveDataStream is an archive data stream
type ArchiveDataStream interface {
Read(p []byte) (n int, err error)
Seek(offset int64, whence int) (int64, error)
Close() error
}

12
d2common/d2interface/archive_manager.go Normal file
View File

@ -0,0 +1,12 @@
package d2interface
// ArchiveManager manages loading files from archives
type ArchiveManager interface {
LoadArchiveForFile(filePath string) (Archive, error)
FileExistsInArchive(filePath string) (bool, error)
LoadArchive(archivePath string) (Archive, error)
CacheArchiveEntries() error
SetVerbose(verbose bool)
ClearCache()
GetCache() Cache
}

75
d2core/d2asset/archive_manager.go
View File

@ -5,42 +5,39 @@ import (
"path"
"sync"
"github.com/OpenDiablo2/OpenDiablo2/d2common"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
"github.com/OpenDiablo2/OpenDiablo2/d2common"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2mpq"
)
type archiveEntry struct {
archivePath string
hashEntryMap d2mpq.HashEntryMap
}
type archiveManager struct {
cache d2interface.Cache
config d2interface.Configuration
entries []archiveEntry
mutex sync.Mutex
cache d2interface.Cache
config d2interface.Configuration
archives []d2interface.Archive
mutex sync.Mutex
}
const (
archiveBudget = 1024 * 1024 * 512
)
func createArchiveManager(config d2interface.Configuration) *archiveManager {
func createArchiveManager(config d2interface.Configuration) d2interface.ArchiveManager {
return &archiveManager{cache: d2common.CreateCache(archiveBudget), config: config}
}
func (am *archiveManager) loadArchiveForFile(filePath string) (*d2mpq.MPQ, error) {
// LoadArchiveForFile loads the archive for the given (in-archive) file path
func (am *archiveManager) LoadArchiveForFile(filePath string) (d2interface.Archive, error) {
am.mutex.Lock()
defer am.mutex.Unlock()
if err := am.cacheArchiveEntries(); err != nil {
if err := am.CacheArchiveEntries(); err != nil {
return nil, err
}
for _, archiveEntry := range am.entries {
if archiveEntry.hashEntryMap.Contains(filePath) {
result, ok := am.loadArchive(archiveEntry.archivePath)
for _, archive := range am.archives {
if archive.Contains(filePath) {
result, ok := am.LoadArchive(archive.Path())
if ok == nil {
return result, nil
}
@ -50,16 +47,17 @@ func (am *archiveManager) loadArchiveForFile(filePath string) (*d2mpq.MPQ, error
return nil, errors.New("file not found")
}
func (am *archiveManager) fileExistsInArchive(filePath string) (bool, error) {
// FileExistsInArchive checks if a file exists in an archive
func (am *archiveManager) FileExistsInArchive(filePath string) (bool, error) {
am.mutex.Lock()
defer am.mutex.Unlock()
if err := am.cacheArchiveEntries(); err != nil {
if err := am.CacheArchiveEntries(); err != nil {
return false, err
}
for _, archiveEntry := range am.entries {
if archiveEntry.hashEntryMap.Contains(filePath) {
for _, archiveEntry := range am.archives {
if archiveEntry.Contains(filePath) {
return true, nil
}
}
@ -67,9 +65,10 @@ func (am *archiveManager) fileExistsInArchive(filePath string) (bool, error) {
return false, nil
}
func (am *archiveManager) loadArchive(archivePath string) (*d2mpq.MPQ, error) {
// LoadArchive loads and caches an archive
func (am *archiveManager) LoadArchive(archivePath string) (d2interface.Archive, error) {
if archive, found := am.cache.Retrieve(archivePath); found {
return archive.(*d2mpq.MPQ), nil
return archive.(d2interface.Archive), nil
}
archive, err := d2mpq.Load(archivePath)
@ -77,33 +76,49 @@ func (am *archiveManager) loadArchive(archivePath string) (*d2mpq.MPQ, error) {
return nil, err
}
if err := am.cache.Insert(archivePath, archive, int(archive.Data.ArchiveSize)); err != nil {
if err := am.cache.Insert(archivePath, archive, int(archive.Size())); err != nil {
return nil, err
}
return archive, nil
}
func (am *archiveManager) cacheArchiveEntries() error {
if len(am.entries) == len(am.config.MpqLoadOrder()) {
// CacheArchiveEntries updates the archive entries
func (am *archiveManager) CacheArchiveEntries() error {
if len(am.archives) == len(am.config.MpqLoadOrder()) {
return nil
}
am.entries = nil
am.archives = nil
for _, archiveName := range am.config.MpqLoadOrder() {
archivePath := path.Join(am.config.MpqPath(), archiveName)
archive, err := am.loadArchive(archivePath)
archive, err := am.LoadArchive(archivePath)
if err != nil {
return err
}
am.entries = append(
am.entries,
archiveEntry{archivePath, archive.HashEntryMap},
am.archives = append(
am.archives,
archive,
)
}
return nil
}
// SetVerbose enables/disables verbose printing for the archive manager
func (am *archiveManager) SetVerbose(verbose bool) {
am.cache.SetVerbose(verbose)
}
// ClearCache clears the archive manager cache
func (am *archiveManager) ClearCache() {
am.cache.Clear()
}
// GetCache returns the archive manager cache
func (am *archiveManager) GetCache() d2interface.Cache {
return am.cache
}

3
d2core/d2asset/asset_manager.go
View File

@ -4,10 +4,11 @@ import (
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2cof"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2dc6"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2dcc"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
)
type assetManager struct {
archiveManager *archiveManager
archiveManager d2interface.ArchiveManager
fileManager *fileManager
paletteManager *paletteManager
paletteTransformManager *paletteTransformManager

9
d2core/d2asset/d2asset.go
View File

@ -6,7 +6,6 @@ import (
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2enum"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2dat"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2mpq"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
)
@ -41,7 +40,7 @@ func Initialize(renderer d2interface.Renderer,
term.OutputInfof("asset manager verbose logging disabled")
}
archiveManager.cache.SetVerbose(verbose)
archiveManager.SetVerbose(verbose)
fileManager.cache.SetVerbose(verbose)
paletteManager.cache.SetVerbose(verbose)
paletteTransformManager.cache.SetVerbose(verbose)
@ -57,7 +56,7 @@ func Initialize(renderer d2interface.Renderer,
return float64(c.GetWeight()) / float64(c.GetBudget()) * percent
}
term.OutputInfof("archive cache: %f", cacheStatistics(archiveManager.cache))
term.OutputInfof("archive cache: %f", cacheStatistics(archiveManager.GetCache()))
term.OutputInfof("file cache: %f", cacheStatistics(fileManager.cache))
term.OutputInfof("palette cache: %f", cacheStatistics(paletteManager.cache))
term.OutputInfof("palette transform cache: %f", cacheStatistics(paletteTransformManager.cache))
@ -68,7 +67,7 @@ func Initialize(renderer d2interface.Renderer,
}
if err := term.BindAction("assetclear", "clear asset manager cache", func() {
archiveManager.cache.Clear()
archiveManager.ClearCache()
fileManager.cache.Clear()
paletteManager.cache.Clear()
paletteTransformManager.cache.Clear()
@ -82,7 +81,7 @@ func Initialize(renderer d2interface.Renderer,
}
// LoadFileStream streams an MPQ file from a source file path
func LoadFileStream(filePath string) (*d2mpq.MpqDataStream, error) {
func LoadFileStream(filePath string) (d2interface.ArchiveDataStream, error) {
data, err := singleton.fileManager.loadFileStream(filePath)
if err != nil {
log.Printf("error loading file stream %s (%v)", filePath, err.Error())

13
d2core/d2asset/file_manager.go
View File

@ -1,7 +1,6 @@
package d2asset
import (
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2fileformats/d2mpq"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2interface"
"strings"
@ -15,12 +14,12 @@ const (
type fileManager struct {
cache d2interface.Cache
archiveManager *archiveManager
archiveManager d2interface.ArchiveManager
config d2interface.Configuration
}
func createFileManager(config d2interface.Configuration,
archiveManager *archiveManager) *fileManager {
archiveManager d2interface.ArchiveManager) *fileManager {
return &fileManager{
d2common.CreateCache(fileBudget),
archiveManager,
@ -28,10 +27,10 @@ func createFileManager(config d2interface.Configuration,
}
}
func (fm *fileManager) loadFileStream(filePath string) (*d2mpq.MpqDataStream, error) {
func (fm *fileManager) loadFileStream(filePath string) (d2interface.ArchiveDataStream, error) {
filePath = fm.fixupFilePath(filePath)
archive, err := fm.archiveManager.loadArchiveForFile(filePath)
archive, err := fm.archiveManager.LoadArchiveForFile(filePath)
if err != nil {
return nil, err
}
@ -45,7 +44,7 @@ func (fm *fileManager) loadFile(filePath string) ([]byte, error) {
return value.([]byte), nil
}
archive, err := fm.archiveManager.loadArchiveForFile(filePath)
archive, err := fm.archiveManager.LoadArchiveForFile(filePath)
if err != nil {
return nil, err
}
@ -64,7 +63,7 @@ func (fm *fileManager) loadFile(filePath string) ([]byte, error) {
func (fm *fileManager) fileExists(filePath string) (bool, error) {
filePath = fm.fixupFilePath(filePath)
return fm.archiveManager.fileExistsInArchive(filePath)
return fm.archiveManager.FileExistsInArchive(filePath)
}
func (fm *fileManager) fixupFilePath(filePath string) string {