gravestench
/
OpenDiablo2
mirror of https://github.com/OpenDiablo2/OpenDiablo2
1
1
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1075 from gucio321/data-encoding2 

general refactor of text dictionary encoder
This commit is contained in:
gravestench 2021-02-26 11:58:34 -08:00 committed by GitHub
parent 23e93886d8 6d88c9eacc
commit a96170561f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 150 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
d2common/d2fileformats/d2tbl/text_dictionary.go
View File

@ -1,89 +1,77 @@
package d2tbl
import (
"errors"
"sort"
"fmt"
"strconv"
"github.com/OpenDiablo2/OpenDiablo2/d2common/d2datautils"
)
// TextDictionary is a string map
type TextDictionary struct {
crcBytes []byte
elementIndex []uint16
hashTableSize uint32
version byte
stringOffset uint32
unknown1 uint32
fileSize uint32
hashEntries []*textDictionaryHashEntry
Entries map[string]string
}
type TextDictionary map[string]string
func (td *TextDictionary) loadHashEntries(br *d2datautils.StreamReader) error {
var err error
for i := 0; i < len(td.hashEntries); i++ {
func (td TextDictionary) loadHashEntries(hashEntries []*textDictionaryHashEntry, br *d2datautils.StreamReader) error {
for i := 0; i < len(hashEntries); i++ {
entry := textDictionaryHashEntry{}
entry.active, err = br.ReadByte()
active, err := br.ReadByte()
if err != nil {
return err
return fmt.Errorf("reading active: %v", err)
}
entry.IsActive = active > 0
entry.Index, err = br.ReadUInt16()
if err != nil {
return err
return fmt.Errorf("reading Index: %v", err)
}
entry.HashValue, err = br.ReadUInt32()
if err != nil {
return err
return fmt.Errorf("reading hash value: %v", err)
}
entry.IndexString, err = br.ReadUInt32()
if err != nil {
return err
return fmt.Errorf("reading index string pos: %v", err)
}
entry.NameString, err = br.ReadUInt32()
if err != nil {
return err
return fmt.Errorf("reading name string pos: %v", err)
}
entry.NameLength, err = br.ReadUInt16()
if err != nil {
return err
return fmt.Errorf("reading name length: %v", err)
}
td.hashEntries[i] = &entry
hashEntries[i] = &entry
}
for idx := range td.hashEntries {
if !td.hashEntries[idx].IsActive() {
for idx := range hashEntries {
if !hashEntries[idx].IsActive {
continue
}
if err := td.loadHashEntry(idx, td.hashEntries[idx], br); err != nil {
return err
if err := td.loadHashEntry(idx, hashEntries[idx], br); err != nil {
return fmt.Errorf("loading entry %d: %v", idx, err)
}
}
return nil
}
func (td *TextDictionary) loadHashEntry(idx int, hashEntry *textDictionaryHashEntry, br *d2datautils.StreamReader) error {
var err error
func (td TextDictionary) loadHashEntry(idx int, hashEntry *textDictionaryHashEntry, br *d2datautils.StreamReader) error {
br.SetPosition(uint64(hashEntry.NameString))
nameVal, err := br.ReadBytes(int(hashEntry.NameLength - 1))
if err != nil {
return err
return fmt.Errorf("reading name value: %v", err)
}
hashEntry.name = string(nameVal)
value := string(nameVal)
br.SetPosition(uint64(hashEntry.IndexString))
@ -96,39 +84,31 @@ func (td *TextDictionary) loadHashEntry(idx int, hashEntry *textDictionaryHashEn
}
if err != nil {
return err
return fmt.Errorf("reading kay char: %v", err)
}
key += string(b)
}
hashEntry.key = key
if hashEntry.key == "x" || hashEntry.key == "X" {
if key == "x" || key == "X" {
key = "#" + strconv.Itoa(idx)
}
_, exists := td.Entries[key]
_, exists := td[key]
if !exists {
td.Entries[key] = hashEntry.name
td[key] = value
}
return nil
}
type textDictionaryHashEntry struct {
name string
key string
IsActive bool
Index uint16
HashValue uint32
IndexString uint32
NameString uint32
Index uint16
NameLength uint16
active byte
}
func (t *textDictionaryHashEntry) IsActive() bool {
return t.active > 0
}
const (
@ -136,133 +116,130 @@ const (
)
// LoadTextDictionary loads the text dictionary from the given data
func LoadTextDictionary(dictionaryData []byte) (*TextDictionary, error) {
var err error
lookupTable := &TextDictionary{
Entries: make(map[string]string),
}
func LoadTextDictionary(dictionaryData []byte) (TextDictionary, error) {
lookupTable := make(TextDictionary)
br := d2datautils.CreateStreamReader(dictionaryData)
// skip past the CRC
lookupTable.crcBytes, err = br.ReadBytes(crcByteCount)
if err != nil {
return nil, err
}
_, _ = br.ReadBytes(crcByteCount)
var err error
/*
number of indicates
(https://d2mods.info/forum/viewtopic.php?p=202077#p202077)
Indices ...
An array of WORD. Each entry is an index into the hash table.
The actual string key index in the .bin file is an index into this table.
So to get a string from a key index ...
*/
numberOfElements, err := br.ReadUInt16()
if err != nil {
return nil, err
return nil, fmt.Errorf("reading number of elements: %v", err)
}
lookupTable.hashTableSize, err = br.ReadUInt32()
hashTableSize, err := br.ReadUInt32()
if err != nil {
return nil, err
return nil, fmt.Errorf("reading hash table size: %v", err)
}
// Version (always 0)
if lookupTable.version, err = br.ReadByte(); err != nil {
return nil, errors.New("error reading Version record")
}
// StringOffset
lookupTable.stringOffset, err = br.ReadUInt32()
// Version
_, err = br.ReadByte()
if err != nil {
return nil, errors.New("error reading string offset")
return nil, fmt.Errorf("reading version: %v", err)
}
_, _ = br.ReadUInt32() // StringOffset
// When the number of times you have missed a match with a
// hash key equals this value, you give up because it is not there.
lookupTable.unknown1, err = br.ReadUInt32()
if err != nil {
return nil, err
}
_, _ = br.ReadUInt32()
// FileSize
lookupTable.fileSize, err = br.ReadUInt32()
if err != nil {
return nil, err
}
_, _ = br.ReadUInt32() // FileSize
elementIndex := make([]uint16, numberOfElements)
for i := 0; i < int(numberOfElements); i++ {
elementIndex[i], err = br.ReadUInt16()
if err != nil {
return nil, err
return nil, fmt.Errorf("reading element index %d: %v", i, err)
}
}
lookupTable.elementIndex = elementIndex
hashEntries := make([]*textDictionaryHashEntry, hashTableSize)
lookupTable.hashEntries = make([]*textDictionaryHashEntry, lookupTable.hashTableSize)
err = lookupTable.loadHashEntries(br)
err = lookupTable.loadHashEntries(hashEntries, br)
if err != nil {
return nil, err
return nil, fmt.Errorf("loading has entries: %v", err)
}
return lookupTable, nil
}
// Marshal encodes text dictionary back to byte slice
// Marshal encodes text dictionary back into byte slice
func (td *TextDictionary) Marshal() []byte {
// create stream writter
sw := d2datautils.CreateStreamWriter()
sw.PushBytes(td.crcBytes...)
sw.PushUint16(uint16(len(td.elementIndex)))
sw.PushUint32(td.hashTableSize)
sw.PushBytes(td.version)
sw.PushUint32(td.stringOffset)
sw.PushUint32(td.unknown1)
// https://github.com/OpenDiablo2/OpenDiablo2/issues/1043
sw.PushBytes(0, 0)
sw.PushUint32(td.fileSize)
sw.PushUint16(0)
for _, i := range td.elementIndex {
sw.PushUint16(i)
sw.PushInt32(int32(len(*td)))
// version (always 0)
sw.PushBytes(0)
// offset of start of data (unnecessary for our decoder)
sw.PushUint32(0)
// Max retry count for a hash hit.
sw.PushUint32(0)
// offset to end of data (noop)
sw.PushUint32(0)
// indicates (len = 0, so nothing here)
// nolint:gomnd // 17 comes from the size of one "data-header index"
// dataPos is a position, when we're placing data stream
dataPos := len(sw.GetBytes()) + 17*len(*td)
for key, value := range *td {
// non-zero if record is used (for us, every record is used ;-)
sw.PushBytes(1)
// generally unused;
// string key index (used in .bin)
sw.PushUint16(0)
// also unused in our decoder
// calculated hash of the string.
sw.PushUint32(0)
sw.PushUint32(uint32(dataPos))
dataPos += len(key) + 1
sw.PushUint32(uint32(dataPos))
dataPos += len(value) + 1
sw.PushUint16(uint16(len(value) + 1))
}
for i := 0; i < len(td.hashEntries); i++ {
sw.PushBytes(td.hashEntries[i].active)
sw.PushUint16(td.hashEntries[i].Index)
sw.PushUint32(td.hashEntries[i].HashValue)
sw.PushUint32(td.hashEntries[i].IndexString)
sw.PushUint32(td.hashEntries[i].NameString)
sw.PushUint16(td.hashEntries[i].NameLength)
}
// data stream: put all data in appropriate order
for key, value := range *td {
for _, i := range key {
sw.PushBytes(byte(i))
}
// values are table entries data (key & values)
var values map[int]string = make(map[int]string)
// valuesSorted are sorted values
var valuesSorted map[int]string = make(map[int]string)
// 0 as separator
sw.PushBytes(0)
// add values key / names to map
for _, i := range td.hashEntries {
values[int(i.IndexString)] = i.key
values[int(i.NameString)] = i.name
}
for _, i := range value {
sw.PushBytes(byte(i))
}
// added map keys
keys := make([]int, 0, len(values))
for k := range values {
keys = append(keys, k)
}
// sort keys
sort.Ints(keys)
// create sorted values map
for _, k := range keys {
valuesSorted[k] = values[k]
}
// add first value (without 0-byte separator)
sw.PushBytes([]byte(valuesSorted[keys[0]])...)
// adds values to result
for i := 1; i < len(valuesSorted); i++ {
sw.PushBytes([]byte(valuesSorted[keys[i]])...)
// 0 as separator
sw.PushBytes(0)
}

37
d2common/d2fileformats/d2tbl/text_dictionary_test.go Normal file
View File

@ -0,0 +1,37 @@
package d2tbl
import (
"testing"
)
func exampleData() *TextDictionary {
result := &TextDictionary{
"abc": "def",
"someStr": "Some long string",
"teststring": "TeSt",
}
return result
}
func TestTBL_Marshal(t *testing.T) {
tbl := exampleData()
data := tbl.Marshal()
newTbl, err := LoadTextDictionary(data)
if err != nil {
t.Error(err)
}
for key, value := range *tbl {
newValue, ok := newTbl[key]
if !ok {
t.Fatal("string wasn't encoded to table")
}
if newValue != value {
t.Fatal("unexpected value set")
}
}
}

6
d2core/d2asset/asset_manager.go
View File

@ -65,7 +65,7 @@ type AssetManager struct {
*d2util.Logger
*d2loader.Loader
tables []*d2tbl.TextDictionary
tables []d2tbl.TextDictionary
dt1s d2interface.Cache
ds1s d2interface.Cache
cofs d2interface.Cache
@ -268,7 +268,7 @@ func (am *AssetManager) LoadPalette(palettePath string) (d2interface.Palette, er
}
// LoadStringTable loads a string table from the given path
func (am *AssetManager) LoadStringTable(tablePath string) (*d2tbl.TextDictionary, error) {
func (am *AssetManager) LoadStringTable(tablePath string) (d2tbl.TextDictionary, error) {
data, err := am.LoadFile(tablePath)
if err != nil {
return nil, err
@ -302,7 +302,7 @@ func (am *AssetManager) TranslateString(input interface{}) string {
}
for idx := range am.tables {
if value, found := am.tables[idx].Entries[key]; found {
if value, found := am.tables[idx][key]; found {
return value
}
}

2
d2core/d2asset/d2asset.go
View File

@ -27,7 +27,7 @@ func NewAssetManager(logLevel d2util.LogLevel) (*AssetManager, error) {
manager := &AssetManager{
Logger: logger,
Loader: loader,
tables: make([]*d2tbl.TextDictionary, 0),
tables: make([]d2tbl.TextDictionary, 0),
animations: d2cache.CreateCache(animationBudget),
fonts: d2cache.CreateCache(fontBudget),
palettes: d2cache.CreateCache(paletteBudget),