diff --git a/d2common/d2fileformats/d2mpq/crypto.go b/d2common/d2fileformats/d2mpq/crypto.go new file mode 100644 index 00000000..636c8bc2 --- /dev/null +++ b/d2common/d2fileformats/d2mpq/crypto.go @@ -0,0 +1,131 @@ +package d2mpq + +import ( + "encoding/binary" + "io" + "strings" +) + +var cryptoBuffer [0x500]uint32 //nolint:gochecknoglobals // will fix later.. +var cryptoBufferReady bool //nolint:gochecknoglobals // will fix later.. + +func cryptoLookup(index uint32) uint32 { + if !cryptoBufferReady { + cryptoInitialize() + + cryptoBufferReady = true + } + + return cryptoBuffer[index] +} + +//nolint:gomnd // Decryption magic +func cryptoInitialize() { + seed := uint32(0x00100001) + + for index1 := 0; index1 < 0x100; index1++ { + index2 := index1 + + for i := 0; i < 5; i++ { + seed = (seed*125 + 3) % 0x2AAAAB + temp1 := (seed & 0xFFFF) << 0x10 + seed = (seed*125 + 3) % 0x2AAAAB + temp2 := seed & 0xFFFF + cryptoBuffer[index2] = temp1 | temp2 + index2 += 0x100 + } + } +} + +//nolint:gomnd // Decryption magic +func decrypt(data []uint32, seed uint32) { + seed2 := uint32(0xeeeeeeee) + + for i := 0; i < len(data); i++ { + seed2 += cryptoLookup(0x400 + (seed & 0xff)) + result := data[i] + result ^= seed + seed2 + + seed = ((^seed << 21) + 0x11111111) | (seed >> 11) + seed2 = result + seed2 + (seed2 << 5) + 3 + data[i] = result + } +} + +//nolint:gomnd // Decryption magic +func decryptBytes(data []byte, seed uint32) { + seed2 := uint32(0xEEEEEEEE) + for i := 0; i < len(data)-3; i += 4 { + seed2 += cryptoLookup(0x400 + (seed & 0xFF)) + result := binary.LittleEndian.Uint32(data[i : i+4]) + result ^= seed + seed2 + seed = ((^seed << 21) + 0x11111111) | (seed >> 11) + seed2 = result + seed2 + (seed2 << 5) + 3 + + data[i+0] = uint8(result & 0xff) + data[i+1] = uint8((result >> 8) & 0xff) + data[i+2] = uint8((result >> 16) & 0xff) + data[i+3] = uint8((result >> 24) & 0xff) + } +} + +//nolint:gomnd // Decryption magic +func decryptTable(r io.Reader, size uint32, name string) ([]uint32, error) { + seed := hashString(name, 3) + seed2 := uint32(0xEEEEEEEE) + size *= 4 + + table := make([]uint32, size) + buf := make([]byte, 4) + + for i := uint32(0); i < size; i++ { + seed2 += cryptoBuffer[0x400+(seed&0xff)] + + if _, err := r.Read(buf); err != nil { + return table, err + } + + result := binary.LittleEndian.Uint32(buf) + result ^= seed + seed2 + + seed = ((^seed << 21) + 0x11111111) | (seed >> 11) + seed2 = result + seed2 + (seed2 << 5) + 3 + table[i] = result + } + + return table, nil +} + +func hashFilename(key string) uint64 { + a, b := hashString(key, 1), hashString(key, 2) + return uint64(a)<<32 | uint64(b) +} + +//nolint:gomnd // Decryption magic +func hashString(key string, hashType uint32) uint32 { + seed1 := uint32(0x7FED7FED) + seed2 := uint32(0xEEEEEEEE) + + /* prepare seeds. */ + for _, char := range strings.ToUpper(key) { + seed1 = cryptoLookup((hashType*0x100)+uint32(char)) ^ (seed1 + seed2) + seed2 = uint32(char) + seed1 + seed2 + (seed2 << 5) + 3 + } + + return seed1 +} + +//nolint:unused,deadcode,gomnd // will use this for creating mpq's +func encrypt(data []uint32, seed uint32) { + seed2 := uint32(0xeeeeeeee) + + for i := 0; i < len(data); i++ { + seed2 += cryptoLookup(0x400 + (seed & 0xff)) + result := data[i] + result ^= seed + seed2 + + seed = ((^seed << 21) + 0x11111111) | (seed >> 11) + seed2 = data[i] + seed2 + (seed2 << 5) + 3 + data[i] = result + } +} diff --git a/d2common/d2fileformats/d2mpq/crypto_buff.go b/d2common/d2fileformats/d2mpq/crypto_buff.go deleted file mode 100644 index 7618743b..00000000 --- a/d2common/d2fileformats/d2mpq/crypto_buff.go +++ /dev/null @@ -1,32 +0,0 @@ -package d2mpq - -var cryptoBuffer [0x500]uint32 //nolint:gochecknoglobals // will fix later.. -var cryptoBufferReady bool //nolint:gochecknoglobals // will fix later.. - -func cryptoLookup(index uint32) uint32 { - if !cryptoBufferReady { - cryptoInitialize() - - cryptoBufferReady = true - } - - return cryptoBuffer[index] -} - -//nolint:gomnd // magic cryptographic stuff here... -func cryptoInitialize() { - seed := uint32(0x00100001) - - for index1 := 0; index1 < 0x100; index1++ { - index2 := index1 - - for i := 0; i < 5; i++ { - seed = (seed*125 + 3) % 0x2AAAAB - temp1 := (seed & 0xFFFF) << 0x10 - seed = (seed*125 + 3) % 0x2AAAAB - temp2 := seed & 0xFFFF - cryptoBuffer[index2] = temp1 | temp2 - index2 += 0x100 - } - } -} diff --git a/d2common/d2fileformats/d2mpq/hash_entry_map.go b/d2common/d2fileformats/d2mpq/hash_entry_map.go deleted file mode 100644 index ab9c0ca1..00000000 --- a/d2common/d2fileformats/d2mpq/hash_entry_map.go +++ /dev/null @@ -1,35 +0,0 @@ -package d2mpq - -// HashEntryMap represents a hash entry map -type HashEntryMap struct { - entries map[uint64]HashTableEntry -} - -// Insert inserts a hash entry into the table -func (hem *HashEntryMap) Insert(entry *HashTableEntry) { - if hem.entries == nil { - hem.entries = make(map[uint64]HashTableEntry) - } - - hem.entries[uint64(entry.NamePartA)<<32|uint64(entry.NamePartB)] = *entry -} - -// Find finds a hash entry -func (hem *HashEntryMap) Find(fileName string) (*HashTableEntry, bool) { - if hem.entries == nil { - return nil, false - } - - hashA := hashString(fileName, 1) - hashB := hashString(fileName, 2) - - entry, found := hem.entries[uint64(hashA)<<32|uint64(hashB)] - - return &entry, found -} - -// Contains returns true if the hash entry contains the values -func (hem *HashEntryMap) Contains(fileName string) bool { - _, found := hem.Find(fileName) - return found -} diff --git a/d2common/d2fileformats/d2mpq/mpq.go b/d2common/d2fileformats/d2mpq/mpq.go index 23deed54..59f3a8b1 100644 --- a/d2common/d2fileformats/d2mpq/mpq.go +++ b/d2common/d2fileformats/d2mpq/mpq.go @@ -2,10 +2,9 @@ package d2mpq import ( "bufio" - "encoding/binary" "errors" + "fmt" "io/ioutil" - "log" "os" "path" "path/filepath" @@ -19,33 +18,11 @@ var _ d2interface.Archive = &MPQ{} // Static check to confirm struct conforms to // MPQ represents an MPQ archive type MPQ struct { - filePath string - file *os.File - hashEntryMap HashEntryMap - blockTableEntries []BlockTableEntry - data Data -} - -// Data Represents a MPQ file -type Data struct { - Magic [4]byte - HeaderSize uint32 - ArchiveSize uint32 - FormatVersion uint16 - BlockSize uint16 - HashTableOffset uint32 - BlockTableOffset uint32 - HashTableEntries uint32 - BlockTableEntries uint32 -} - -// HashTableEntry represents a hashed file entry in the MPQ file -type HashTableEntry struct { // 16 bytes - NamePartA uint32 - NamePartB uint32 - Locale uint16 - Platform uint16 - BlockIndex uint32 + filePath string + file *os.File + hashes map[uint64]*Hash + blocks []*Block + header Header } // PatchInfo represents patch info for the MPQ. @@ -53,71 +30,153 @@ type PatchInfo struct { Length uint32 // Length of patch info header, in bytes Flags uint32 // Flags. 0x80000000 = MD5 (?) DataSize uint32 // Uncompressed size of the patch file - Md5 [16]byte // MD5 of the entire patch file after decompression + MD5 [16]byte // MD5 of the entire patch file after decompression } -// FileFlag represents flags for a file record in the MPQ archive -type FileFlag uint32 - -const ( - // FileImplode - File is compressed using PKWARE Data compression library - FileImplode FileFlag = 0x00000100 - // FileCompress - File is compressed using combination of compression methods - FileCompress FileFlag = 0x00000200 - // FileEncrypted - The file is encrypted - FileEncrypted FileFlag = 0x00010000 - // FileFixKey - The decryption key for the file is altered according to the position of the file in the archive - FileFixKey FileFlag = 0x00020000 - // FilePatchFile - The file contains incremental patch for an existing file in base MPQ - FilePatchFile FileFlag = 0x00100000 - // FileSingleUnit - Instead of being divided to 0x1000-bytes blocks, the file is stored as single unit - FileSingleUnit FileFlag = 0x01000000 - // FileDeleteMarker - File is a deletion marker, indicating that the file no longer exists. This is used to allow patch - // archives to delete files present in lower-priority archives in the search chain. The file usually - // has length of 0 or 1 byte and its name is a hash - FileDeleteMarker FileFlag = 0x02000000 - // FileSectorCrc - File has checksums for each sector. Ignored if file is not compressed or imploded. - FileSectorCrc FileFlag = 0x04000000 - // FileExists - Set if file exists, reset when the file was deleted - FileExists FileFlag = 0x80000000 -) - -// BlockTableEntry represents an entry in the block table -type BlockTableEntry struct { // 16 bytes - FilePosition uint32 - CompressedFileSize uint32 - UncompressedFileSize uint32 - Flags FileFlag - // Local Stuff... - FileName string - EncryptionSeed uint32 -} - -// HasFlag returns true if the specified flag is present -func (v BlockTableEntry) HasFlag(flag FileFlag) bool { - return (v.Flags & flag) != 0 -} - -// Load loads an MPQ file and returns a MPQ structure -func Load(fileName string) (d2interface.Archive, error) { - result := &MPQ{filePath: fileName} +// New loads an MPQ file and only reads the header +func New(fileName string) (*MPQ, error) { + mpq := &MPQ{filePath: fileName} var err error if runtime.GOOS == "linux" { - result.file, err = openIgnoreCase(fileName) + mpq.file, err = openIgnoreCase(fileName) } else { - result.file, err = os.Open(fileName) //nolint:gosec // Will fix later + mpq.file, err = os.Open(fileName) //nolint:gosec // Will fix later } if err != nil { return nil, err } - if err := result.readHeader(); err != nil { + if err := mpq.readHeader(); err != nil { + return nil, fmt.Errorf("failed to read reader: %v", err) + } + + return mpq, nil +} + +// FromFile loads an MPQ file and returns a MPQ structure +func FromFile(fileName string) (*MPQ, error) { + mpq, err := New(fileName) + if err != nil { return nil, err } - return result, nil + if err := mpq.readHashTable(); err != nil { + return nil, fmt.Errorf("failed to read hash table: %v", err) + } + + if err := mpq.readBlockTable(); err != nil { + return nil, fmt.Errorf("failed to read block table: %v", err) + } + + return mpq, nil +} + +// getFileBlockData gets a block table entry +func (mpq *MPQ) getFileBlockData(fileName string) (*Block, error) { + fileEntry, ok := mpq.hashes[hashFilename(fileName)] + if !ok { + return nil, errors.New("file not found") + } + + if fileEntry.BlockIndex >= uint32(len(mpq.blocks)) { + return nil, errors.New("invalid block index") + } + + return mpq.blocks[fileEntry.BlockIndex], nil +} + +// Close closes the MPQ file +func (mpq *MPQ) Close() error { + return mpq.file.Close() +} + +// ReadFile reads a file from the MPQ and returns a memory stream +func (mpq *MPQ) ReadFile(fileName string) ([]byte, error) { + fileBlockData, err := mpq.getFileBlockData(fileName) + if err != nil { + return []byte{}, err + } + + fileBlockData.FileName = strings.ToLower(fileName) + + stream, err := CreateStream(mpq, fileBlockData, fileName) + if err != nil { + return []byte{}, err + } + + buffer := make([]byte, fileBlockData.UncompressedFileSize) + if _, err := stream.Read(buffer, 0, fileBlockData.UncompressedFileSize); err != nil { + return []byte{}, err + } + + return buffer, nil +} + +// ReadFileStream reads the mpq file data and returns a stream +func (mpq *MPQ) ReadFileStream(fileName string) (d2interface.DataStream, error) { + fileBlockData, err := mpq.getFileBlockData(fileName) + if err != nil { + return nil, err + } + + fileBlockData.FileName = strings.ToLower(fileName) + + stream, err := CreateStream(mpq, fileBlockData, fileName) + if err != nil { + return nil, err + } + + return &MpqDataStream{stream: stream}, nil +} + +// ReadTextFile reads a file and returns it as a string +func (mpq *MPQ) ReadTextFile(fileName string) (string, error) { + data, err := mpq.ReadFile(fileName) + + if err != nil { + return "", err + } + + return string(data), nil +} + +// Listfile returns the list of files in this MPQ +func (mpq *MPQ) Listfile() ([]string, error) { + data, err := mpq.ReadFile("(listfile)") + + if err != nil { + return nil, err + } + + raw := strings.TrimRight(string(data), "\x00") + s := bufio.NewScanner(strings.NewReader(raw)) + + var filePaths []string + + for s.Scan() { + filePath := s.Text() + filePaths = append(filePaths, filePath) + } + + return filePaths, nil +} + +// Path returns the MPQ file path +func (mpq *MPQ) Path() string { + return mpq.filePath +} + +// Contains returns bool for whether the given filename exists in the mpq +func (mpq *MPQ) Contains(filename string) bool { + _, ok := mpq.hashes[hashFilename(filename)] + return ok +} + +// Size returns the size of the mpq in bytes +func (mpq *MPQ) Size() uint32 { + return mpq.header.ArchiveSize } func openIgnoreCase(mpqPath string) (*os.File, error) { @@ -142,258 +201,5 @@ func openIgnoreCase(mpqPath string) (*os.File, error) { } } - file, err := os.Open(path.Join(mpqDir, mpqName)) //nolint:gosec // Will fix later - - return file, err -} - -func (v *MPQ) readHeader() error { - err := binary.Read(v.file, binary.LittleEndian, &v.data) - - if err != nil { - return err - } - - if string(v.data.Magic[:]) != "MPQ\x1A" { - return errors.New("invalid mpq header") - } - - err = v.loadHashTable() - if err != nil { - return err - } - - v.loadBlockTable() - - return nil -} - -func (v *MPQ) loadHashTable() error { - _, err := v.file.Seek(int64(v.data.HashTableOffset), 0) - if err != nil { - log.Panic(err) - } - - hashData := make([]uint32, v.data.HashTableEntries*4) //nolint:gomnd // // Decryption magic - hash := make([]byte, 4) - - for i := range hashData { - _, err := v.file.Read(hash) - if err != nil { - log.Print(err) - } - - hashData[i] = binary.LittleEndian.Uint32(hash) - } - - decrypt(hashData, hashString("(hash table)", 3)) - - for i := uint32(0); i < v.data.HashTableEntries; i++ { - v.hashEntryMap.Insert(&HashTableEntry{ - NamePartA: hashData[i*4], - NamePartB: hashData[(i*4)+1], - // https://github.com/OpenDiablo2/OpenDiablo2/issues/812 - Locale: uint16(hashData[(i*4)+2] >> 16), //nolint:gomnd // // binary data - Platform: uint16(hashData[(i*4)+2] & 0xFFFF), //nolint:gomnd // // binary data - BlockIndex: hashData[(i*4)+3], - }) - } - - return nil -} - -func (v *MPQ) loadBlockTable() { - _, err := v.file.Seek(int64(v.data.BlockTableOffset), 0) - if err != nil { - log.Panic(err) - } - - blockData := make([]uint32, v.data.BlockTableEntries*4) //nolint:gomnd // // binary data - hash := make([]byte, 4) - - for i := range blockData { - _, err = v.file.Read(hash) //nolint:errcheck // Will fix later - if err != nil { - log.Print(err) - } - - blockData[i] = binary.LittleEndian.Uint32(hash) - } - - decrypt(blockData, hashString("(block table)", 3)) - - for i := uint32(0); i < v.data.BlockTableEntries; i++ { - v.blockTableEntries = append(v.blockTableEntries, BlockTableEntry{ - FilePosition: blockData[(i * 4)], - CompressedFileSize: blockData[(i*4)+1], - UncompressedFileSize: blockData[(i*4)+2], - Flags: FileFlag(blockData[(i*4)+3]), - }) - } -} - -func decrypt(data []uint32, seed uint32) { - seed2 := uint32(0xeeeeeeee) //nolint:gomnd // Decryption magic - - for i := 0; i < len(data); i++ { - seed2 += cryptoLookup(0x400 + (seed & 0xff)) //nolint:gomnd // Decryption magic - result := data[i] - result ^= seed + seed2 - - seed = ((^seed << 21) + 0x11111111) | (seed >> 11) - seed2 = result + seed2 + (seed2 << 5) + 3 //nolint:gomnd // Decryption magic - data[i] = result - } -} - -func decryptBytes(data []byte, seed uint32) { - seed2 := uint32(0xEEEEEEEE) //nolint:gomnd // Decryption magic - for i := 0; i < len(data)-3; i += 4 { - seed2 += cryptoLookup(0x400 + (seed & 0xFF)) //nolint:gomnd // Decryption magic - result := binary.LittleEndian.Uint32(data[i : i+4]) - result ^= seed + seed2 - seed = ((^seed << 21) + 0x11111111) | (seed >> 11) - seed2 = result + seed2 + (seed2 << 5) + 3 //nolint:gomnd // Decryption magic - - data[i+0] = uint8(result & 0xff) //nolint:gomnd // Decryption magic - data[i+1] = uint8((result >> 8) & 0xff) //nolint:gomnd // Decryption magic - data[i+2] = uint8((result >> 16) & 0xff) //nolint:gomnd // Decryption magic - data[i+3] = uint8((result >> 24) & 0xff) //nolint:gomnd // Decryption magic - } -} - -func hashString(key string, hashType uint32) uint32 { - seed1 := uint32(0x7FED7FED) //nolint:gomnd // Decryption magic - seed2 := uint32(0xEEEEEEEE) //nolint:gomnd // Decryption magic - - /* prepare seeds. */ - for _, char := range strings.ToUpper(key) { - seed1 = cryptoLookup((hashType*0x100)+uint32(char)) ^ (seed1 + seed2) - seed2 = uint32(char) + seed1 + seed2 + (seed2 << 5) + 3 //nolint:gomnd // Decryption magic - } - - return seed1 -} - -// GetFileBlockData gets a block table entry -func (v *MPQ) getFileBlockData(fileName string) (BlockTableEntry, error) { - fileEntry, found := v.hashEntryMap.Find(fileName) - - if !found || fileEntry.BlockIndex >= uint32(len(v.blockTableEntries)) { - return BlockTableEntry{}, errors.New("file not found") - } - - return v.blockTableEntries[fileEntry.BlockIndex], nil -} - -// Close closes the MPQ file -func (v *MPQ) Close() { - err := v.file.Close() - if err != nil { - log.Panic(err) - } -} - -// FileExists checks the mpq to see if the file exists -func (v *MPQ) FileExists(fileName string) bool { - return v.hashEntryMap.Contains(fileName) -} - -// ReadFile reads a file from the MPQ and returns a memory stream -func (v *MPQ) ReadFile(fileName string) ([]byte, error) { - fileBlockData, err := v.getFileBlockData(fileName) - if err != nil { - return []byte{}, err - } - - fileBlockData.FileName = strings.ToLower(fileName) - - fileBlockData.calculateEncryptionSeed() - mpqStream, err := CreateStream(v, fileBlockData, fileName) - - if err != nil { - return []byte{}, err - } - - buffer := make([]byte, fileBlockData.UncompressedFileSize) - mpqStream.Read(buffer, 0, fileBlockData.UncompressedFileSize) - - return buffer, nil -} - -// ReadFileStream reads the mpq file data and returns a stream -func (v *MPQ) ReadFileStream(fileName string) (d2interface.DataStream, error) { - fileBlockData, err := v.getFileBlockData(fileName) - - if err != nil { - return nil, err - } - - fileBlockData.FileName = strings.ToLower(fileName) - fileBlockData.calculateEncryptionSeed() - - mpqStream, err := CreateStream(v, fileBlockData, fileName) - if err != nil { - return nil, err - } - - return &MpqDataStream{stream: mpqStream}, nil -} - -// ReadTextFile reads a file and returns it as a string -func (v *MPQ) ReadTextFile(fileName string) (string, error) { - data, err := v.ReadFile(fileName) - - if err != nil { - return "", err - } - - return string(data), nil -} - -func (v *BlockTableEntry) calculateEncryptionSeed() { - fileName := path.Base(v.FileName) - v.EncryptionSeed = hashString(fileName, 3) - - if !v.HasFlag(FileFixKey) { - return - } - - v.EncryptionSeed = (v.EncryptionSeed + v.FilePosition) ^ v.UncompressedFileSize -} - -// GetFileList returns the list of files in this MPQ -func (v *MPQ) GetFileList() ([]string, error) { - data, err := v.ReadFile("(listfile)") - - if err != nil { - return nil, err - } - - raw := strings.TrimRight(string(data), "\x00") - s := bufio.NewScanner(strings.NewReader(raw)) - - var filePaths []string - - for s.Scan() { - filePath := s.Text() - filePaths = append(filePaths, filePath) - } - - return filePaths, nil -} - -// Path returns the MPQ file path -func (v *MPQ) Path() string { - return v.filePath -} - -// Contains returns bool for whether the given filename exists in the mpq -func (v *MPQ) Contains(filename string) bool { - return v.hashEntryMap.Contains(filename) -} - -// Size returns the size of the mpq in bytes -func (v *MPQ) Size() uint32 { - return v.data.ArchiveSize + return os.Open(path.Join(mpqDir, mpqName)) //nolint:gosec // Will fix later } diff --git a/d2common/d2fileformats/d2mpq/mpq_block.go b/d2common/d2fileformats/d2mpq/mpq_block.go new file mode 100644 index 00000000..112e0e89 --- /dev/null +++ b/d2common/d2fileformats/d2mpq/mpq_block.go @@ -0,0 +1,77 @@ +package d2mpq + +import ( + "io" + "strings" +) + +// FileFlag represents flags for a file record in the MPQ archive +type FileFlag uint32 + +const ( + // FileImplode - File is compressed using PKWARE Data compression library + FileImplode FileFlag = 0x00000100 + // FileCompress - File is compressed using combination of compression methods + FileCompress FileFlag = 0x00000200 + // FileEncrypted - The file is encrypted + FileEncrypted FileFlag = 0x00010000 + // FileFixKey - The decryption key for the file is altered according to the position of the file in the archive + FileFixKey FileFlag = 0x00020000 + // FilePatchFile - The file contains incremental patch for an existing file in base MPQ + FilePatchFile FileFlag = 0x00100000 + // FileSingleUnit - Instead of being divided to 0x1000-bytes blocks, the file is stored as single unit + FileSingleUnit FileFlag = 0x01000000 + // FileDeleteMarker - File is a deletion marker, indicating that the file no longer exists. This is used to allow patch + // archives to delete files present in lower-priority archives in the search chain. The file usually + // has length of 0 or 1 byte and its name is a hash + FileDeleteMarker FileFlag = 0x02000000 + // FileSectorCrc - File has checksums for each sector. Ignored if file is not compressed or imploded. + FileSectorCrc FileFlag = 0x04000000 + // FileExists - Set if file exists, reset when the file was deleted + FileExists FileFlag = 0x80000000 +) + +// Block represents an entry in the block table +type Block struct { // 16 bytes + FilePosition uint32 + CompressedFileSize uint32 + UncompressedFileSize uint32 + Flags FileFlag + // Local Stuff... + FileName string + EncryptionSeed uint32 +} + +// HasFlag returns true if the specified flag is present +func (b *Block) HasFlag(flag FileFlag) bool { + return (b.Flags & flag) != 0 +} + +func (b *Block) calculateEncryptionSeed(fileName string) { + fileName = fileName[strings.LastIndex(fileName, `\`)+1:] + seed := hashString(fileName, 3) + b.EncryptionSeed = (seed + b.FilePosition) ^ b.UncompressedFileSize +} + +//nolint:gomnd // number +func (mpq *MPQ) readBlockTable() error { + if _, err := mpq.file.Seek(int64(mpq.header.BlockTableOffset), io.SeekStart); err != nil { + return err + } + + blockData, err := decryptTable(mpq.file, mpq.header.BlockTableEntries, "(block table)") + if err != nil { + return err + } + + for n, i := uint32(0), uint32(0); i < mpq.header.BlockTableEntries; n, i = n+4, i+1 { + mpq.blocks = append(mpq.blocks, &Block{ + FilePosition: blockData[n], + CompressedFileSize: blockData[n+1], + UncompressedFileSize: blockData[n+2], + Flags: FileFlag(blockData[n+3]), + }) + } + + return nil +} diff --git a/d2common/d2fileformats/d2mpq/mpq_data_stream.go b/d2common/d2fileformats/d2mpq/mpq_data_stream.go index db66260c..3a92064a 100644 --- a/d2common/d2fileformats/d2mpq/mpq_data_stream.go +++ b/d2common/d2fileformats/d2mpq/mpq_data_stream.go @@ -11,14 +11,14 @@ type MpqDataStream struct { // Read reads data from the data stream func (m *MpqDataStream) Read(p []byte) (n int, err error) { - totalRead := m.stream.Read(p, 0, uint32(len(p))) - return int(totalRead), nil + totalRead, err := m.stream.Read(p, 0, uint32(len(p))) + return int(totalRead), err } // Seek sets the position of the data stream func (m *MpqDataStream) Seek(offset int64, whence int) (int64, error) { - m.stream.CurrentPosition = uint32(offset + int64(whence)) - return int64(m.stream.CurrentPosition), nil + m.stream.Position = uint32(offset + int64(whence)) + return int64(m.stream.Position), nil } // Close closes the data stream diff --git a/d2common/d2fileformats/d2mpq/mpq_hash.go b/d2common/d2fileformats/d2mpq/mpq_hash.go new file mode 100644 index 00000000..3f1f744a --- /dev/null +++ b/d2common/d2fileformats/d2mpq/mpq_hash.go @@ -0,0 +1,45 @@ +package d2mpq + +import "io" + +// Hash represents a hashed file entry in the MPQ file +type Hash struct { // 16 bytes + A uint32 + B uint32 + Locale uint16 + Platform uint16 + BlockIndex uint32 +} + +// Name64 returns part A and B as uint64 +func (h *Hash) Name64() uint64 { + return uint64(h.A)<<32 | uint64(h.B) +} + +//nolint:gomnd // number +func (mpq *MPQ) readHashTable() error { + if _, err := mpq.file.Seek(int64(mpq.header.HashTableOffset), io.SeekStart); err != nil { + return err + } + + hashData, err := decryptTable(mpq.file, mpq.header.HashTableEntries, "(hash table)") + if err != nil { + return err + } + + mpq.hashes = make(map[uint64]*Hash) + + for n, i := uint32(0), uint32(0); i < mpq.header.HashTableEntries; n, i = n+4, i+1 { + e := &Hash{ + A: hashData[n], + B: hashData[n+1], + // https://github.com/OpenDiablo2/OpenDiablo2/issues/812 + Locale: uint16(hashData[n+2] >> 16), //nolint:gomnd // // binary data + Platform: uint16(hashData[n+2] & 0xFFFF), //nolint:gomnd // // binary data + BlockIndex: hashData[n+3], + } + mpq.hashes[e.Name64()] = e + } + + return nil +} diff --git a/d2common/d2fileformats/d2mpq/mpq_header.go b/d2common/d2fileformats/d2mpq/mpq_header.go new file mode 100644 index 00000000..f27cfaf7 --- /dev/null +++ b/d2common/d2fileformats/d2mpq/mpq_header.go @@ -0,0 +1,36 @@ +package d2mpq + +import ( + "encoding/binary" + "errors" + "io" +) + +// Header Represents a MPQ file +type Header struct { + Magic [4]byte + HeaderSize uint32 + ArchiveSize uint32 + FormatVersion uint16 + BlockSize uint16 + HashTableOffset uint32 + BlockTableOffset uint32 + HashTableEntries uint32 + BlockTableEntries uint32 +} + +func (mpq *MPQ) readHeader() error { + if _, err := mpq.file.Seek(0, io.SeekStart); err != nil { + return err + } + + if err := binary.Read(mpq.file, binary.LittleEndian, &mpq.header); err != nil { + return err + } + + if string(mpq.header.Magic[:]) != "MPQ\x1A" { + return errors.New("invalid mpq header") + } + + return nil +} diff --git a/d2common/d2fileformats/d2mpq/mpq_stream.go b/d2common/d2fileformats/d2mpq/mpq_stream.go index b6156322..5be4951d 100644 --- a/d2common/d2fileformats/d2mpq/mpq_stream.go +++ b/d2common/d2fileformats/d2mpq/mpq_stream.go @@ -6,8 +6,7 @@ import ( "encoding/binary" "errors" "fmt" - "log" - "strings" + "io" "github.com/JoshVarga/blast" @@ -17,80 +16,63 @@ import ( // Stream represents a stream of data in an MPQ archive type Stream struct { - BlockTableEntry BlockTableEntry - BlockPositions []uint32 - CurrentData []byte - FileName string - MPQData *MPQ - EncryptionSeed uint32 - CurrentPosition uint32 - CurrentBlockIndex uint32 - BlockSize uint32 + Data []byte + Positions []uint32 + MPQ *MPQ + Block *Block + Index uint32 + Size uint32 + Position uint32 } // CreateStream creates an MPQ stream -func CreateStream(mpq *MPQ, blockTableEntry BlockTableEntry, fileName string) (*Stream, error) { - result := &Stream{ - MPQData: mpq, - BlockTableEntry: blockTableEntry, - CurrentBlockIndex: 0xFFFFFFFF, //nolint:gomnd // MPQ magic - } - fileSegs := strings.Split(fileName, `\`) - result.EncryptionSeed = hashString(fileSegs[len(fileSegs)-1], 3) - - if result.BlockTableEntry.HasFlag(FileFixKey) { - result.EncryptionSeed = (result.EncryptionSeed + result.BlockTableEntry.FilePosition) ^ result.BlockTableEntry.UncompressedFileSize +func CreateStream(mpq *MPQ, block *Block, fileName string) (*Stream, error) { + s := &Stream{ + MPQ: mpq, + Block: block, + Index: 0xFFFFFFFF, //nolint:gomnd // MPQ magic } - result.BlockSize = 0x200 << result.MPQData.data.BlockSize //nolint:gomnd // MPQ magic - - if result.BlockTableEntry.HasFlag(FilePatchFile) { - log.Fatal("Patching is not supported") + if s.Block.HasFlag(FileFixKey) { + s.Block.calculateEncryptionSeed(fileName) } - var err error + s.Size = 0x200 << s.MPQ.header.BlockSize //nolint:gomnd // MPQ magic - if (result.BlockTableEntry.HasFlag(FileCompress) || result.BlockTableEntry.HasFlag(FileImplode)) && - !result.BlockTableEntry.HasFlag(FileSingleUnit) { - err = result.loadBlockOffsets() + if s.Block.HasFlag(FilePatchFile) { + return nil, errors.New("patching is not supported") } - return result, err + if (s.Block.HasFlag(FileCompress) || s.Block.HasFlag(FileImplode)) && !s.Block.HasFlag(FileSingleUnit) { + if err := s.loadBlockOffsets(); err != nil { + return nil, err + } + } + + return s, nil } func (v *Stream) loadBlockOffsets() error { - blockPositionCount := ((v.BlockTableEntry.UncompressedFileSize + v.BlockSize - 1) / v.BlockSize) + 1 - v.BlockPositions = make([]uint32, blockPositionCount) - - _, err := v.MPQData.file.Seek(int64(v.BlockTableEntry.FilePosition), 0) - if err != nil { + if _, err := v.MPQ.file.Seek(int64(v.Block.FilePosition), io.SeekStart); err != nil { return err } - mpqBytes := make([]byte, blockPositionCount*4) //nolint:gomnd // MPQ magic + blockPositionCount := ((v.Block.UncompressedFileSize + v.Size - 1) / v.Size) + 1 + v.Positions = make([]uint32, blockPositionCount) - _, err = v.MPQData.file.Read(mpqBytes) - if err != nil { + if err := binary.Read(v.MPQ.file, binary.LittleEndian, &v.Positions); err != nil { return err } - for i := range v.BlockPositions { - idx := i * 4 //nolint:gomnd // MPQ magic - v.BlockPositions[i] = binary.LittleEndian.Uint32(mpqBytes[idx : idx+4]) - } + if v.Block.HasFlag(FileEncrypted) { + decrypt(v.Positions, v.Block.EncryptionSeed-1) - blockPosSize := blockPositionCount << 2 //nolint:gomnd // MPQ magic - - if v.BlockTableEntry.HasFlag(FileEncrypted) { - decrypt(v.BlockPositions, v.EncryptionSeed-1) - - if v.BlockPositions[0] != blockPosSize { - log.Println("Decryption of MPQ failed!") + blockPosSize := blockPositionCount << 2 //nolint:gomnd // MPQ magic + if v.Positions[0] != blockPosSize { return errors.New("decryption of MPQ failed") } - if v.BlockPositions[1] > v.BlockSize+blockPosSize { - log.Println("Decryption of MPQ failed!") + if v.Positions[1] > v.Size+blockPosSize { return errors.New("decryption of MPQ failed") } } @@ -98,16 +80,18 @@ func (v *Stream) loadBlockOffsets() error { return nil } -func (v *Stream) Read(buffer []byte, offset, count uint32) uint32 { - if v.BlockTableEntry.HasFlag(FileSingleUnit) { +func (v *Stream) Read(buffer []byte, offset, count uint32) (readTotal uint32, err error) { + if v.Block.HasFlag(FileSingleUnit) { return v.readInternalSingleUnit(buffer, offset, count) } - toRead := count - readTotal := uint32(0) + var read uint32 + toRead := count for toRead > 0 { - read := v.readInternal(buffer, offset, toRead) + if read, err = v.readInternal(buffer, offset, toRead); err != nil { + return 0, err + } if read == 0 { break @@ -118,149 +102,153 @@ func (v *Stream) Read(buffer []byte, offset, count uint32) uint32 { toRead -= read } - return readTotal + return readTotal, nil } -func (v *Stream) readInternalSingleUnit(buffer []byte, offset, count uint32) uint32 { - if len(v.CurrentData) == 0 { - v.loadSingleUnit() +func (v *Stream) readInternalSingleUnit(buffer []byte, offset, count uint32) (uint32, error) { + if len(v.Data) == 0 { + if err := v.loadSingleUnit(); err != nil { + return 0, err + } } - bytesToCopy := d2math.Min(uint32(len(v.CurrentData))-v.CurrentPosition, count) - - copy(buffer[offset:offset+bytesToCopy], v.CurrentData[v.CurrentPosition:v.CurrentPosition+bytesToCopy]) - - v.CurrentPosition += bytesToCopy - - return bytesToCopy + return v.copy(buffer, offset, v.Position, count) } -func (v *Stream) readInternal(buffer []byte, offset, count uint32) uint32 { - v.bufferData() +func (v *Stream) readInternal(buffer []byte, offset, count uint32) (uint32, error) { + if err := v.bufferData(); err != nil { + return 0, err + } - localPosition := v.CurrentPosition % v.BlockSize - bytesToCopy := d2math.MinInt32(int32(len(v.CurrentData))-int32(localPosition), int32(count)) + localPosition := v.Position % v.Size + return v.copy(buffer, offset, localPosition, count) +} + +func (v *Stream) copy(buffer []byte, offset, pos, count uint32) (uint32, error) { + bytesToCopy := d2math.Min(uint32(len(v.Data))-pos, count) if bytesToCopy <= 0 { - return 0 + return 0, nil } - copy(buffer[offset:offset+uint32(bytesToCopy)], v.CurrentData[localPosition:localPosition+uint32(bytesToCopy)]) + copy(buffer[offset:offset+bytesToCopy], v.Data[pos:pos+bytesToCopy]) + v.Position += bytesToCopy - v.CurrentPosition += uint32(bytesToCopy) - - return uint32(bytesToCopy) + return bytesToCopy, nil } -func (v *Stream) bufferData() { - requiredBlock := v.CurrentPosition / v.BlockSize +func (v *Stream) bufferData() (err error) { + blockIndex := v.Position / v.Size - if requiredBlock == v.CurrentBlockIndex { - return + if blockIndex == v.Index { + return nil } - expectedLength := d2math.Min(v.BlockTableEntry.UncompressedFileSize-(requiredBlock*v.BlockSize), v.BlockSize) - v.CurrentData = v.loadBlock(requiredBlock, expectedLength) - v.CurrentBlockIndex = requiredBlock + expectedLength := d2math.Min(v.Block.UncompressedFileSize-(blockIndex*v.Size), v.Size) + if v.Data, err = v.loadBlock(blockIndex, expectedLength); err != nil { + return err + } + + v.Index = blockIndex + + return nil } -func (v *Stream) loadSingleUnit() { - fileData := make([]byte, v.BlockSize) - - _, err := v.MPQData.file.Seek(int64(v.MPQData.data.HeaderSize), 0) - if err != nil { - log.Print(err) +func (v *Stream) loadSingleUnit() (err error) { + if _, err = v.MPQ.file.Seek(int64(v.MPQ.header.HeaderSize), io.SeekStart); err != nil { + return err } - _, err = v.MPQData.file.Read(fileData) - if err != nil { - log.Print(err) + fileData := make([]byte, v.Size) + + if _, err = v.MPQ.file.Read(fileData); err != nil { + return err } - if v.BlockSize == v.BlockTableEntry.UncompressedFileSize { - v.CurrentData = fileData - return + if v.Size == v.Block.UncompressedFileSize { + v.Data = fileData + return nil } - v.CurrentData = decompressMulti(fileData, v.BlockTableEntry.UncompressedFileSize) + v.Data, err = decompressMulti(fileData, v.Block.UncompressedFileSize) + + return err } -func (v *Stream) loadBlock(blockIndex, expectedLength uint32) []byte { +func (v *Stream) loadBlock(blockIndex, expectedLength uint32) ([]byte, error) { var ( offset uint32 toRead uint32 ) - if v.BlockTableEntry.HasFlag(FileCompress) || v.BlockTableEntry.HasFlag(FileImplode) { - offset = v.BlockPositions[blockIndex] - toRead = v.BlockPositions[blockIndex+1] - offset + if v.Block.HasFlag(FileCompress) || v.Block.HasFlag(FileImplode) { + offset = v.Positions[blockIndex] + toRead = v.Positions[blockIndex+1] - offset } else { - offset = blockIndex * v.BlockSize + offset = blockIndex * v.Size toRead = expectedLength } - offset += v.BlockTableEntry.FilePosition + offset += v.Block.FilePosition data := make([]byte, toRead) - _, err := v.MPQData.file.Seek(int64(offset), 0) - if err != nil { - log.Print(err) + if _, err := v.MPQ.file.Seek(int64(offset), io.SeekStart); err != nil { + return []byte{}, err } - _, err = v.MPQData.file.Read(data) - if err != nil { - log.Print(err) + if _, err := v.MPQ.file.Read(data); err != nil { + return []byte{}, err } - if v.BlockTableEntry.HasFlag(FileEncrypted) && v.BlockTableEntry.UncompressedFileSize > 3 { - if v.EncryptionSeed == 0 { - panic("Unable to determine encryption key") + if v.Block.HasFlag(FileEncrypted) && v.Block.UncompressedFileSize > 3 { + if v.Block.EncryptionSeed == 0 { + return []byte{}, errors.New("unable to determine encryption key") } - decryptBytes(data, blockIndex+v.EncryptionSeed) + decryptBytes(data, blockIndex+v.Block.EncryptionSeed) } - if v.BlockTableEntry.HasFlag(FileCompress) && (toRead != expectedLength) { - if !v.BlockTableEntry.HasFlag(FileSingleUnit) { - data = decompressMulti(data, expectedLength) - } else { - data = pkDecompress(data) + if v.Block.HasFlag(FileCompress) && (toRead != expectedLength) { + if !v.Block.HasFlag(FileSingleUnit) { + return decompressMulti(data, expectedLength) } + + return pkDecompress(data) } - if v.BlockTableEntry.HasFlag(FileImplode) && (toRead != expectedLength) { - data = pkDecompress(data) + if v.Block.HasFlag(FileImplode) && (toRead != expectedLength) { + return pkDecompress(data) } - return data + return data, nil } //nolint:gomnd // Will fix enum values later -func decompressMulti(data []byte /*expectedLength*/, _ uint32) []byte { +func decompressMulti(data []byte /*expectedLength*/, _ uint32) ([]byte, error) { compressionType := data[0] switch compressionType { case 1: // Huffman - panic("huffman decompression not supported") + return []byte{}, errors.New("huffman decompression not supported") case 2: // ZLib/Deflate return deflate(data[1:]) case 8: // PKLib/Impode return pkDecompress(data[1:]) case 0x10: // BZip2 - panic("bzip2 decompression not supported") + return []byte{}, errors.New("bzip2 decompression not supported") case 0x80: // IMA ADPCM Stereo - return d2compression.WavDecompress(data[1:], 2) + return d2compression.WavDecompress(data[1:], 2), nil case 0x40: // IMA ADPCM Mono - return d2compression.WavDecompress(data[1:], 1) + return d2compression.WavDecompress(data[1:], 1), nil case 0x12: - panic("lzma decompression not supported") + return []byte{}, errors.New("lzma decompression not supported") // Combos case 0x22: // sparse then zlib - panic("sparse decompression + deflate decompression not supported") + return []byte{}, errors.New("sparse decompression + deflate decompression not supported") case 0x30: // sparse then bzip2 - panic("sparse decompression + bzip2 decompression not supported") + return []byte{}, errors.New("sparse decompression + bzip2 decompression not supported") case 0x41: sinput := d2compression.HuffmanDecompress(data[1:]) sinput = d2compression.WavDecompress(sinput, 1) @@ -268,69 +256,68 @@ func decompressMulti(data []byte /*expectedLength*/, _ uint32) []byte { copy(tmp, sinput) - return tmp + return tmp, nil case 0x48: // byte[] result = PKDecompress(sinput, outputLength); // return MpqWavCompression.Decompress(new MemoryStream(result), 1); - panic("pk + mpqwav decompression not supported") + return []byte{}, errors.New("pk + mpqwav decompression not supported") case 0x81: sinput := d2compression.HuffmanDecompress(data[1:]) sinput = d2compression.WavDecompress(sinput, 2) tmp := make([]byte, len(sinput)) copy(tmp, sinput) - return tmp + return tmp, nil case 0x88: // byte[] result = PKDecompress(sinput, outputLength); // return MpqWavCompression.Decompress(new MemoryStream(result), 2); - panic("pk + wav decompression not supported") - default: - panic(fmt.Sprintf("decompression not supported for unknown compression type %X", compressionType)) + return []byte{}, errors.New("pk + wav decompression not supported") } + + return []byte{}, fmt.Errorf("decompression not supported for unknown compression type %X", compressionType) } -func deflate(data []byte) []byte { +func deflate(data []byte) ([]byte, error) { b := bytes.NewReader(data) + r, err := zlib.NewReader(b) - if err != nil { - panic(err) + return []byte{}, err } buffer := new(bytes.Buffer) _, err = buffer.ReadFrom(r) if err != nil { - log.Panic(err) + return []byte{}, err } err = r.Close() if err != nil { - log.Panic(err) + return []byte{}, err } - return buffer.Bytes() + return buffer.Bytes(), nil } -func pkDecompress(data []byte) []byte { +func pkDecompress(data []byte) ([]byte, error) { b := bytes.NewReader(data) - r, err := blast.NewReader(b) + r, err := blast.NewReader(b) if err != nil { - panic(err) + return []byte{}, err } buffer := new(bytes.Buffer) - _, err = buffer.ReadFrom(r) - if err != nil { - panic(err) + if _, err = buffer.ReadFrom(r); err != nil { + return []byte{}, err } err = r.Close() if err != nil { - panic(err) + return []byte{}, err } - return buffer.Bytes() + return buffer.Bytes(), nil } diff --git a/d2common/d2interface/archive.go b/d2common/d2interface/archive.go index 5e9d60da..9de0f9ea 100644 --- a/d2common/d2interface/archive.go +++ b/d2common/d2interface/archive.go @@ -8,10 +8,9 @@ type Archive interface { Path() string Contains(string) bool Size() uint32 - Close() - FileExists(fileName string) bool + Close() error ReadFile(fileName string) ([]byte, error) ReadFileStream(fileName string) (DataStream, error) ReadTextFile(fileName string) (string, error) - GetFileList() ([]string, error) + Listfile() ([]string, error) } diff --git a/d2common/d2loader/asset/types/source_types.go b/d2common/d2loader/asset/types/source_types.go index 563c3a7b..32a2b402 100644 --- a/d2common/d2loader/asset/types/source_types.go +++ b/d2common/d2loader/asset/types/source_types.go @@ -37,7 +37,8 @@ func Ext2SourceType(ext string) SourceType { func CheckSourceType(path string) SourceType { // on MacOS, the MPQ's from blizzard don't have file extensions // so we just attempt to init the file as an mpq - if _, err := d2mpq.Load(path); err == nil { + if mpq, err := d2mpq.New(path); err == nil { + _ = mpq.Close() return AssetSourceMPQ } diff --git a/d2common/d2loader/mpq/source.go b/d2common/d2loader/mpq/source.go index 4a326a42..d5b4236d 100644 --- a/d2common/d2loader/mpq/source.go +++ b/d2common/d2loader/mpq/source.go @@ -14,7 +14,7 @@ var _ asset.Source = &Source{} // NewSource creates a new MPQ Source func NewSource(sourcePath string) (asset.Source, error) { - loaded, err := d2mpq.Load(sourcePath) + loaded, err := d2mpq.FromFile(sourcePath) if err != nil { return nil, err } diff --git a/utils/extract-mpq/extract-mpq.go b/utils/extract-mpq/extract-mpq.go index 6c556b77..2d4d42c0 100644 --- a/utils/extract-mpq/extract-mpq.go +++ b/utils/extract-mpq/extract-mpq.go @@ -33,13 +33,13 @@ func main() { } filename := flag.Arg(0) - mpq, err := d2mpq.Load(filename) + mpq, err := d2mpq.FromFile(filename) if err != nil { log.Fatal(err) } - list, err := mpq.GetFileList() + list, err := mpq.Listfile() if err != nil { log.Fatal(err) }