Add Starfield support (#2589)

* Add support for new Starfield BA2 versions

* Add Starfield BA2 LZ4 texture compression support (WIP, not finished)

* Work on replacing DDS.cs with DirectXTexUtility

* Fix reading Starfield BA2s with new DirectXTexUtil

* Fix builder not exporting new Starfield header options

* Fix writing LZ4 chunks in frame format instead of block format

* Rename FO4Archive to BA2Archive, merge SFArchive and FO4Archive

* Clean up testing code & CLI launch settings

* Update changelog

CHANGELOG.md

@@ -1,6 +1,8 @@
 ### Changelog
 
-#### Version - 3.6.2.0 - TBD
+#### Version - 3.7.0.0 - TBD
+* Added Starfield support
+    * Note: Hashes were added earlier, but the earlier version was not fully compatible due to Wabbajack extracting the BA2 archives incorrectly. This has been fixed.
 * Updated GameFinder dependency
 * Updated WebView dependency
 * Updated other dependencies

Wabbajack.App.Wpf/Properties/PublishProfiles/FolderProfile.pubxml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+https://go.microsoft.com/fwlink/?LinkID=208121.
+-->
+<Project>
+  <PropertyGroup>
+    <Configuration>Release</Configuration>
+    <Platform>Any CPU</Platform>
+    <PublishDir>bin\Release\Publish</PublishDir>
+    <PublishProtocol>FileSystem</PublishProtocol>
+    <_TargetId>Folder</_TargetId>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <RuntimeIdentifier>win-x64</RuntimeIdentifier>
+    <SelfContained>true</SelfContained>
+    <PublishSingleFile>false</PublishSingleFile>
+    <PublishReadyToRun>false</PublishReadyToRun>
+  </PropertyGroup>
+</Project>

Wabbajack.CLI/Properties/launchSettings.json

@@ -0,0 +1,7 @@
+{
+  "profiles": {
+    "Wabbajack.CLI": {
+      "commandName": "Project"
+    }
+  }
+}

Wabbajack.Compression.BSA.Test/CompressionTests.cs

@@ -55,6 +55,7 @@ public async Task CanRecreateBSAs(string name, AbsolutePath path)
     {
         if (name == "tes4.bsa") return; // not sure why is is failing
 
+
         var reader = await BSADispatch.Open(path);
 
         var dataStates = await reader.Files
@@ -78,7 +79,6 @@ await dataStates.PDoAll(
 
         var rebuiltStream = new MemoryStream();
         await build.Build(rebuiltStream, CancellationToken.None);
-        rebuiltStream.Position = 0;
 
         var reader2 = await BSADispatch.Open(new MemoryStreamFactory(rebuiltStream, path, path.LastModifiedUtc()));
         await reader.Files.Zip(reader2.Files)

Wabbajack.Compression.BSA/FO4Archive/Builder.cs → Wabbajack.Compression.BSA/BA2Archive/Builder.cs

@@ -10,7 +10,7 @@
 using Wabbajack.DTOs.BSA.FileStates;
 using Wabbajack.Paths.IO;
 
-namespace Wabbajack.Compression.BSA.FO4Archive;
+namespace Wabbajack.Compression.BSA.BA2Archive;
 
 public class Builder : IBuilder
 {
@@ -33,7 +33,7 @@ public async ValueTask AddFile(AFile state, Stream src, CancellationToken token)
 
                     break;
                 case BA2EntryType.DX10:
-                    var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File)state, src, _slab, token);
+                    var resultdx10 = await DX10FileEntryBuilder.Create((BA2DX10File)state, src, _slab, _state.Compression == 3, token);
                     lock (_entries)
                     {
                         _entries.Add(resultdx10);
@@ -59,6 +59,13 @@ public async ValueTask Build(Stream fs, CancellationToken token)
         bw.Write((uint) _entries.Count);
         var tableOffsetLoc = bw.BaseStream.Position;
         bw.Write((ulong) 0);
+        if(_state.Version == 2 || _state.Version == 3)
+        {
+            bw.Write(_state.Unknown1);
+            bw.Write(_state.Unknown2);
+            if (_state.Version == 3)
+                bw.Write(_state.Compression);
+        }
 
         foreach (var entry in _entries) entry.WriteHeader(bw, token);
 

Wabbajack.Compression.BSA/BA2Archive/ChunkBuilder.cs

@@ -0,0 +1,93 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using ICSharpCode.SharpZipLib.Zip.Compression;
+using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
+using K4os.Compression.LZ4;
+using K4os.Compression.LZ4.Encoders;
+using Wabbajack.Common;
+using Wabbajack.DTOs.BSA.FileStates;
+using Wabbajack.DTOs.GitHub;
+
+namespace Wabbajack.Compression.BSA.BA2Archive;
+
+public class ChunkBuilder
+{
+    private BA2Chunk _chunk;
+    private Stream _dataSlab;
+    private long _offsetOffset;
+    private uint _packSize;
+
+    public static async Task<ChunkBuilder> Create(BA2DX10File state, BA2Chunk chunk, Stream src,
+        DiskSlabAllocator slab, bool useLz4Compression, CancellationToken token)
+    {
+        var builder = new ChunkBuilder {_chunk = chunk};
+
+        if (!chunk.Compressed)
+        {
+            builder._dataSlab = slab.Allocate(chunk.FullSz);
+            await src.CopyToLimitAsync(builder._dataSlab, (int) chunk.FullSz, token);
+        }
+        else
+        {
+            if (!useLz4Compression)
+            {
+                var deflater = new Deflater(Deflater.BEST_COMPRESSION);
+                await using var ms = new MemoryStream();
+                await using (var ds = new DeflaterOutputStream(ms, deflater))
+                {
+                    ds.IsStreamOwner = false;
+                    await src.CopyToLimitAsync(ds, (int)chunk.FullSz, token);
+                }
+
+                builder._dataSlab = slab.Allocate(ms.Length);
+                ms.Position = 0;
+                await ms.CopyToLimitAsync(builder._dataSlab, (int)ms.Length, token);
+                builder._packSize = (uint)ms.Length;
+            }
+            else
+            {
+                byte[] full = new byte[chunk.FullSz];
+                await using (var copyStream = new MemoryStream())
+                {
+                    await src.CopyToLimitAsync(copyStream, (int)chunk.FullSz, token);
+                    full = copyStream.ToArray(); 
+                }
+                var maxOutput = LZ4Codec.MaximumOutputSize((int)chunk.FullSz);
+                byte[] compressed = new byte[maxOutput];
+                int compressedSize = LZ4Codec.Encode(full, 0, full.Length, compressed, 0, compressed.Length, LZ4Level.L12_MAX);
+                var ms = new MemoryStream(compressed, 0, compressedSize);
+                builder._dataSlab = slab.Allocate(compressedSize);
+                ms.Position = 0;
+                await ms.CopyToLimitAsync(builder._dataSlab, compressedSize, token);
+                builder._packSize = (uint)compressedSize;
+            }
+        }
+
+        builder._dataSlab.Position = 0;
+
+        return builder;
+    }
+
+    public void WriteHeader(BinaryWriter bw)
+    {
+        _offsetOffset = bw.BaseStream.Position;
+        bw.Write((ulong) 0);
+        bw.Write(_packSize);
+        bw.Write(_chunk.FullSz);
+        bw.Write(_chunk.StartMip);
+        bw.Write(_chunk.EndMip);
+        bw.Write(_chunk.Align);
+    }
+
+    public async ValueTask WriteData(BinaryWriter bw, CancellationToken token)
+    {
+        var pos = bw.BaseStream.Position;
+        bw.BaseStream.Position = _offsetOffset;
+        bw.Write((ulong) pos);
+        bw.BaseStream.Position = pos;
+        await _dataSlab.CopyToLimitAsync(bw.BaseStream, (int) _dataSlab.Length, token);
+        await _dataSlab.DisposeAsync();
+    }
+}

Wabbajack.Compression.BSA/BA2Archive/DX10Entry.cs

@@ -0,0 +1,179 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using DirectXTex;
+using ICSharpCode.SharpZipLib.Zip.Compression;
+using K4os.Compression.LZ4;
+using K4os.Compression.LZ4.Streams;
+using Wabbajack.Common;
+using Wabbajack.Compression.BSA.BA2Archive;
+using Wabbajack.DTOs.BSA.FileStates;
+using Wabbajack.DTOs.Streams;
+using Wabbajack.Paths;
+
+namespace Wabbajack.Compression.BSA.BA2Archive;
+
+public class DX10Entry : IBA2FileEntry
+{
+    private readonly Reader _bsa;
+    private ushort _chunkHdrLen;
+    private List<TextureChunk> _chunks;
+    private uint _dirHash;
+    private string _extension;
+    private byte _format;
+    private ushort _height;
+    private int _index;
+    private uint _nameHash;
+    private byte _numChunks;
+    private byte _numMips;
+    private ushort _unk16;
+    private byte _unk8;
+    private ushort _width;
+    private readonly byte _isCubemap;
+    private readonly byte _tileMode;
+
+    public DX10Entry(Reader ba2Reader, int idx)
+    {
+        _bsa = ba2Reader;
+        var _rdr = ba2Reader._rdr;
+        _nameHash = _rdr.ReadUInt32();
+        FullPath = _nameHash.ToString("X");
+        _extension = Encoding.UTF8.GetString(_rdr.ReadBytes(4));
+        _dirHash = _rdr.ReadUInt32();
+        _unk8 = _rdr.ReadByte();
+        _numChunks = _rdr.ReadByte();
+        _chunkHdrLen = _rdr.ReadUInt16();
+        _height = _rdr.ReadUInt16();
+        _width = _rdr.ReadUInt16();
+        _numMips = _rdr.ReadByte();
+        _format = _rdr.ReadByte();
+        _isCubemap = _rdr.ReadByte();
+        _tileMode = _rdr.ReadByte();
+        _index = idx;
+
+        _chunks = Enumerable.Range(0, _numChunks)
+            .Select(_ => new TextureChunk(_rdr))
+            .ToList();
+    }
+    private DirectXTexUtility.TexMetadata? _metadata = null;
+
+    public DirectXTexUtility.TexMetadata Metadata
+    {
+        get
+        {
+            if (_metadata == null)
+                _metadata = DirectXTexUtility.GenerateMetadata(_width, _height, _numMips, (DirectXTexUtility.DXGIFormat)_format, _isCubemap == 1);
+            return (DirectXTexUtility.TexMetadata)_metadata;
+        }
+    }
+
+    private uint _headerSize = 0;
+    public uint HeaderSize
+    {
+        get
+        {
+            if (_headerSize > 0)
+                return _headerSize;
+            uint size = 0;
+            size += (uint)Marshal.SizeOf(DirectXTexUtility.DDSHeader.DDSMagic);
+            size += (uint)Marshal.SizeOf<DirectXTexUtility.DDSHeader>();
+            var pixelFormat = DirectXTexUtility.GetPixelFormat(Metadata);
+            var hasDx10Header = DirectXTexUtility.HasDx10Header(pixelFormat);
+            if (hasDx10Header)
+                size += (uint)Marshal.SizeOf<DirectXTexUtility.DX10Header>();
+
+            return _headerSize = size;
+        }
+    }
+
+    public string FullPath { get; set; }
+
+    public RelativePath Path => FullPath.ToRelativePath();
+    public uint Size => (uint)_chunks.Sum(f => f._fullSz) + HeaderSize;
+
+    public AFile State => new BA2DX10File
+    {
+        Path = Path,
+        NameHash = _nameHash,
+        Extension = _extension,
+        DirHash = _dirHash,
+        Unk8 = _unk8,
+        ChunkHdrLen = _chunkHdrLen,
+        Height = _height,
+        Width = _width,
+        NumMips = _numMips,
+        PixelFormat = _format,
+        IsCubeMap = _isCubemap,
+        TileMode = _tileMode,
+        Index = _index,
+        Chunks = _chunks.Select(ch => new BA2Chunk
+        {
+            FullSz = ch._fullSz,
+            StartMip = ch._startMip,
+            EndMip = ch._endMip,
+            Align = ch._align,
+            Compressed = ch._packSz != 0
+        }).ToArray()
+    };
+
+    public async ValueTask CopyDataTo(Stream output, CancellationToken token)
+    {
+        var bw = new BinaryWriter(output);
+
+        WriteHeader(bw);
+
+        await using var fs = await _bsa._streamFactory.GetStream();
+        using var br = new BinaryReader(fs);
+        foreach (var chunk in _chunks)
+        {
+            var full = new byte[chunk._fullSz];
+            var isCompressed = chunk._packSz != 0;
+
+            br.BaseStream.Seek((long)chunk._offset, SeekOrigin.Begin);
+
+            if (!isCompressed)
+            {
+                await br.BaseStream.ReadAsync(full, token);
+            }
+            else
+            {
+                var compressed = new byte[chunk._packSz];
+                await br.BaseStream.ReadAsync(compressed, token);
+                if (_bsa._compression == 3)
+                {
+                    LZ4Codec.PartialDecode(compressed, full);
+                }
+                else
+                {
+                    var inflater = new Inflater();
+                    inflater.SetInput(compressed);
+                    inflater.Inflate(full);
+                }
+            }
+
+            await bw.BaseStream.WriteAsync(full, token);
+        }
+    }
+
+
+    public async ValueTask<IStreamFactory> GetStreamFactory(CancellationToken token)
+    {
+        var ms = new MemoryStream();
+        await CopyDataTo(ms, token);
+        ms.Position = 0;
+        return new MemoryStreamFactory(ms, Path, _bsa._streamFactory.LastModifiedUtc);
+    }
+
+    private void WriteHeader(BinaryWriter bw)
+    {
+        DirectXTexUtility.GenerateDDSHeader(Metadata, DirectXTexUtility.DDSFlags.FORCEDX10EXTMISC2, out var header, out var header10);
+        var headerBytes = DirectXTexUtility.EncodeDDSHeader(header, header10);
+        bw.Write(headerBytes);
+    }
+}
+