Merge pull request #887 from majorro/improve-rar-memory-usage

Improve rar memory usage

Directory.Packages.props

@@ -7,6 +7,7 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
     <PackageVersion Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageVersion Include="SimpleExec" Version="12.0.0" />
+    <PackageVersion Include="System.Buffers" Version="4.6.0" />
     <PackageVersion Include="System.Memory" Version="4.5.5" />
     <PackageVersion Include="System.Text.Encoding.CodePages" Version="8.0.0" />
     <PackageVersion Include="xunit" Version="2.9.0" />
@@ -15,4 +16,4 @@
     <PackageVersion Include="ZstdSharp.Port" Version="0.8.1" />
     <GlobalPackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
   </ItemGroup>
-</Project>
+</Project>

src/SharpCompress/Archives/Rar/RarArchive.cs

@@ -14,6 +14,7 @@ namespace SharpCompress.Archives.Rar;
 
 public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
 {
+    private bool _disposed;
     internal Lazy<IRarUnpack> UnpackV2017 { get; } =
         new(() => new Compressors.Rar.UnpackV2017.Unpack());
     internal Lazy<IRarUnpack> UnpackV1 { get; } = new(() => new Compressors.Rar.UnpackV1.Unpack());
@@ -25,6 +26,20 @@ public class RarArchive : AbstractArchive<RarArchiveEntry, RarVolume>
     private RarArchive(SourceStream sourceStream)
         : base(ArchiveType.Rar, sourceStream) { }
 
+    public override void Dispose()
+    {
+        if (!_disposed)
+        {
+            if (UnpackV1.IsValueCreated && UnpackV1.Value is IDisposable unpackV1)
+            {
+                unpackV1.Dispose();
+            }
+
+            _disposed = true;
+            base.Dispose();
+        }
+    }
+
     protected override IEnumerable<RarArchiveEntry> LoadEntries(IEnumerable<RarVolume> volumes) =>
         RarArchiveEntryFactory.GetEntries(this, volumes, ReaderOptions);
 

src/SharpCompress/Compressors/Rar/RarCRC.cs

@@ -9,7 +9,7 @@ internal static class RarCRC
     public static uint CheckCrc(uint startCrc, byte b) =>
         (crcTab[((int)startCrc ^ b) & 0xff] ^ (startCrc >> 8));
 
-    public static uint CheckCrc(uint startCrc, byte[] data, int offset, int count)
+    public static uint CheckCrc(uint startCrc, ReadOnlySpan<byte> data, int offset, int count)
     {
         var size = Math.Min(data.Length - offset, count);
 

src/SharpCompress/Compressors/Rar/RarStream.cs

@@ -1,6 +1,7 @@
 #nullable disable
 
 using System;
+using System.Buffers;
 using System.IO;
 using SharpCompress.Common.Rar.Headers;
 
@@ -14,7 +15,7 @@ internal class RarStream : Stream
 
     private bool fetch;
 
-    private byte[] tmpBuffer = BufferPool.Rent(65536);
+    private byte[] tmpBuffer = ArrayPool<byte>.Shared.Rent(65536);
     private int tmpOffset;
     private int tmpCount;
 
@@ -42,7 +43,7 @@ protected override void Dispose(bool disposing)
         {
             if (disposing)
             {
-                BufferPool.Return(this.tmpBuffer);
+                ArrayPool<byte>.Shared.Return(this.tmpBuffer);
                 this.tmpBuffer = null;
             }
             isDisposed = true;
@@ -143,11 +144,11 @@ private void EnsureBufferCapacity(int count)
                 this.tmpBuffer.Length * 2 > this.tmpCount + count
                     ? this.tmpBuffer.Length * 2
                     : this.tmpCount + count;
-            var newBuffer = BufferPool.Rent(newLength);
+            var newBuffer = ArrayPool<byte>.Shared.Rent(newLength);
             Buffer.BlockCopy(this.tmpBuffer, 0, newBuffer, 0, this.tmpCount);
             var oldBuffer = this.tmpBuffer;
             this.tmpBuffer = newBuffer;
-            BufferPool.Return(oldBuffer);
+            ArrayPool<byte>.Shared.Return(oldBuffer);
         }
     }
 }

src/SharpCompress/Compressors/Rar/UnpackV1/Unpack.cs

@@ -1,6 +1,7 @@
 #nullable disable
 
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
 using SharpCompress.Common;
@@ -12,14 +13,28 @@
 
 namespace SharpCompress.Compressors.Rar.UnpackV1;
 
-internal sealed partial class Unpack : BitInput, IRarUnpack
+internal sealed partial class Unpack : BitInput, IRarUnpack, IDisposable
 {
     private readonly BitInput Inp;
+    private bool disposed;
 
     public Unpack() =>
         // to ease in porting Unpack50.cs
         Inp = this;
 
+    public void Dispose()
+    {
+        if (!disposed)
+        {
+            if (!externalWindow)
+            {
+                ArrayPool<byte>.Shared.Return(window);
+                window = null;
+            }
+            disposed = true;
+        }
+    }
+
     public bool FileExtracted { get; private set; }
 
     public long DestSize
@@ -74,7 +89,7 @@ public int Char
 
     private BlockTypes unpBlockType;
 
-    //private bool externalWindow;
+    private bool externalWindow;
 
     private long writtenFileSize;
 
@@ -111,15 +126,14 @@ public int Char
 
     private void Init(byte[] window)
     {
-        if (window is null)
+        if (this.window is null && window is null)
         {
-            this.window = new byte[PackDef.MAXWINSIZE];
+            this.window = ArrayPool<byte>.Shared.Rent(PackDef.MAXWINSIZE);
         }
-        else
+        else if (window is not null)
         {
             this.window = window;
-
-            //externalWindow = true;
+            externalWindow = true;
         }
         inAddr = 0;
         UnpInitData(false);
@@ -175,31 +189,24 @@ public void DoUnpack()
 
     private void UnstoreFile()
     {
-        var buffer = new byte[0x10000];
-        while (true)
+        Span<byte> buffer = stackalloc byte[(int)Math.Min(0x10000, destUnpSize)];
+        do
         {
-            var code = readStream.Read(buffer, 0, (int)Math.Min(buffer.Length, destUnpSize));
+            var code = readStream.Read(buffer);
             if (code == 0 || code == -1)
             {
                 break;
             }
             code = code < destUnpSize ? code : (int)destUnpSize;
-            writeStream.Write(buffer, 0, code);
-            if (destUnpSize >= 0)
-            {
-                destUnpSize -= code;
-            }
-            if (suspended)
-            {
-                return;
-            }
-        }
+            writeStream.Write(buffer.Slice(0, code));
+            destUnpSize -= code;
+        } while (!suspended && destUnpSize > 0);
     }
 
     private void Unpack29(bool solid)
     {
-        var DDecode = new int[PackDef.DC];
-        var DBits = new byte[PackDef.DC];
+        Span<int> DDecode = stackalloc int[PackDef.DC];
+        Span<byte> DBits = stackalloc byte[PackDef.DC];
 
         int Bits;
 
@@ -859,9 +866,9 @@ private bool ReadEndOfBlock()
 
     private bool ReadTables()
     {
-        var bitLength = new byte[PackDef.BC];
+        Span<byte> bitLength = stackalloc byte[PackDef.BC];
+        Span<byte> table = stackalloc byte[PackDef.HUFF_TABLE_SIZE];
 
-        var table = new byte[PackDef.HUFF_TABLE_SIZE];
         if (inAddr > readTop - 25)
         {
             if (!unpReadBuf())
@@ -989,7 +996,7 @@ private bool ReadTables()
 
         // memcpy(unpOldTable,table,sizeof(unpOldTable));
 
-        Buffer.BlockCopy(table, 0, unpOldTable, 0, unpOldTable.Length);
+        table.CopyTo(unpOldTable);
         return (true);
     }
 
@@ -1190,7 +1197,7 @@ private bool AddVMCode(int firstByte, List<byte> vmCode, int length)
             {
                 return (false);
             }
-            var VMCode = new byte[VMCodeSize];
+            Span<byte> VMCode = stackalloc byte[VMCodeSize];
             for (var I = 0; I < VMCodeSize; I++)
             {
                 if (Inp.Overflow(3))

src/SharpCompress/Compressors/Rar/UnpackV1/Unpack20.cs

@@ -391,8 +391,8 @@ private void CopyString20(int Length, int Distance)
 
     private bool ReadTables20()
     {
-        var BitLength = new byte[PackDef.BC20];
-        var Table = new byte[PackDef.MC20 * 4];
+        Span<byte> BitLength = stackalloc byte[PackDef.BC20];
+        Span<byte> Table = stackalloc byte[PackDef.MC20 * 4];
         int TableSize,
             N,
             I;

src/SharpCompress/Compressors/Rar/UnpackV1/UnpackUtility.cs

@@ -181,7 +181,12 @@ internal static int decodeNumber(this BitInput input, Decode.Decode dec)
         return (dec.DecodeNum[N]);
     }
 
-    internal static void makeDecodeTables(byte[] lenTab, int offset, Decode.Decode dec, int size)
+    internal static void makeDecodeTables(
+        Span<byte> lenTab,
+        int offset,
+        Decode.Decode dec,
+        int size
+    )
     {
         Span<int> lenCount = stackalloc int[16];
         Span<int> tmpPos = stackalloc int[16];

src/SharpCompress/Compressors/Rar/VM/RarVM.cs

@@ -776,14 +776,14 @@ private bool ExecuteCode(List<VMPreparedCommand> preparedCode, int cmdCount)
         }
     }
 
-    public void prepare(byte[] code, int codeSize, VMPreparedProgram prg)
+    public void prepare(ReadOnlySpan<byte> code, int codeSize, VMPreparedProgram prg)
     {
         InitBitInput();
         var cpLength = Math.Min(MAX_SIZE, codeSize);
 
         // memcpy(inBuf,Code,Min(CodeSize,BitInput::MAX_SIZE));
 
-        Buffer.BlockCopy(code, 0, InBuf, 0, cpLength);
+        code.Slice(0, cpLength).CopyTo(InBuf);
         byte xorSum = 0;
         for (var i = 1; i < codeSize; i++)
         {
@@ -1105,7 +1105,7 @@ internal static int ReadData(BitInput rarVM)
         }
     }
 
-    private VMStandardFilters IsStandardFilter(byte[] code, int codeSize)
+    private VMStandardFilters IsStandardFilter(ReadOnlySpan<byte> code, int codeSize)
     {
         VMStandardFilterSignature[] stdList =
         {

src/SharpCompress/Readers/AbstractReader.cs

@@ -44,7 +44,7 @@ internal AbstractReader(ReaderOptions options, ArchiveType archiveType)
 
     #region IDisposable Members
 
-    public void Dispose()
+    public virtual void Dispose()
     {
         _entriesForCurrentReadStream?.Dispose();
         Volume?.Dispose();

src/SharpCompress/Readers/Rar/RarReader.cs

@@ -13,6 +13,7 @@ namespace SharpCompress.Readers.Rar;
 /// </summary>
 public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
 {
+    private bool _disposed;
     private RarVolume? volume;
     private Lazy<IRarUnpack> UnpackV2017 { get; } =
         new(() => new Compressors.Rar.UnpackV2017.Unpack());
@@ -21,6 +22,20 @@ public abstract class RarReader : AbstractReader<RarReaderEntry, RarVolume>
     internal RarReader(ReaderOptions options)
         : base(options, ArchiveType.Rar) { }
 
+    public override void Dispose()
+    {
+        if (!_disposed)
+        {
+            if (UnpackV1.IsValueCreated && UnpackV1.Value is IDisposable unpackV1)
+            {
+                unpackV1.Dispose();
+            }
+
+            _disposed = true;
+            base.Dispose();
+        }
+    }
+
     protected abstract void ValidateArchive(RarVolume archive);
 
     public override RarVolume? Volume => volume;

src/SharpCompress/SharpCompress.csproj

@@ -33,6 +33,7 @@
     <IsTrimmable>true</IsTrimmable>
   </PropertyGroup>
   <ItemGroup>
+		<PackageReference Include="System.Buffers" />
 		<PackageReference Include="ZstdSharp.Port" />
   </ItemGroup>
 	<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.1' ">