From 4f819ee187a0bf528c4a7520b5a77aa990c12287 Mon Sep 17 00:00:00 2001 From: Suman Tokuri Date: Wed, 18 Jan 2023 12:49:19 +0530 Subject: [PATCH] Reduce Memory usage while creating pnp template Intead of using MemoryStreams use FileStreams --- .../Connectors/OpenXML/Model/PnPInfo.cs | 10 +++ .../Connectors/OpenXML/PnPPackage.cs | 31 ++++++++- .../Connectors/OpenXML/PnPPackageHelper.cs | 68 ++++++++++++++++--- .../Connectors/OpenXMLConnector.cs | 66 ++++++++++++++---- .../Connectors/SharePointConnector.cs | 1 + .../Providers/Xml/XMLTemplateProvider.cs | 13 ++-- 6 files changed, 160 insertions(+), 29 deletions(-) diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs index c0946d019..69180333e 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/Model/PnPInfo.cs @@ -28,5 +28,15 @@ public class PnPInfo /// Defines the mapping between original file names and OpenXML file names /// public PnPFilesMap FilesMap { get; set; } + + /// + /// Specifies whether the file streams should be used for file contenets instead of the MemoryStream. + /// + public bool UseFileStreams { get; set; } = false; + + /// + /// Path to be used for saving file contenets instead of the MemoryStream. + /// + public string PnPFilesPath { get; set; } } } diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs index dcf2dfb5a..efd9c3780 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackage.cs @@ -211,7 +211,10 @@ public static PnPPackage Open(Stream stream, FileMode mode, FileAccess access) { Package = Package.Open(stream, mode, access) }; - package.EnsureMandatoryPackageComponents(); + if (mode != FileMode.Create) + { + package.EnsureMandatoryPackageComponents(); + } return package; } @@ -228,6 +231,21 @@ public void AddFile(string fileName, Byte[] value) SetPackagePartValue(value, part); } + /// + /// Adds file to the package + /// + /// Name of the file + /// Stream of the file + public void AddFilePart(string fileName, Stream stream) + { + fileName = fileName.TrimStart('/'); + string uriStr = U_DIR_FILES + fileName; + // create part + Uri uri = GetUri(uriStr); + PackagePart part = Package.CreatePart(uri, CT_FILE, PACKAGE_COMPRESSION_LEVEL); + SetPackagePartValue(stream, part); + } + /// /// Clear the files having package parts with specific relationship type /// @@ -349,7 +367,7 @@ private T GetXamlSerializedPackagePartValue(PackagePart part) where T : class return obj; } - private void SetXamlSerializedPackagePartValue(object value, PackagePart part) + static public void SetXamlSerializedPackagePartValue(object value, PackagePart part) { if (value == null) return; @@ -383,6 +401,15 @@ private void SetPackagePartValue(Byte[] value, PackagePart part) } } + private void SetPackagePartValue(Stream stream, PackagePart part) + { + using (Stream destStream = part.GetStream(FileMode.OpenOrCreate)) + { + stream.Position = 0; + stream.CopyTo(destStream); + } + } + private PackagePart CreatePackagePart(string relType, string contentType, string uriStr, PackagePart parent) { // create part & relationship diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs index 4b8e28875..98f0b51ff 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXML/PnPPackageHelper.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.IO.Packaging; namespace PnP.Framework.Provisioning.Connectors.OpenXML { @@ -28,6 +29,14 @@ public static MemoryStream PackTemplateAsStream(this PnPInfo pnpInfo) return stream; } + public static void PackTemplateToStream(this PnPInfo pnpInfo, Stream stream) + { + using (PnPPackage package = PnPPackage.Open(stream, FileMode.Create, FileAccess.Write)) + { + SavePnPPackage(pnpInfo, package); + } + } + /// /// Packs template as a stream array /// @@ -113,20 +122,63 @@ private static PnPInfo LoadPnPPackage(PnPPackage package) private static void SavePnPPackage(PnPInfo pnpInfo, PnPPackage package) { - package.Manifest = pnpInfo.Manifest; - package.Properties = pnpInfo.Properties; Debug.Assert(pnpInfo.Files.TrueForAll(f => !string.IsNullOrWhiteSpace(f.InternalName)), "All files need an InternalFileName"); - package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))); - package.ClearFiles(); - if (pnpInfo.Files != null) + if (!pnpInfo.UseFileStreams) { - foreach (PnPFileInfo file in pnpInfo.Files) + package.Manifest = pnpInfo.Manifest; + package.Properties = pnpInfo.Properties; + package.FilesMap = new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))); + package.ClearFiles(); + if (pnpInfo.Files != null) { - package.AddFile(file.InternalName, file.Content); + foreach (PnPFileInfo file in pnpInfo.Files) + { + package.AddFile(file.InternalName, file.Content); + } } } - } + else + { + // Package with Create mode does not allow reads. Prepare and write the parts along with their relations in one go. + // This is a workaround for(Memory leak with Append mode) https://github.com/dotnet/runtime/issues/1544 + var uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_MANIFEST, UriKind.Relative); + PackagePart manifest = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_MANIFEST, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + PnPPackage.SetXamlSerializedPackagePartValue(pnpInfo.Manifest, manifest); + package.Package.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_MANIFEST); + uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_PROPERTIES, UriKind.Relative); + PackagePart properties = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_PROPERTIES, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_PROPERTIES); + + uriPath = new Uri(PnPPackage.U_FILES_ORIGIN, UriKind.Relative); + PackagePart filesOrigin = package.Package.CreatePart(uriPath, PnPPackage.CT_ORIGIN, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_ORIGIN); + + uriPath = new Uri(PnPPackage.U_PROVISIONINGTEMPLATE_FILES_MAP, UriKind.Relative); + PackagePart filesMap = package.Package.CreatePart(uriPath, PnPPackage.CT_PROVISIONINGTEMPLATE_FILES_MAP, PnPPackage.PACKAGE_COMPRESSION_LEVEL); + PnPPackage.SetXamlSerializedPackagePartValue(new PnPFilesMap(pnpInfo.Files.ToDictionary(f => f.InternalName, f => Path.Combine(f.Folder, f.OriginalName).Replace('\\', '/').TrimStart('/'))), filesMap); + manifest.CreateRelationship(uriPath, TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILES_MAP); + + if (pnpInfo.Files != null) + { + foreach (PnPFileInfo file in pnpInfo.Files) + { + +#if NET6_0_OR_GREATER + // Set the file stream options to delete the files automatically once closed. + var fileStreamOptions = new FileStreamOptions { Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.DeleteOnClose, Share = FileShare.Delete }; + using (FileStream fs = File.Open(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'), fileStreamOptions)) +#else + using (FileStream fs = File.OpenRead(Path.Combine(pnpInfo.PnPFilesPath, file.InternalName).Replace('\\', '/').TrimStart('/'))) +#endif + { + package.AddFilePart(file.InternalName, fs); + filesOrigin.CreateRelationship(new Uri(PnPPackage.U_DIR_FILES + file.InternalName.TrimStart('/'), UriKind.Relative), TargetMode.Internal, PnPPackage.R_PROVISIONINGTEMPLATE_FILE); + } + } + } + } + } #endregion } } diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs index fe04ca5fe..759b2de45 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/OpenXMLConnector.cs @@ -52,10 +52,12 @@ public OpenXMLConnector(Stream packageStream) : base() /// The Author of the .PNP package file, if any. Optional /// The X.509 certificate to use for digital signature of the template, optional /// The name of the tempalte file, optional + /// Wheter to to use FileStream instead of MemoryStream while reading files, optional + /// Optional path to save files when using FileStream instead of MemoryStream while reading files, optional public OpenXMLConnector(string packageFileName, FileConnectorBase persistenceConnector, string author = null, - X509Certificate2 signingCertificate = null, string templateFileName = null) + X509Certificate2 signingCertificate = null, string templateFileName = null, bool useFileStreams = false, string pnpFilesPath = null) : base() { if (string.IsNullOrEmpty(packageFileName)) @@ -99,6 +101,8 @@ public OpenXMLConnector(string packageFileName, Author = !string.IsNullOrEmpty(author) ? author : string.Empty, TemplateFileName = templateFileName ?? "" }, + UseFileStreams = useFileStreams, + PnPFilesPath = useFileStreams ? (string.IsNullOrEmpty(pnpFilesPath) ? persistenceConnector.GetConnectionString() : pnpFilesPath) : string.Empty, }; } } @@ -294,24 +298,48 @@ public override void SaveFileStream(string fileName, string container, Stream st try { - var memoryStream = stream.ToMemoryStream(); - byte[] bytes = memoryStream.ToArray(); - // Check if the file already exists in the package var existingFile = pnpInfo.Files.FirstOrDefault(f => f.OriginalName.Equals(fileName, StringComparison.InvariantCultureIgnoreCase) && f.Folder.Equals(container, StringComparison.InvariantCultureIgnoreCase)); if (existingFile != null) { - existingFile.Content = bytes; + if (pnpInfo.UseFileStreams) + { + using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, existingFile.InternalName).Replace('\\', '/').TrimStart('/'))) + { + stream.CopyTo(fs); + } + } + else + { + existingFile.Content = stream.ToMemoryStream().ToArray(); + } } else { - pnpInfo.Files.Add(new PnPFileInfo + if (pnpInfo.UseFileStreams) + { + var internalFileName = fileName.AsInternalFilename(); + using (FileStream fs = File.Create(Path.Combine(pnpInfo.PnPFilesPath, internalFileName).Replace('\\', '/').TrimStart('/'))) + { + stream.CopyTo(fs); + } + pnpInfo.Files.Add(new PnPFileInfo + { + InternalName = internalFileName, + OriginalName = fileName, + Folder = container, + }); + } + else { - InternalName = fileName.AsInternalFilename(), - OriginalName = fileName, - Folder = container, - Content = bytes, - }); + pnpInfo.Files.Add(new PnPFileInfo + { + InternalName = fileName.AsInternalFilename(), + OriginalName = fileName, + Folder = container, + Content = stream.ToMemoryStream().ToArray(), + }); + } } Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_OpenXML_FileSaved, fileName, container); @@ -436,8 +464,20 @@ internal override string GetContainer() /// public void Commit() { - MemoryStream stream = pnpInfo.PackTemplateAsStream(); - persistenceConnector.SaveFileStream(this.packageFileName, stream); + if (pnpInfo.UseFileStreams) + { + using (FileStream fs = File.Create(Path.Combine(persistenceConnector.GetConnectionString(), this.packageFileName).Replace('\\', '/').TrimStart('/'))) + { + pnpInfo.PackTemplateToStream(fs); + } + } + else + { + using (MemoryStream stream = pnpInfo.PackTemplateAsStream()) + { + persistenceConnector.SaveFileStream(this.packageFileName, stream); + } + } } #endregion diff --git a/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs b/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs index 942b34ef3..0fc7aa02f 100644 --- a/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs +++ b/src/lib/PnP.Framework/Provisioning/Connectors/SharePointConnector.cs @@ -440,6 +440,7 @@ private MemoryStream GetFileFromStorage(string fileName, string container) cc.ExecuteQueryRetry(); streamResult.Value.CopyTo(stream); + streamResult.Value.Dispose(); Log.Info(Constants.LOGGING_SOURCE, CoreResources.Provisioning_Connectors_SharePoint_FileRetrieved, fileName, GetConnectionString(), container); diff --git a/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs b/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs index 6e96a1688..99e0c2716 100644 --- a/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs +++ b/src/lib/PnP.Framework/Provisioning/Providers/Xml/XMLTemplateProvider.cs @@ -259,13 +259,14 @@ public override void SaveAs(ProvisioningHierarchy hierarchy, string uri, ITempla } formatter.Initialize(this); - var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy); - - this.Connector.SaveFileStream(uri, stream); - - if (this.Connector is ICommitableFileConnector) + using (var stream = ((IProvisioningHierarchyFormatter)formatter).ToFormattedHierarchy(hierarchy)) { - ((ICommitableFileConnector)this.Connector).Commit(); + this.Connector.SaveFileStream(uri, stream); + + if (this.Connector is ICommitableFileConnector) + { + ((ICommitableFileConnector)this.Connector).Commit(); + } } }