Merge pull request #16951 from Nexus-Mods/collections-improvements

Collections improvements

src/extensions/download_management/actions/settings.ts

@@ -8,3 +8,4 @@ export const setShowDLDropzone = safeCreateAction('SET_SHOW_DL_DROPZONE', show =
 export const setShowDLGraph = safeCreateAction('SET_SHOW_DL_GRAPH', show => show);
 export const setCopyOnIFF = safeCreateAction('SET_COPY_ON_IFF', enabled => enabled);
 export const setMaxBandwidth = safeCreateAction('SET_MAX_BANDWIDTH', bandwidth => bandwidth);
+export const setCollectionConcurrency = safeCreateAction('SET_COLLECTION_INSTALL_DOWNLOAD_CONCURRENCY', enabled => enabled);

src/extensions/download_management/reducers/settings.ts

@@ -23,16 +23,19 @@ export const settingsReducer: IReducerSpec<ISettingsDownloads> = {
       setSafe(state, ['copyOnIFF'], payload),
     [actions.setMaxBandwidth as any]: (state, payload) =>
       setSafe(state, ['maxBandwidth'], payload),
+    [actions.setCollectionConcurrency as any]: (state, payload) =>
+      setSafe(state, ['collectionsInstallWhileDownloading'], payload),
   },
   defaults: {
     minChunkSize: 1024 * 1024,
-    maxChunks: 4,
+    maxChunks: 10,
     maxParallelDownloads: 1,
     maxBandwidth: 0,
     path: path.join('{USERDATA}', 'downloads'),
     showDropzone: true,
     showGraph: true,
     copyOnIFF: false,
+    collectionsInstallWhileDownloading: false,
   },
   verifiers: {
     path: {

src/extensions/download_management/views/Settings.tsx

@@ -25,7 +25,7 @@ import { Campaign, ciEqual, isChildPath, isPathValid, isReservedDirectory,
          nexusModsURL, Section, Source } from '../../../util/util';
 import getTextMod from '../../mod_management/texts';
 import { PREMIUM_PATH } from '../../nexus_integration/constants';
-import { setCopyOnIFF, setDownloadPath, setMaxBandwidth, setMaxDownloads } from '../actions/settings';
+import { setCopyOnIFF, setDownloadPath, setMaxBandwidth, setMaxDownloads, setCollectionConcurrency } from '../actions/settings';
 import { setTransferDownloads } from '../actions/transactions';
 
 import { DOWNLOADS_DIR_TAG, writeDownloadsTag } from '../util/downloadDirectory';
@@ -54,6 +54,7 @@ interface IConnectedProps {
   instanceId: string;
   copyOnIFF: boolean;
   maxBandwidth: number;
+  collectionsInstallWhileDownloading: boolean;
 }
 
 interface IActionProps {
@@ -66,6 +67,7 @@ interface IActionProps {
                 allowReport: boolean, isBBCode?: boolean) => void;
   onSetCopyOnIFF: (enabled: boolean) => void;
   onSetMaxBandwidth: (bps: number) => void;
+  onSetCollectionConcurrency: (enabled: boolean) => void;
 }
 
 type IProps = IActionProps & IConnectedProps;
@@ -223,6 +225,12 @@ class Settings extends ComponentEx<IProps, IComponentState> {
           </div>
         </FormGroup>
         <FormGroup>
+          <Toggle
+            checked={this.props.collectionsInstallWhileDownloading}
+            onToggle={this.toggleCollectionInstallConcurrency}
+          >
+            {t('Install mods during collection downloads')}
+          </Toggle>
           <Toggle
             checked={copyOnIFF}
             onToggle={this.toggleCopyOnIFF}
@@ -246,6 +254,10 @@ class Settings extends ComponentEx<IProps, IComponentState> {
     this.props.onSetCopyOnIFF(newValue);
   }
 
+  private toggleCollectionInstallConcurrency = (newValue: boolean) => {
+    this.props.onSetCollectionConcurrency(newValue);
+  }
+
   private isPathSensible(input: string): boolean {
     const sanitizeSep = new RegExp('/', 'g');
     const trimTrailingSep = new RegExp(`\\${path.sep}*$`, 'g');
@@ -704,6 +716,7 @@ function mapStateToProps(state: IState): IConnectedProps {
     instanceId: state.app.instanceId,
     copyOnIFF: state.settings.downloads.copyOnIFF,
     maxBandwidth: state.settings.downloads.maxBandwidth,
+    collectionsInstallWhileDownloading: state.settings.downloads.collectionsInstallWhileDownloading
   };
 }
 
@@ -719,6 +732,7 @@ function mapDispatchToProps(dispatch: ThunkDispatch<any, null, Redux.Action>): I
       showError(dispatch, message, details, { allowReport, isBBCode }),
     onSetCopyOnIFF: (enabled: boolean) => dispatch(setCopyOnIFF(enabled)),
     onSetMaxBandwidth: (bps: number) => dispatch(setMaxBandwidth(bps)),
+    onSetCollectionConcurrency: (enabled: boolean) => dispatch(setCollectionConcurrency(enabled)),
   };
 }
 

src/extensions/installer_fomod/index.ts

@@ -213,11 +213,10 @@ function processAttributes(input: any, modPath: string): Bluebird<any> {
       const xmlDoc = parser.parseFromString(data.slice(offset).toString(encoding), 'text/xml');
       const name: Element = xmlDoc.querySelector('fomod Name');
       return truthy(name)
-        ? {
-          customFileName: name.childNodes[0].nodeValue,
-        } : {};
+        ? Bluebird.resolve({ customFileName: name.childNodes[0].nodeValue })
+        : Bluebird.resolve({});
     })
-    .catch(() => ({}));
+    .catch(() => Bluebird.resolve({}));
 }
 
 function spawnAsync(command: string, args: string[]): Promise<void> {

src/extensions/mod_management/InstallManager.ts

@@ -384,10 +384,10 @@ class InstallManager {
       .then(() => withContext('Installing', baseName, () => ((forceGameId !== undefined)
         ? Bluebird.resolve(forceGameId)
         : queryGameId(api.store, downloadGameIds, modId))
-      .tap(gameId => {
+      .then(async gameId => {
         installGameId = gameId;
         if (installGameId === undefined) {
-          return Bluebird.reject(
+          return Promise.reject(
             new ProcessCanceled('You need to select a game before installing this mod'));
         }
         if (installGameId === 'site' && baseName.toLowerCase().includes('extension')) {
@@ -397,7 +397,7 @@ class InstallManager {
           //  do without API providing a unique tag for us to identify Vortex extensions. (AFAIK we can't even query the existing tags from the website)
           // Installation of non-Vortex tools with the extension basename will just install as a mod for
           //  the current game which I guess should be fine.
-          return Promise.resolve();
+          return Promise.resolve(installGameId);
         }
         const state = api.getState();
         const games = knownGames(state);
@@ -408,20 +408,26 @@ class InstallManager {
           const installProfileId = lastActiveProfileForGame(state, installGameId);
           installProfile = profileById(state, installProfileId);
         }
-        return api.emitAndAwait('will-install-mod', gameId, archiveId, modId, fullInfo);
+        // TODO make the download first functionality optional
+        await api.emitAndAwait('will-install-mod', gameId, archiveId, modId, fullInfo);
+        return Bluebird.resolve(gameId);
       })
       // calculate the md5 hash here so we can store it with the mod meta information later,
       // otherwise we'd not remember the hash when installing from external file
       .tap(() => genHash(archivePath).then(hash => {
         archiveMD5 = hash.md5sum;
         archiveSize = hash.numBytes;
-        _.merge(fullInfo, {
-          download: {
-            fileMD5: archiveMD5,
-            size: archiveSize,
-          },
-        });
-      }).catch(() => null))
+        try {
+          _.merge(fullInfo, {
+            download: {
+              fileMD5: archiveMD5,
+              size: archiveSize,
+            },
+          });
+        } catch (err) {
+          // no operation
+        }
+    }))
       .then(gameId => {
         if (installGameId === 'site') {
           // install an already-downloaded extension
@@ -1480,8 +1486,8 @@ class InstallManager {
       }
     }
 
-    log('debug', 'installer instructions',
-        JSON.stringify(result.instructions.map(instr => _.omit(instr, ['data']))));
+    // log('debug', 'installer instructions',
+    //     JSON.stringify(result.instructions.map(instr => _.omit(instr, ['data']))));
     this.reportUnsupported(api, instructionGroups.unsupported, archivePath);
 
     return this.processMKDir(instructionGroups.mkdir, destinationPath)
@@ -2633,7 +2639,7 @@ class InstallManager {
 
           return (dep.mod === undefined)
             ? queryWrongMD5()
-                .then(() => installDownload(dep, downloadId))
+                .then(() => api.getState().settings.downloads.collectionsInstallWhileDownloading ? installDownload(dep, downloadId) : Bluebird.resolve(null))
                 .catch(err => {
                   if (dep['reresolveDownloadHint'] === undefined) {
                     return Bluebird.reject(err);
@@ -2666,6 +2672,9 @@ class InstallManager {
 
     const phaseList = Object.values(phases);
 
+    const findDownloadId = (dep: IDependency) => {
+      return Object.keys(downloads).find(dlId => downloads[dlId].modInfo?.referenceTag === dep.reference.tag);
+    }
     const res: Bluebird<IDependency[]> = Bluebird.reduce(phaseList,
       (prev: IDependency[], depList: IDependency[], idx: number) => {
         if (depList.length === 0) {
@@ -2674,6 +2683,18 @@ class InstallManager {
         return this.doInstallDependenciesPhase(api, depList, gameId, sourceModId,
                                                recommended,
                                                doDownload, abort)
+          .then(async (updated: IDependency[]) => api.getState().settings.downloads.collectionsInstallWhileDownloading
+            ? Promise.resolve(updated)
+            : new Promise<IDependency[]>(async (resolve, reject) => {
+              const sorted = ([...updated]).sort((a, b) => (a.reference.fileSize ?? 0) - (b.reference.fileSize ?? 0));
+              try {
+                // Give the state a chance to catch up
+                await Promise.all(sorted.map(dep => installDownload(dep, findDownloadId(dep))));
+                return resolve(updated);
+              } catch (err) {
+                return reject(err);
+              }
+          }))
           .then((updated: IDependency[]) => {
             if (idx === phaseList.length - 1) {
               return Bluebird.resolve(updated);
@@ -3255,13 +3276,13 @@ class InstallManager {
       const fullPath: string =
         path.join(downloadPathForGame(state, downloadGame[0]), download.localPath);
       this.install(downloadId, fullPath, downloadGame,
-        api, { ...modInfo, download }, false, false, (error, id) => {
+        api, { ...modInfo, download }, false, silent, (error, id) => {
           if (error === null) {
-            resolve(id);
+            return resolve(id);
           } else {
-            reject(error);
+            return reject(error);
           }
-        }, forceGameId, fileList, true, undefined, silent);
+        }, forceGameId, fileList, silent, undefined, false);
     });
   }
 

src/extensions/mod_management/index.ts

@@ -153,8 +153,24 @@ function registerMerge(test: MergeTest, merge: MergeFunc, modType: string) {
   mergers.push({ test, merge, modType });
 }
 
+const shouldSuppressUpdate = (api: IExtensionApi) => {
+  const state = api.getState();
+  const suppressOnActivities = ['conflicts', 'installing_dependencies', 'purging'];
+  const isActivityRunning = (activity: string) =>
+    getSafe(state, ['session', 'base', 'activity', 'mods'], []).includes(activity) // purge/deploy
+    || getSafe(state, ['session', 'base', 'activity', activity], []).length > 0; // installing_dependencies
+  const suppressingActivities = suppressOnActivities.filter(activity => isActivityRunning(activity));
+  const suppressing = suppressingActivities.length > 0;
+  if (suppressing) {
+    log('info', 'skipping settings bake', { activities: suppressingActivities });
+  }
+  return suppressing;
+}
+
 function bakeSettings(api: IExtensionApi, profile: IProfile, sortedModList: IMod[]) {
-  return api.emitAndAwait('bake-settings', profile.gameId, sortedModList, profile);
+  return shouldSuppressUpdate(api)
+    ? Promise.resolve()
+    : api.emitAndAwait('bake-settings', profile.gameId, sortedModList, profile);
 }
 
 function showCycles(api: IExtensionApi, cycles: string[][], gameId: string) {

src/extensions/mod_management/util/filterModInfo.ts

@@ -1,23 +1,36 @@
 import { AttributeExtractor } from '../../../types/IExtensionContext';
 
-import Promise from 'bluebird';
-import * as _ from 'lodash';
+import { log } from '../../../util/log';
 
 const attributeExtractors: Array<{ priority: number, extractor: AttributeExtractor}> = [];
 
 export function registerAttributeExtractor(priority: number, extractor: AttributeExtractor) {
   attributeExtractors.push({ priority, extractor });
+  attributeExtractors.sort((lhs, rhs) => rhs.priority - lhs.priority);
 }
 
 function filterUndefined(input: { [key: string]: any }) {
-  return _.omitBy(input, val => val === undefined);
+  return Object.fromEntries(Object.entries(input).filter(([_, val]) => val !== undefined));
+}
+
+// Every mod installation is run through the attributeExtractors in order of priority.
+//  Imagine the simplest use case where installing a collection with 1000 mods - and one extractor takes over 1.5 seconds to run,
+//  that's at a minimum 25 minutes of waiting for the user. Keep in mind that incorrect usage of the attributeExtractors in community
+//  extensions will raise this time even further. This is why we have a timeout of 1 second for each extractor. All core extractors
+//  should never take more than a few milliseconds to run.
+function extractorOrSkip(extractor: AttributeExtractor, input: any, modPath: string): Promise<any> {
+  return Promise.race([
+    extractor(input, modPath),
+    new Promise((_, reject) => setTimeout(() => reject(new Error('Extractor timed out')), 1000))
+  ]).catch(err => {
+    log('error', `Extractor skipped: "${extractor.name ?? extractor.toString()}" - ${err.message}`);
+    return {};
+  });
 }
 
 function filterModInfo(input: any, modPath: string): Promise<any> {
-  return Promise.map(
-    attributeExtractors.sort((lhs, rhs) => rhs.priority - lhs.priority),
-    extractor => extractor.extractor(input, modPath),
-  ).then(infoBlobs => Object.assign({}, ...infoBlobs.map(filterUndefined)));
+  return Promise.all(attributeExtractors.map(extractor => extractorOrSkip(extractor.extractor, input, modPath)))
+    .then(infoBlobs => Object.assign({}, ...infoBlobs.map(filterUndefined)));
 }
 
 export default filterModInfo;

src/types/IState.ts

@@ -218,6 +218,7 @@ export interface ISettingsDownloads {
   showDropzone: boolean;
   showGraph: boolean;
   copyOnIFF: boolean;
+  collectionsInstallWhileDownloading: boolean;
 }
 
 export interface IStatePaths {