When the dependencies of a file change the cache is invalidated

package.json

@@ -24,10 +24,12 @@
   },
   "homepage": "https://github.com/pinterest/esprint#readme",
   "dependencies": {
+    "dependency-tree": "^8.0.0",
     "fb-watchman": "^2.0.1",
     "glob": "^7.1.6",
     "jayson": "^3.3.4",
     "jest-worker": "^26.6.2",
+    "micromatch": "^4.0.2",
     "sane": "^4.1.0",
     "yargs": "^16.2.0"
   },
@@ -48,6 +50,7 @@
     "babel-jest": "^26.6.3",
     "chalk": "^4.1.0",
     "eslint": "^7.15.0",
+    "eslint-plugin-import": "^2.22.1",
     "jest": "^26.6.3"
   }
 }

src/Server.js

@@ -1,5 +1,7 @@
+import dependencyTree from 'dependency-tree';
 import glob from 'glob';
 import jayson from 'jayson';
+import micromatch from 'micromatch';
 import path from 'path';
 import sane from 'sane';
 import LintRunner from './LintRunner';
@@ -9,16 +11,30 @@ const ROOT_DIR = process.cwd();
 
 const eslint = new CLIEngine({ cwd: ROOT_DIR });
 
+const GLOBAL_DEPENDENCIES = Object.freeze([
+  'package-lock.json',
+  'yarn.lock',
+  '.eslintrc',
+  '.eslintrc.yml',
+  '.eslintrc.yaml',
+  '.eslintrc.js',
+  '.eslintrc.cjs',
+  '.eslintrc.json',
+  'package.json'
+]);
+
+const delay = (amount) => new Promise(resolve => setTimeout(resolve, amount));
+
+const MAX_ATTEMPTS = 8;
+
 export default class Server {
   constructor(options) {
     const {
       workers,
       port,
-      paths,
-      ignored,
       rcPath,
       quiet,
-      fix
+      fix,
     } = options;
 
     this.port = port;
@@ -30,12 +46,12 @@ export default class Server {
 
     const rootDir = path.dirname(this.rcPath);
 
-    this._setupWatcher(rootDir, paths.split(','), ignored.split(','));
+    this._setupWatcher(rootDir, options);
 
     const that = this;
 
     const server = jayson.server({
-      status: function(args, cb) {
+      status: function(_, cb) {
         if (that.filesToProcess === 0) {
           cb(null, that.getResultsFromCache());
         } else {
@@ -44,60 +60,114 @@ export default class Server {
       }
     });
 
-    process.send({server: server});
+    if(process.send) process.send({server: server});
 
     server.http().listen(this.port);
   }
 
-  _setupWatcher(root, paths, ignored) {
-    const watcher = sane(root, {
-      glob: paths,
-      ignored: ignored,
+  _setupWatcher(root, options) {
+    const {
+      paths,
+      ignored,
+      requireConfig,
+      webpackConfig,
+      tsConfig,
+      watchman,
+    } = options;
+    const ignoredArr = ignored.split(',');
+    const pathsArr = paths.split(',');
+
+    const dependentsOf = {};
+    const dependenciesOf = {};
+
+    const removeDependencies = (filename) => {
+      (dependenciesOf[filename] || []).forEach((path) => {
+        const updatedDependents = (dependentsOf[path] || []).filter((path2) => path2 !== filename);
+        if (updatedDependents.length > 0) {
+          dependentsOf[path] = updatedDependents;
+        } else {
+          delete dependentsOf[path];
+        }
+      });
+      delete dependenciesOf[filename];
+    };
+
+    const updateDependencies = (filename) => {
+      removeDependencies(filename);
+      dependenciesOf[filename] = dependencyTree.toList({
+        filename,
+        directory: root,
+        requireConfig,
+        webpackConfig,
+        tsConfig,
+        filter: path => !micromatch.some(path, ignoredArr, { dot: true }) && !eslint.isPathIgnored(path) && path.indexOf('eslint') === -1,
+      }).concat(GLOBAL_DEPENDENCIES).map((abs) => path.relative(root, abs));
+      dependenciesOf[filename].forEach((file) => {
+        dependentsOf[file] = [...(dependentsOf[file] || []), filename];
+      });
+    };
+
+    const updateDependenciesWatcher = () => {
+      if (dependenciesWatcher) dependenciesWatcher.close();
+      dependenciesWatcher = sane(root, {
+        glob: Object.keys(dependentsOf),
+        ignored: ignoredArr,
+        dot: true,
+        watchman: process.env.NODE_ENV !== 'test' && watchman,
+      });
+      const onChange = (filepath) => {
+        if (dependentsOf[filepath] && dependentsOf[filepath].length > 0) {
+          dependentsOf[filepath].forEach((path) => {
+            updateDependencies(path);
+            this.lintFile(path);
+          });
+          updateDependenciesWatcher();
+        }
+      };
+      dependenciesWatcher.on('change', onChange);
+      dependenciesWatcher.on('delete', (filepath) => {
+        onChange(filepath);
+        delete dependentsOf[filepath];
+        updateDependenciesWatcher();
+      });
+    };
+
+    const globWatcher = sane(root, {
+      glob: pathsArr,
+      ignored: ignoredArr,
       dot: true,
-      watchman: process.env.NODE_ENV !== 'test',
+      watchman: process.env.NODE_ENV !== 'test' && watchman,
     });
 
-    watcher.on('ready', () => {
-      process.send({message: 'Reading files to be linted...[this may take a little bit]'});
-      let filePaths = [];
-      for (let i = 0; i < paths.length; i++) {
-        const files = glob.sync(paths[i], {
+    let dependenciesWatcher = null;
+
+    globWatcher.on('ready', () => {
+      if(process.send) process.send({message: 'Reading files to be linted...[this may take a little bit]'});
+      const allFiles = new Set();
+      for (const path of pathsArr) {
+        const files = glob.sync(path, {
           cwd: root,
           absolute: true,
-          ignore: ignored
-        });
-        files.forEach((file) => {
-          filePaths.push(file);
+          ignore: ignoredArr,
         });
+        files.forEach((file) => allFiles.add(file));
       }
-
+      const filePaths = [...allFiles].filter((file) => file && !eslint.isPathIgnored(file) && file.indexOf('eslint') === -1);
+      filePaths.forEach(updateDependencies);
+      updateDependenciesWatcher();
       this.lintAllFiles(filePaths);
     });
-
-    watcher.on('change', (filepath) => {
-      let filePaths = [];
-      if (filepath.indexOf('.eslintrc') !== -1) {
-        this.cache = {};
-        for (let i = 0; i < paths.length; i++) {
-          const files = glob.sync(paths[i], {
-            cwd: root,
-            absolute: true,
-            ignore: ignored
-          });
-          files.forEach((file) => {
-            filePaths.push(file);
-          });
-        }
-        this.lintAllFiles(filePaths);
-      } else {
-        this.lintFile(filepath);
+    globWatcher.on('add', (file) => {
+      if (!eslint.isPathIgnored(file) && file.indexOf('eslint') === -1) {
+        updateDependencies(file);
+        this.lintFile(file);
+        updateDependenciesWatcher();
       }
     });
-    watcher.on('add', (filepath) => {
-      this.lintFile(filepath);
-    });
-    watcher.on('delete', (filepath) => {
+    globWatcher.on('delete', (filepath) => {
       delete this.cache[filepath];
+      removeDependencies(filepath);
+      updateDependenciesWatcher();
     });
   }
 
@@ -119,27 +189,34 @@ export default class Server {
     };
   }
 
-  lintFile(file) {
+  async lintFile(file, attempt = 1) {
     if (eslint.isPathIgnored(file) || file.indexOf('eslint') !== -1) {
       return;
     }
     this.filesToProcess++;
-    const that = this;
-    this.lintRunner.run([file])
-      .then(function(results) {
-        const record = results.records[0];
-        if (record) {
-          delete record.source;
-          that.cache[record.filePath] = record;
-        }
-        that.filesToProcess--;
-      })
-      .catch(e => console.error(e));
+    delete this.cache[file];
+    try {
+      const { records: [record] } = await this.lintRunner.run([file]);
+      if (record) {
+        delete record.source;
+        this.cache[record.filePath] = record;
+      }
+      this.filesToProcess--;
+    } catch (e) {
+      console.error(`Failed to run on '${file} on attempt ${attempt} out of ${MAX_ATTEMPTS}`, e);
+      if (attempt < MAX_ATTEMPTS) {
+        await delay(250 * 2 ** attempt);
+        this.fileToProcess--;
+        // Return here so that they aren't nested and can't run out the stack space.
+        return this.lintFile(file, attempt + 1);
+      }
+      this.filesToProcess--;
+    }
   }
 
-  lintAllFiles(files) {
-    files.map((file) => {
+  async lintAllFiles(files) {
+    return Promise.all(files.map((file) => {
       this.lintFile(file);
-    });
+    }));
   }
 }

src/commands/connect.js

@@ -6,9 +6,8 @@ import Client from '../Client';
 
 export const connect = (options) => {
   const args = [];
-  // eslint-disable-next-line
-  for (const key in options) {
-    args.push(`--${key}=${options[key]}`);
+  for (const [key, value] of Object.entries(options)) {
+    args.push(`--${key}=${value}`);
   }
 
   const port = options.port;
@@ -20,7 +19,8 @@ export const connect = (options) => {
     if (!isTaken) {
       const child = fork(
         require.resolve('../startServer.js'), args, {
-          silent: true
+          silent: true,
+          stdio: process.env.NODE_ENV !== 'test' ? 'inherit' : 'pipe',
         }
       );
 

tests/dependencies/.eslintignore

@@ -0,0 +1 @@
+__tests__/*

tests/dependencies/.eslintrc.json

@@ -0,0 +1,17 @@
+{
+  "plugins": [
+    "import"
+  ],
+  "rules": {
+    "no-var": 2,
+    "import/named": 2
+  },
+  "parserOptions": {
+    "ecmaVersion": 6,
+    "sourceType": "module"
+  },
+  "env": {
+    "es6": true
+  },
+  "root": true
+}

tests/dependencies/.esprintrc

@@ -0,0 +1,11 @@
+
+{
+  "port": 5004 ,
+  "paths": [
+    "fixture.js"
+  ],
+
+  "ignored": [
+    "**/node_modules/**/*"
+  ]
+}

tests/dependencies/__tests__/dependencies.test.js

@@ -0,0 +1,45 @@
+const fs = require('fs');
+const path = require('path');
+const runEsprint = require('../../runEsprint.js');
+const killProcess = require('../../killProcess.js');
+
+const FIXTURE_DIR = path.join(__dirname, '..');
+const FIXTURE_1_PATH = path.join(FIXTURE_DIR, 'fixture.js')
+const FIXTURE_2_PATH = path.join(FIXTURE_DIR, 'fixture2.js')
+
+function writeStartingFileContents() {
+  const file1Contents = `import { foo } from './fixture2';\nfoo(5);`;
+  const file2Contents = `import getPid from '../getPid';\n\nconst bar = (x) => console.log(x);`;
+
+  if (!fs.existsSync(FIXTURE_1_PATH)) {
+    fs.writeFileSync(FIXTURE_1_PATH, file1Contents);
+  }
+  fs.writeFileSync(FIXTURE_2_PATH, file2Contents);
+}
+
+function writeFile2ContentsWithExport() {
+  const file2Contents = `export const foo = (x) => console.log(x);`;
+  fs.writeFileSync(FIXTURE_2_PATH, file2Contents);
+}
+
+beforeEach(() => {
+  killProcess();
+  writeStartingFileContents();
+});
+
+afterEach(() => {
+  killProcess();
+  writeStartingFileContents();
+});
+
+test('Properly updates when a dependency updates', () => {
+  const results = runEsprint(FIXTURE_DIR);
+  const expectedError = expect.stringContaining(`error  foo not found in './fixture2'  import/named`);
+  expect(results.error).toBeDefined();
+  expect(results.error.stdout.toString()).toEqual(expectedError);
+
+  writeFile2ContentsWithExport();
+  const newResults = runEsprint(FIXTURE_DIR);
+  const newExpectedResults = expect.stringContaining('');
+  expect(newResults.error).toBeUndefined();
+});

tests/dependencies/fixture.js

@@ -0,0 +1,2 @@
+import { foo } from './fixture2';
+foo(5);

tests/dependencies/fixture2.js

@@ -0,0 +1,3 @@
+import getPid from '../getPid';
+
+const bar = (x) => console.log(x);