Support for analyzing multiple local modules (multi-module projects) …

…by using PYTHONPATH (#176)

Fixes:
- #163
- #177
- #178

com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java

@@ -48,7 +48,8 @@ protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M, List<Mod
     RewritingTranslatorToCAst x =
         new RewritingTranslatorToCAst(
             M,
-            new PythonModuleParser((SourceModule) M, typeDictionary, allModules) {
+            new PythonModuleParser(
+                (SourceModule) M, typeDictionary, allModules, this.getPythonPath()) {
               @Override
               public CAstEntity translateToCAst() throws Error, IOException {
                 CAstEntity ce = super.translateToCAst();

com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonFileParser.java

@@ -10,6 +10,7 @@
  *****************************************************************************/
 package com.ibm.wala.cast.python.parser;
 
+import com.ibm.wala.cast.python.util.Util;
 import com.ibm.wala.cast.tree.CAstEntity;
 import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl;
 import com.ibm.wala.cast.util.CAstPrinter;
@@ -18,13 +19,15 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.net.URL;
+import java.util.List;
 
 public class PythonFileParser extends PythonParser<File> {
 
   private final File fileName;
 
-  public PythonFileParser(File fileName, CAstTypeDictionaryImpl<String> types) {
-    super(types);
+  public PythonFileParser(
+      File fileName, CAstTypeDictionaryImpl<String> types, List<File> pythonPath) {
+    super(types, pythonPath);
     this.fileName = fileName;
   }
 
@@ -43,8 +46,10 @@ protected WalaPythonParser makeParser() throws IOException {
   }
 
   public static void main(String[] args) throws Exception {
+    List<File> pythonPath = Util.getPathFiles(args[1]);
+
     PythonParser<File> p =
-        new PythonFileParser(new File(args[0]), new CAstTypeDictionaryImpl<String>());
+        new PythonFileParser(new File(args[0]), new CAstTypeDictionaryImpl<String>(), pythonPath);
     CAstEntity script = p.translateToCAst();
     System.err.println(script);
     System.err.println(CAstPrinter.print(script));

com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java

@@ -10,13 +10,15 @@
  *****************************************************************************/
 package com.ibm.wala.cast.python.parser;
 
+import com.ibm.wala.cast.python.util.Util;
 import com.ibm.wala.cast.tree.CAstEntity;
 import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl;
 import com.ibm.wala.cast.util.CAstPrinter;
 import com.ibm.wala.classLoader.Module;
 import com.ibm.wala.classLoader.ModuleEntry;
 import com.ibm.wala.classLoader.SourceModule;
 import com.ibm.wala.classLoader.SourceURLModule;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
@@ -40,8 +42,11 @@ protected WalaPythonParser makeParser() throws IOException {
   }
 
   public PythonModuleParser(
-      SourceModule fileName, CAstTypeDictionaryImpl<String> types, List<Module> allModules) {
-    super(types);
+      SourceModule fileName,
+      CAstTypeDictionaryImpl<String> types,
+      List<Module> allModules,
+      List<File> pythonPath) {
+    super(types, pythonPath);
     this.fileName = fileName;
   }
 
@@ -52,11 +57,14 @@ protected String scriptName() {
 
   public static void main(String[] args) throws Exception {
     URL url = new URL(args[0]);
+    List<File> pythonPath = Util.getPathFiles(args[1]);
+
     PythonParser<ModuleEntry> p =
         new PythonModuleParser(
             new SourceURLModule(url),
             new CAstTypeDictionaryImpl<String>(),
-            Collections.singletonList(new SourceURLModule(url)));
+            Collections.singletonList(new SourceURLModule(url)),
+            pythonPath);
     CAstEntity script = p.translateToCAst();
     System.err.println(script);
     System.err.println(CAstPrinter.print(script));

com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java

@@ -39,9 +39,12 @@
 import com.ibm.wala.util.collections.HashMapFactory;
 import com.ibm.wala.util.collections.HashSetFactory;
 import com.ibm.wala.util.collections.ReverseIterator;
+import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
 import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -1821,8 +1824,18 @@ public CAstNode visitYield(Yield arg0) throws Exception {
 
   private final CAstTypeDictionaryImpl<String> types;
 
-  protected PythonParser(CAstTypeDictionaryImpl<String> types) {
+  /**
+   * The <a href="https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH">PYTHONPATH</a> to
+   * use in the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   * @see https://docs.python.org/3/tutorial/modules.html#the-module-search-path.
+   */
+  protected java.util.List<File> pythonPath;
+
+  protected PythonParser(CAstTypeDictionaryImpl<String> types, java.util.List<File> pythonPath) {
     this.types = types;
+    this.pythonPath = pythonPath;
   }
 
   @Override
@@ -1906,6 +1919,35 @@ public Position getPosition(int arg) {
             public Position getNamePosition() {
               return null;
             }
+
+            @Override
+            public String getSignature() {
+              File file = this.getFile();
+              java.util.List<File> pythonPath = getPythonPath();
+
+              // If the PYTHONPATH isn't specified.
+              if (pythonPath.isEmpty())
+                // Revert to just the name.
+                return this.getName();
+
+              for (File pathEntry : pythonPath) {
+                String pathEntryAbsolutePath = pathEntry.getAbsoluteFile().getPath();
+                // Remove protocol.
+                pathEntryAbsolutePath = pathEntryAbsolutePath.replaceFirst("file:.*!/", "");
+
+                String fileAbsolutePath = file.getAbsolutePath();
+
+                if (fileAbsolutePath.startsWith(pathEntryAbsolutePath)) {
+                  // Found it.
+                  Path filePath = Paths.get(fileAbsolutePath);
+                  Path pathEntryPath = Paths.get(pathEntryAbsolutePath);
+
+                  Path scriptRelativePath = pathEntryPath.relativize(filePath);
+                  return "script " + scriptRelativePath.toString();
+                }
+              }
+              return null; // Not found.
+            }
           };
 
       return script;
@@ -1924,4 +1966,16 @@ public String getMsg() {
   public void print(PyObject ast) {
     System.err.println(ast.getClass());
   }
+
+  /**
+   * Gets the <a
+   * href="https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH">PYTHONPATH</a> to use in
+   * the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   * @see https://docs.python.org/3/tutorial/modules.html#the-module-search-path.
+   */
+  public java.util.List<File> getPythonPath() {
+    return pythonPath;
+  }
 }

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/client/PytestAnalysisEngine.java

@@ -1,5 +1,7 @@
 package com.ibm.wala.cast.python.client;
 
+import static java.util.Collections.emptyList;
+
 import com.ibm.wala.cast.python.loader.PytestLoader;
 import com.ibm.wala.cast.python.loader.PytestLoaderFactory;
 import com.ibm.wala.classLoader.CallSiteReference;
@@ -18,6 +20,8 @@
 import com.ibm.wala.ssa.SSAAbstractInvokeInstruction;
 import com.ibm.wala.util.CancelException;
 import com.ibm.wala.util.intset.OrdinalSet;
+import java.io.File;
+import java.util.List;
 
 public class PytestAnalysisEngine<T> extends PythonAnalysisEngine<T> {
 
@@ -72,8 +76,11 @@ public IMethod getCalleeTarget(CGNode caller, CallSiteReference site, IClass rec
   }
 
   public PytestAnalysisEngine() {
-    super();
-    loader = new PytestLoaderFactory();
+    loader = new PytestLoaderFactory(emptyList());
+  }
+
+  public PytestAnalysisEngine(List<File> pythonPath) {
+    loader = new PytestLoaderFactory(pythonPath);
   }
 
   protected void addBypassLogic(IClassHierarchy cha, AnalysisOptions options) {

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoader.java

@@ -22,6 +22,8 @@
 import com.ibm.wala.types.TypeReference;
 import com.ibm.wala.util.collections.HashMapFactory;
 import com.ibm.wala.util.collections.Pair;
+import java.io.File;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -33,8 +35,8 @@ public class PytestLoader extends Python3Loader {
 
   private final Map<String, Map<String, Integer>> testParams = HashMapFactory.make();
 
-  public PytestLoader(IClassHierarchy cha) {
-    super(cha);
+  public PytestLoader(IClassHierarchy cha, List<File> pythonPath) {
+    super(cha, pythonPath);
   }
 
   @Override

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoaderFactory.java

@@ -2,11 +2,37 @@
 
 import com.ibm.wala.classLoader.IClassLoader;
 import com.ibm.wala.ipa.cha.IClassHierarchy;
+import java.io.File;
+import java.util.List;
 
 public class PytestLoaderFactory extends PythonLoaderFactory {
 
+  /**
+   * The <a
+   * href="https://docs.python.org/3/tutorial/modules.html#the-module-search-path">PYTHONPATH</a> to
+   * use in the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   */
+  protected List<File> pythonPath;
+
+  public PytestLoaderFactory(List<File> pythonPath) {
+    this.pythonPath = pythonPath;
+  }
+
   @Override
   protected IClassLoader makeTheLoader(IClassHierarchy cha) {
-    return new PytestLoader(cha);
+    return new PytestLoader(cha, this.getPythonPath());
+  }
+
+  /**
+   * Gets the <a
+   * href="https://docs.python.org/3/tutorial/modules.html#the-module-search-path">PYTHONPATH</a> to
+   * use in the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   */
+  public List<File> getPythonPath() {
+    return pythonPath;
   }
 }

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java

@@ -29,17 +29,19 @@
 import com.ibm.wala.classLoader.ModuleEntry;
 import com.ibm.wala.classLoader.SourceModule;
 import com.ibm.wala.ipa.cha.IClassHierarchy;
+import java.io.File;
 import java.io.IOException;
 import java.util.List;
 import org.python.core.PyObject;
 
 public class Python3Loader extends PythonLoader {
-  public Python3Loader(IClassHierarchy cha, IClassLoader parent) {
-    super(cha, parent);
+
+  public Python3Loader(IClassHierarchy cha, IClassLoader parent, List<File> pythonPath) {
+    super(cha, parent, pythonPath);
   }
 
-  public Python3Loader(IClassHierarchy cha) {
-    super(cha);
+  public Python3Loader(IClassHierarchy cha, List<File> pythonPath) {
+    super(cha, pythonPath);
   }
 
   @Override
@@ -48,7 +50,8 @@ protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M, List<Mod
     RewritingTranslatorToCAst x =
         new RewritingTranslatorToCAst(
             M,
-            new PythonModuleParser((SourceModule) M, typeDictionary, allModules) {
+            new PythonModuleParser(
+                (SourceModule) M, typeDictionary, allModules, this.getPythonPath()) {
               @Override
               public CAstEntity translateToCAst() throws Error, IOException {
                 CAstEntity ce = super.translateToCAst();

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3LoaderFactory.java

@@ -2,11 +2,38 @@
 
 import com.ibm.wala.classLoader.IClassLoader;
 import com.ibm.wala.ipa.cha.IClassHierarchy;
+import java.io.File;
+import java.util.List;
 
 public class Python3LoaderFactory extends PythonLoaderFactory {
 
+  /**
+   * The <a href="https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH">PYTHONPATH</a> to
+   * use in the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   * @see https://docs.python.org/3/tutorial/modules.html#the-module-search-path.
+   */
+  protected List<File> pythonPath;
+
+  public Python3LoaderFactory(List<File> pythonPath) {
+    this.pythonPath = pythonPath;
+  }
+
   @Override
   protected IClassLoader makeTheLoader(IClassHierarchy cha) {
-    return new Python3Loader(cha);
+    return new Python3Loader(cha, this.getPythonPath());
+  }
+
+  /**
+   * Gets the <a
+   * href="https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH">PYTHONPATH</a> to use in
+   * the analysis.
+   *
+   * @apiNote PYTHONPATH is currently only supported for Python 3.
+   * @see https://docs.python.org/3/tutorial/modules.html#the-module-search-path.
+   */
+  public List<File> getPythonPath() {
+    return pythonPath;
   }
 }

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonFileParser.java

@@ -10,6 +10,7 @@
  *****************************************************************************/
 package com.ibm.wala.cast.python.parser;
 
+import com.ibm.wala.cast.python.util.Util;
 import com.ibm.wala.cast.tree.CAstEntity;
 import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl;
 import com.ibm.wala.cast.util.CAstPrinter;
@@ -18,13 +19,15 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.net.URL;
+import java.util.List;
 
 public class PythonFileParser extends PythonParser<File> {
 
   private final File fileName;
 
-  public PythonFileParser(File fileName, CAstTypeDictionaryImpl<String> types) {
-    super(types);
+  public PythonFileParser(
+      File fileName, CAstTypeDictionaryImpl<String> types, List<File> pythonPath) {
+    super(types, pythonPath);
     this.fileName = fileName;
   }
 
@@ -43,8 +46,10 @@ protected WalaPythonParser makeParser() throws IOException {
   }
 
   public static void main(String[] args) throws Exception {
+    List<File> pythonPath = Util.getPathFiles(args[1]);
+
     PythonParser<File> p =
-        new PythonFileParser(new File(args[0]), new CAstTypeDictionaryImpl<String>());
+        new PythonFileParser(new File(args[0]), new CAstTypeDictionaryImpl<String>(), pythonPath);
     CAstEntity script = p.translateToCAst();
     System.err.println(script);
     System.err.println(CAstPrinter.print(script));