reuses changes from python3 branch plus ... (#29)

* code cleanup

* python3 and augmaker graphtool and code cleanup

* closes Issue#22

* updates log file name

* removes unused imports

* more formatting changes and additional comments

Extending.md

@@ -0,0 +1,96 @@
+Extending DLJC
+==================
+
+## Supporting a new build system
+
+Create a new Python file in `do_like_javac/capture/` named after the build system you
+want to support. Start with this template:
+
+```python
+from . import generic
+
+supported_commands = [...]
+
+class MyCapture(generic.GenericCapture):
+    def __init__(self, cmd, args):
+        super(MyCapture, self).__init__(cmd, args)
+        self.build_cmd = [...]
+
+    def get_javac_commands(self, verbose_output):
+        return []
+
+    get_target_jars(self, verbose_output):
+        return []
+
+def gen_instance(cmd, args):
+    return MyCapture(cmd, args)
+```
+
+Where supported\_commands is a list of commands that might be used to invoke the
+build system (e.g. "gradle", "gradlew", "ant", "maven").
+
+For the constructor, cmd is a sys.argv style list containing the build command
+the user supplied at the command line. For example, if they wrote
+
+    dljc -t print -- mybuildsystem -arg1 -arg2 -arg3
+
+then cmd would contain `['mybuildsystem', '-arg1', '-arg2', '-arg3']`. You
+should set self.build_cmd to be a version of that command that invokes the build
+system with flags that give you the verbose output you'll need to parse the
+javac commands from.
+
+`args` is the argparse structure that dljc stores its own arguments in. You can
+find more details in `do_like_javac/arg.py`.
+
+Your job now is to implement `get_javac_commands` and optionally
+`get_target_jars`. verbose\_output contains a list of lines of output from
+running the build system. If your build system outputs entire javac commands
+on a single line, or if you can reconstitute the commands in their entirety,
+then there's a convenience method called `javac_parse` defined in the
+superclass, which expects a list of words in the command and produces the
+output you need.
+
+If not, the ouput format is a list of javac command dicts, which are of the
+form:
+
+```python
+{
+    'java_files': ['Foo.java', 'Bar.java', ...],
+    'javac_switches': {
+        # Switches have the - prefix removed.
+        # Value is either the argument given to the switch,
+        # or True for no-argument flags.
+        'classpath': ...,
+        ...
+    }
+}
+```
+
+`get_target_jars` returns a list of paths to jar files output by the build.
+
+Finally, add your new module to the `capture_modules` list at the top of
+`do_like_javac/capture/__init__.py`.
+
+## Adding a new analysis tool
+
+Create a new Python file in `do_like_javac/tools/` named after the tool you
+want to add. Use this template:
+
+```python
+argparser = None
+
+def run(args, javac_commands, jars):
+    print("My tool results.")
+```
+
+`args` is the argparse structure created by DLJC. `argparser` is where you can
+plug in additional command line options you'd like to be available for your
+tool. You can see an example in `do_like_javac/tools/infer.py`.
+
+`javac_commands` and `jars` are the structures discussed above, consisting of
+the javac command used to build the project and the jar files generated (if
+that information is available).
+
+After augmenting `run()` to do what you want, add your tool to the `TOOLS`
+dictionary at the top of `do_like_javac/tools/__init__.py`, along with a
+`from . import mytoolname` statement.

README.md

@@ -24,18 +24,12 @@ project, selecting "Export" and choosing the "Ant Buildfiles" option.
 Dependencies
 ============
 
-* Python 2.7
-
-That's it. No other external dependencies for the core `do-like-javac` scripts.
-
-Of course, you will also need to have installed:
-
+* Python 3
 * The analysis tool(s) you want to run.
 * Any build dependencies of the project you're analyzing.
 
-`do-like-javac` was built and tested on Mac OS X and GNU/Linux. It probably also
-works on Microsoft Windows, but the method of invocation is probably different and
-we provide no support.
+`do-like-javac` was built and tested on Mac OS X and GNU/Linux. It may also
+work on Microsoft Windows, but we provide no support.
 
 Installation
 ============
@@ -69,15 +63,21 @@ If you're running checking tools, there are a couple more flags that may be
 helpful. `--quiet` suppresses output from tools, and `--timeout <seconds>`
 kills any tool subcommand that runs longer than `<seconds>`.
 
+Extending
+===========
+
+Instructions for adding support for new tools or build systems to DLJC can be
+found in [Extending.md](./Extending.md).
+
 Caching
 =======
 
 `do-like-javac` can only extract data from a full compile of a project. That means
 if you want to re-run it with new arguments or different analysis tools, you will
-have to clean and fully re-compile your project. To save time and shortcut this
+have to clean and fully re-compile the project. To save time and shortcut this
 process, we save a cache of the results in the output directory. If you want `dljc`
 to use this cache, simply add the `--cache` flag and the cache (if available) will
-be used instead of recompiling your project.
+be used instead of recompiling the project.
 
 **IMPORTANT NOTE**: We don't do any sort of cache invalidation or freshness checking.
 If you add new files to your project and want `dljc` to pick up on them, you will have
@@ -101,7 +101,7 @@ The Bixie tool will run your project through [Bixie](http://sri-csl.github.io/bi
 Dyntrace
 ---------
 
-The Dyntrace tool will run your project through Randoop to generate tests, then run those tests with Daikon/Chicory to generate invariants. You will need to provide a library directory with the following jars using the `--lib` option:
+The Dyntrace tool will run your project through Randoop to generate tests, then run those tests with Daikon/Chicory to generate likely invariants. You will need to provide a library directory with the following jars using the `--lib` option:
 
 * randoop.jar
 * junit-4.12.jar

dljc

@@ -1,5 +1,5 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 
 import do_like_javac
 
-do_like_javac.run()
+do_like_javac.run()

do_like_javac/__init__.py

@@ -1,3 +1,3 @@
-import command
+from . import command
 
 run = command.main

do_like_javac/__main__.py

@@ -1,4 +1,4 @@
-from command import main
+from .command import main
 
 if __name__ == '__main__':
     main()

do_like_javac/arg.py

@@ -10,8 +10,7 @@
 import os
 import sys
 
-import tools
-import capture
+from . import capture, tools
 
 DEFAULT_OUTPUT_DIRECTORY = os.path.join(os.getcwd(), 'dljc-out')
 
@@ -30,8 +29,10 @@ def __call__(self, parser, namespace, values, option_string=None):
                         default=DEFAULT_OUTPUT_DIRECTORY, dest='output_directory',
                         action=AbsolutePathAction,
                         help='The directory to log results.')
+
 base_group.add_argument('--log_to_stderr', action='store_true',
                         help='''Redirect log messages to stderr instead of log file''')
+
 base_group.add_argument('-t', '--tool', metavar='<tool>',
                         action='store',default=None,
                         help='A comma separated list of tools to run. Valid tools: ' + ', '.join(tools.TOOLS))
@@ -51,13 +52,19 @@ def __call__(self, parser, namespace, values, option_string=None):
                         help='''Use the dljc cache (if available)''')
 
 base_group.add_argument('-c', '--checker', metavar='<checker>',
-                        action='store',default='NullnessChecker',
+                        action='store', 
+                        # do not run the NullnessChecker by default
+                        # default='NullnessChecker',
                         help='A checker to check (for checker/inference tools)')
 
 base_group.add_argument('-l', '--lib', metavar='<lib_dir>',
                         action='store',dest='lib_dir',
                         help='Library directory with JARs for tools that need them.')
 
+base_group.add_argument('--jdkVersion', metavar='<jdkVersion>',
+                        action='store',
+                        help='Version of the JDK to use with the Checker Framework.')
+
 def split_args_to_parse():
     split_index = len(sys.argv)
     if CMD_MARKER in sys.argv:

do_like_javac/cache.py

@@ -1,6 +1,7 @@
 import os
 import pickle
 
+
 def retrieve(cmd, args, capturer):
   cache_file = os.path.join(args.output_directory, "dljc.cache")
 

do_like_javac/capture/__init__.py

@@ -1,5 +1,4 @@
-import ant, gradle, javac, mvn
-import itertools
+from . import ant, gradle, javac, mvn
 
 capture_modules = [ant, gradle, javac, mvn]
 

do_like_javac/capture/ant.py

@@ -6,7 +6,7 @@
 # additional grant of patent rights can be found in the PATENTS_Facebook file
 # in the same directory.
 
-import generic
+from . import generic
 
 supported_commands = ['ant']
 
@@ -66,4 +66,4 @@ def get_javac_commands(self, verbose_output):
                         javac_arguments.append(arg)
         if javac_arguments != []:
             javac_commands.append(javac_arguments)
-        return map(self.javac_parse, javac_commands)
+        return list(map(self.javac_parse, javac_commands))

do_like_javac/capture/generic.py

@@ -1,16 +1,26 @@
 import os
-import zipfile
 import timeit
+import zipfile
+
 import do_like_javac.tools.common as cmdtools
 
+
 def is_switch(s):
     return s != None and s.startswith('-')
 
+## brought this from github.com/kelloggm/do-like-javac
+def is_switch_first_part(s):
+    return s != None and s.startswith('-') and ("=" not in s)
+
 def get_entry_point(jar):
     class_pattern = "Main-Class:"
 
     with zipfile.ZipFile(jar, 'r') as zip:
-        metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF"))
+        metadata = []
+        try:
+            metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF").decode("utf-8"))
+        except TypeError as e:
+            print(f"ERROR: unable to read META-INF/MANIFEST.MF. See: {e}")
         for line in metadata:
             if class_pattern in line:
                 content = line[len(class_pattern):].strip()
@@ -59,10 +69,12 @@ def capture(self):
         stats = {}
 
         start_time = timeit.default_timer()
-        result = cmdtools.run_cmd(self.build_cmd)
-        stats['build_time'] = result['time']
+        result = cmdtools.run_cmd(self.build_cmd, self.args)
+        # stats['build_time'] = result['time']
+        stats['build_time'] = timeit.default_timer() - start_time
 
-        with open(os.path.join(self.args.output_directory, 'build_output.txt'), 'w') as f:
+        build_out_file = os.path.join(self.args.output_directory, 'build_output.txt')
+        with open(build_out_file, 'w') as f:
             f.write(result['output'])
 
         if result['return_code'] != 0:
@@ -72,7 +84,7 @@ def capture(self):
 
         javac_commands = self.get_javac_commands(build_lines)
         target_jars = self.get_target_jars(build_lines)
-        jars_with_entry_points = map(get_entry_point, target_jars)
+        jars_with_entry_points = list(map(get_entry_point, target_jars))
 
         self.record_stats(stats, javac_commands, jars_with_entry_points)
 
@@ -94,7 +106,7 @@ def javac_parse(self, javac_command):
                 files.append(a)
                 possible_switch_arg = False
 
-            if is_switch(prev_arg):
+            if is_switch_first_part(prev_arg):
                 if possible_switch_arg:
                     switches[prev_arg[1:]] = a
                 else:

do_like_javac/capture/gradle.py

@@ -6,9 +6,10 @@
 # additional grant of patent rights can be found in the PATENTS_Facebook file
 # in the same directory.
 
-import generic
 import os
 
+from . import generic
+
 supported_commands = ['gradle', 'gradlew']
 
 def gen_instance(cmd, args):
@@ -32,4 +33,4 @@ def get_javac_commands(self, verbose_output):
                 content = line.partition(argument_start_pattern)[2].strip()
                 results.append(content.split(' '))
 
-        return map(self.javac_parse, results)
+        return list(map(self.javac_parse, results))

do_like_javac/capture/javac.py

@@ -6,7 +6,7 @@
 # additional grant of patent rights can be found in the PATENTS_Facebook file
 # in the same directory.
 
-import generic
+from . import generic
 
 supported_commands = ['javac']
 
@@ -20,4 +20,4 @@ def __init__(self, cmd, args):
         self.cmd = cmd[1:]
 
     def get_javac_commands(self, verbose_output):
-        return map(self.javac_parse, [self.cmd])
+        return list(map(self.javac_parse, [self.cmd]))

do_like_javac/capture/mvn.py

@@ -7,7 +7,8 @@
 # in the same directory.
 
 import re
-import generic
+
+from . import generic
 
 supported_commands = ['mvn']
 
@@ -55,4 +56,4 @@ def get_javac_commands(self, verbose_output):
                 if found:
                     files_to_compile.append(found.group(1))
 
-        return map(self.javac_parse, javac_commands)
+        return list(map(self.javac_parse, javac_commands))

do_like_javac/command.py

@@ -1,9 +1,10 @@
+import json
+import os
 import pprint
-import arg
-import log
-import tools
-import cache
-import os,json,sys
+import sys
+
+from . import arg, cache, log, tools
+
 
 def output_json(filename, obj):
     with open(filename, 'w') as f:
@@ -20,10 +21,12 @@ def main():
     result = cache.retrieve(cmd, args, capturer)
 
     if not result:
-        print "DLJC: Build command failed."
+        print("DLJC: Build command failed.")
         sys.exit(1)
 
     javac_commands, jars, stats = result
+    if not javac_commands or len(javac_commands) == 0:
+        raise ValueError(f"no javac commands found by capturer:\n\tcmd = {cmd}\n\targs = {args}")
 
     log.info('Results: %s', pprint.pformat(javac_commands))
     output_json(os.path.join(args.output_directory, 'javac.json'), javac_commands)

do_like_javac/log.py

@@ -6,11 +6,10 @@
 # additional grant of patent rights can be found in the PATENTS_Facebook file
 # in the same directory.
 
+import logging
 import os
-import sys
 import platform
-import shutil
-import logging
+import sys
 
 FORMAT = '[%(levelname)s] %(message)s'
 LOG_FILE = 'toplevel.log'