diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..409e9bf --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +Running +======= + +To run, add bin/ to your PATH and invoke from the directory of the project you want to analyze: + + dljc -o logs -- ant build + +Where "ant build" is replaced by whatever command builds your project. Output will be emitted to logs/toplevel.log + +You may also run a checking tool on the discovered java files, by invoking with the -t option and a tool to use (e.g. "-t soot", "-t inference" or "-t checker"). + +LICENSE +======= + +Parts of the code in this directory were taken from the Facebook Infer project. Its license is available at + + https://github.com/facebook/infer/blob/master/LICENSE diff --git a/bin/arg.py b/bin/arg.py new file mode 100644 index 0000000..7743963 --- /dev/null +++ b/bin/arg.py @@ -0,0 +1,141 @@ +import argparse +import os +import sys +import imp + +DEFAULT_OUTPUT_DIRECTORY = os.path.join(os.getcwd(), 'dljc-out') + +# token that identifies the end of the options for do-like-javac and the beginning +# of the compilation command +CMD_MARKER = '--' + +# insert here the correspondence between module name and the list of +# compiler/build-systems it handles. +# All supported commands should be listed here +MODULE_TO_COMMAND = { + 'javac': ['javac'], + 'ant': ['ant'], + 'gradle': ['gradle', 'gradlew'], + 'mvn': ['mvn'] +} + +CAPTURE_PACKAGE = 'capture' +LIB_FOLDER = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.path.pardir, 'lib') + +class AbsolutePathAction(argparse.Action): + """Convert a path from relative to absolute in the arg parser""" + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, os.path.abspath(values)) + +base_parser = argparse.ArgumentParser(add_help=False) +base_group = base_parser.add_argument_group('global arguments') +base_group.add_argument('-o', '--out', metavar='', + default=DEFAULT_OUTPUT_DIRECTORY, dest='output_directory', + action=AbsolutePathAction, + help='Set the results directory') +base_group.add_argument('-t', '--tool', metavar='', + action='store',default=None, + help='choose a tool to run. Valid tools include soot, checker, and inference.') +base_group.add_argument('-c', '--checker', metavar='', + action='store',default='NullnessChecker', + help='choose a checker to check') +base_group.add_argument('-s', '--solver', metavar='', + action='store',default='checkers.inference.solver.DebugSolver', + help='solver to use on constraints') +base_group.add_argument('-afud', '--afuOutputDir', metavar='', + action='store',default='afud/', + help='Annotation File Utilities output directory') +base_group.add_argument('-m', '--mode', metavar='', + action='store',default='INFER', + help='Modes of operation: TYPECHECK, INFER, ROUNDTRIP,ROUNDTRIP_TYPECHECK') +base_group.add_argument('-i', '--incremental', action='store_true', + help='''Do not delete the results directory across + runs''') +base_group.add_argument('--log_to_stderr', action='store_true', + help='''When set, all logging will go to stderr instead + of log file''') +base_group.add_argument('-j', '--jar', metavar='', + action='store',default=None, + help='Set the path to either prog2dfg.jar or apilearner.jar.') + +def get_commands(): + """Return all commands that are supported.""" + #flatten and dedup the list of commands + return set(sum(MODULE_TO_COMMAND.values(), [])) + +def get_module_name(command): + """ Return module that is able to handle the command. None if + there is no such module.""" + for module, commands in MODULE_TO_COMMAND.iteritems(): + if command in commands: + return module + return None + +def split_args_to_parse(): + dd_index = \ + sys.argv.index(CMD_MARKER) if CMD_MARKER in sys.argv else len(sys.argv) + + args, cmd = sys.argv[1:dd_index], sys.argv[dd_index + 1:] + capture_module_name = os.path.basename(cmd[0]) if len(cmd) > 0 else None + mod_name = get_module_name(capture_module_name) + return args, cmd, mod_name + +def create_argparser(parents=[]): + parser = argparse.ArgumentParser( + parents=[base_parser] + parents, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + group = parser.add_argument_group( + 'supported compiler/build-system commands') + + supported_commands = ', '.join(get_commands()) + group.add_argument( + CMD_MARKER, + metavar='', + dest='nullarg', + default=None, + help=('Command to run the compiler/build-system. ' + 'Supported build commands (run `do-like-javac.py --help -- ` for ' + 'extra help, e.g. `do-like-javac.py --help -- ant`): ' + supported_commands), + ) + return parser + +def load_module(mod_name): + # load the 'capture' package in lib + pkg_info = imp.find_module(CAPTURE_PACKAGE, [LIB_FOLDER]) + imported_pkg = imp.load_module(CAPTURE_PACKAGE, *pkg_info) + # load the requested module (e.g. make) + mod_file, mod_path, mod_descr = \ + imp.find_module(mod_name, imported_pkg.__path__) + try: + return imp.load_module( + '{pkg}.{mod}'.format(pkg=imported_pkg.__name__, mod=mod_name), + mod_file, mod_path, mod_descr) + finally: + if mod_file: + mod_file.close() + +def parse_args(): + to_parse, cmd, mod_name = split_args_to_parse() + # get the module name (if any), then load it + imported_module = None + if mod_name: + imported_module = load_module(mod_name) + + # get the module's argparser and merge it with the global argparser + module_argparser = [] + if imported_module: + module_argparser.append( + imported_module.create_argparser(mod_name) + ) + global_argparser = create_argparser(module_argparser) + + args = global_argparser.parse_args(to_parse) + + if imported_module: + return args, cmd, imported_module + else: + global_argparser.print_help() + sys.exit(os.EX_OK) diff --git a/bin/check.py b/bin/check.py new file mode 100644 index 0000000..7f0aa35 --- /dev/null +++ b/bin/check.py @@ -0,0 +1,26 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback + + +def run_checker(javac_commands,args): + # checker-framework javac. + javacheck = os.environ['JSR308']+"/checker-framework/checker/bin/javac" + checker_command = [] + checker_command.extend([javacheck]) + + for jc in javac_commands: + pprint.pformat(jc) + javac_switches = jc['javac_switches'] + cp = javac_switches['classpath'] + java_files = ' '.join(jc['java_files']) + cmd = checker_command + ["-processor", args.checker, "-classpath", cp, java_files] + print ("Running %s" % cmd) + try: + print (subprocess.check_output(cmd, stderr=subprocess.STDOUT)) + except subprocess.CalledProcessError as e: + print e.output diff --git a/bin/dljc b/bin/dljc new file mode 100755 index 0000000..8351e43 --- /dev/null +++ b/bin/dljc @@ -0,0 +1,64 @@ +#!/usr/bin/env python2.7 + +import logging +import os +import sys +import platform +import pprint +import arg +import log +import soot +import infer +import check +import jprint +import randoop +import graphtools + +def soot_tool(results,jars,args): + soot.run_soot(results) + +def checker_tool(results,jars,args): + check.run_checker(results,args) + +def inference_tool(results,jars,args): + infer.run_inference(results,args) + +def print_tool(results,jars,args): + jprint.run_printer(results, jars) + +def randoop_tool(results,jars,args): + randoop.run_randoop(results) + +def graph_tool(results,jars,args): + graphtools.run(results,args) + + +def log_header(): + logging.info('Running command %s', ' '.join(sys.argv)) + logging.info('Platform: %s', platform.platform()) + logging.info('PATH=%s', os.getenv('PATH')) + logging.info('SHELL=%s', os.getenv('SHELL')) + logging.info('PWD=%s', os.getenv('PWD')) + +def main(): + args, cmd, imported_module = arg.parse_args() + log.configure_logging(args.output_directory, args.incremental, args.log_to_stderr) + + log_header() + + javac_commands, jars = imported_module.gen_instance(cmd).capture() + logging.info('Results: %s', pprint.pformat(javac_commands)) + + options = {'soot' : soot_tool, + 'checker' : checker_tool, + 'inference' : inference_tool, + 'print' : print_tool, + 'randoop' : randoop_tool, + 'graphtool' : graph_tool, + } + + if args.tool: + options[args.tool](javac_commands,jars,args) + +if __name__ == '__main__': + main() diff --git a/bin/graphtools.py b/bin/graphtools.py new file mode 100644 index 0000000..2bdf043 --- /dev/null +++ b/bin/graphtools.py @@ -0,0 +1,42 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback + + +def run(javac_commands, args): + run_tool(args.jar, javac_commands, args.output_directory) + + +def run_tool(jarfile, javac_commands, outdir): + # first add the call to the soot jar. + tool_command = [] + tool_command.extend(["java", "-jar", jarfile]) + + pp = pprint.PrettyPrinter(indent=2) + for jc in javac_commands: + pp.pformat(jc) + #jc['java_files'] + javac_switches = jc['javac_switches'] + class_dir = os.path.abspath(javac_switches['d']) + + java_files = jc['java_files'] + java_files_file = os.path.join(os.getcwd(), '__java_file_names.txt') + with open(java_files_file, 'w') as f: + for s in java_files: + f.write(s) + f.write("\n") + + current_outdir = os.path.join(outdir, class_dir.replace(os.getcwd(),'').replace(os.sep,"_")) + + cmd = tool_command + ["-o", current_outdir, "-j", class_dir, "-source", java_files_file ] + print ("Running %s" % ' '.join(cmd)) + try: + print (subprocess.check_output(cmd, stderr=subprocess.STDOUT)) + except: + print ('calling {cmd} failed\n{trace}'.format(cmd=' '.join(cmd),trace=traceback.format_exc())) + + diff --git a/bin/infer.py b/bin/infer.py new file mode 100644 index 0000000..b06d00f --- /dev/null +++ b/bin/infer.py @@ -0,0 +1,32 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback + + +def run_inference(javac_commands,args): + + # the dist directory if CFI. + CFI_dist = os.environ['JSR308']+"/checker-framework-inference/dist" + CFI_command = [] + + CFI_command.extend(["java"]) + + for jc in javac_commands: + pprint.pformat(jc) + javac_switches = jc['javac_switches'] + target_cp = javac_switches['classpath'] + java_files = ' '.join(jc['java_files']) + cp = target_cp +":"+ CFI_dist + "/checker.jar:" + CFI_dist + "/plume.jar:" + \ + CFI_dist + "/checker-framework-inference.jar" + cmd = CFI_command + ["-classpath", cp, "checkers.inference.InferenceLauncher" , + "--checker" ,args.checker, "--solver", args.solver , + "--mode" , args.mode ,"--targetclasspath", target_cp, "-afud", args.afuOutputDir, java_files] + print ("Running %s" % cmd) + try: + print (subprocess.check_output(cmd, stderr=subprocess.STDOUT)) + except: + print ('calling {cmd} failed\n{trace}'.format(cmd=' '.join(cmd),trace=traceback.format_exc())) diff --git a/bin/jprint.py b/bin/jprint.py new file mode 100644 index 0000000..acd6342 --- /dev/null +++ b/bin/jprint.py @@ -0,0 +1,15 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback + +def run_printer(javac_commands, jars): + pp = pprint.PrettyPrinter(indent=2) + for jc in javac_commands: + pp.pprint(jc) + javac_switches = jc['javac_switches'] + print("Target JARs (experimental):") + pp.pprint(jars) diff --git a/bin/log.py b/bin/log.py new file mode 100644 index 0000000..76afd55 --- /dev/null +++ b/bin/log.py @@ -0,0 +1,40 @@ +# Copyright (c) 2013 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import os +import shutil +import logging + +FORMAT = '[%(levelname)s] %(message)s' +LOG_FILE = 'toplevel.log' + +def remove_output_directory(output_directory): + # it is safe to ignore errors here because recreating the + # output_directory will fail later + shutil.rmtree(output_directory, True) + +def create_results_dir(results_dir): + try: + os.mkdir(results_dir) + except OSError: + pass + +def configure_logging(output_directory, incremental, log_to_stderr): + #if not incremental: + # remove_output_directory(output_directory) + + create_results_dir(output_directory) + + if log_to_stderr: + logging.basicConfig(level=logging.INFO, format=FORMAT) + else: + logging.basicConfig(level=logging.INFO, + format=FORMAT, + filename=os.path.join(output_directory, LOG_FILE), + filemode='w') + +# vim: set sw=4 ts=4 et: diff --git a/bin/randoop.py b/bin/randoop.py new file mode 100644 index 0000000..4a4f279 --- /dev/null +++ b/bin/randoop.py @@ -0,0 +1,133 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback +import getpass +import shutil +from glob import glob +import urllib + +def run_randoop(javac_commands): + pp = pprint.PrettyPrinter(indent=2) + i = 0 + for jc in javac_commands: + javac_switches = jc['javac_switches'] + cp = javac_switches['classpath'] + class_file_dir = javac_switches['d'] + class_files = [y for x in os.walk(class_file_dir) for y in glob(os.path.join(x[0], '*.class'))] + + if len(class_files)==0: + continue + + out_dir_name = "__randoop_%04d" % (i) + if not os.path.exists(out_dir_name): + os.makedirs(out_dir_name) + + class_files_file_name = os.path.join(out_dir_name, 'class_files.txt') + print ("Creating list of files %d in %s." % (len(class_files), os.path.abspath(out_dir_name)) ) + with open(class_files_file_name, mode='w') as myfile: + for class_file_name in class_files: + myfile.write(get_qualified_class_name_from_file(class_file_name, class_file_dir)) + myfile.write(os.linesep) + + (randoop_jar, junit_jar, hamcrest_jar) = find_or_download_jars() + + cp_entries = cp.split(os.pathsep) + clean_cp = list() + clean_cp.append(randoop_jar) + clean_cp.append(class_file_dir) + + lib_dir_name = "__randoop_libs" + if not os.path.exists(lib_dir_name): + os.makedirs(lib_dir_name) + + for cp_entry in cp_entries: + if cp_entry.endswith(".jar"): + #check if the jar is in a sub directory of the project. + #if not copy it into a new subdirectory and add the + #corresponding classpath entry. + if not os.path.realpath(cp_entry).startswith(os.getcwd()): + new_jar_name = os.path.join(lib_dir_name, os.path.basename(cp_entry)) + if not os.path.isfile(new_jar_name): + shutil.copyfile(cp_entry, new_jar_name) + clean_cp.append(new_jar_name) + else: + clean_cp.append(cp_entry) + pass + else: + #todo what happens here? + clean_cp.append(cp_entry) + + randoop_cmd = ['java', '-ea', '-classpath', os.pathsep.join(clean_cp), + "randoop.main.Main", "gentests", "--classlist=%s"%class_files_file_name, + "--timelimit=60", "--silently-ignore-bad-class-names=true", + "--junit-output-dir=%s"%out_dir_name] + + junit_cp = list(clean_cp) + junit_cp.append(junit_jar) + junit_build_cmd = ['javac', '-classpath', os.pathsep.join(junit_cp), os.path.join(out_dir_name, 'RandoopTest*.java'), '-d', out_dir_name] + + junit_run_cp = list(junit_cp) + junit_run_cp.append(hamcrest_jar) + junit_run_cp.append(out_dir_name) + + junit_run_cmd = ['java', '-classpath', os.pathsep.join(junit_run_cp), "org.junit.runner.JUnitCore", 'RandoopTest'] + + bash_script_name = "run_randoop_%04d.sh" % (i) + with open(bash_script_name, mode='w') as myfile: + myfile.write("#!/bin/bash\n") + myfile.write("echo \"Run Randoop\"\n") + myfile.write(" ".join(randoop_cmd)) + myfile.write("\n") + myfile.write("echo \"Build tests\"\n") + myfile.write(" ".join(junit_build_cmd)) + myfile.write("\n") + myfile.write("echo \"Run tests\"\n") + myfile.write(" ".join(junit_run_cmd)) + myfile.write("\n") + print ("Written script to %s" % bash_script_name) + + i += 1 + +def get_qualified_class_name_from_file(class_file_name, class_file_path): + """ terrible hack for now """ + suffix = class_file_name.replace(class_file_path+os.sep, "") + mid_section = suffix.replace(".class", "") + return mid_section.replace(os.sep, ".") + + +def find_or_download_jars(): + ''' + Finds or downloads the randoop, junit, and hamrest jars. + ''' + randoop_jar_dir = os.path.join(os.getcwd(), '__randoop_files') + if not os.path.isdir(randoop_jar_dir): + os.makedirs(randoop_jar_dir) + + randoop_jar = os.path.join(randoop_jar_dir, "randoop-2.0.jar") + if not os.path.isfile(randoop_jar): + print("Downloading randoop to %s" % randoop_jar ) + urllib.urlretrieve ("https://github.com/randoop/randoop/releases/download/v2.0/randoop-2.0.jar", randoop_jar) + + junit_jar = os.path.join(randoop_jar_dir, "junit-4.12.jar") + if not os.path.isfile(junit_jar): + print("Downloading junit to %s" % junit_jar ) + urllib.urlretrieve ("https://github.com/junit-team/junit/releases/download/r4.12/junit-4.12.jar", junit_jar) + + hamcrest_jar = os.path.join(randoop_jar_dir, "hamcrest-core-1.3.jar") + if not os.path.isfile(hamcrest_jar): + print("Downloading hamcrest to %s" % hamcrest_jar ) + urllib.urlretrieve ("http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", hamcrest_jar) + + return (randoop_jar, junit_jar, hamcrest_jar) + + + + + + + + diff --git a/bin/soot.py b/bin/soot.py new file mode 100644 index 0000000..f22a540 --- /dev/null +++ b/bin/soot.py @@ -0,0 +1,35 @@ +import logging +import os +import sys +import platform +import pprint +import subprocess +import traceback + + +def run_soot(javac_commands): + + soot_jar = os.path.dirname(os.path.realpath(__file__))+"/../soot-trunk.jar" + soot_command = [] + # first add the call to the soot jar. + soot_command.extend(["java", "-jar", soot_jar]) + + # now add the generic soot args that we want to use. + # TODO: these should actually be parsed from command line. + soot_command.extend(["-pp", "-src-prec", "c"]) + + for jc in javac_commands: + pprint.pformat(jc) + #jc['java_files'] + javac_switches = jc['javac_switches'] + cp = javac_switches['classpath'] + class_dir = javac_switches['d'] + + cmd = soot_command + ["-cp", cp, "-process-dir", class_dir] + print ("Running %s" % cmd) + try: + print (subprocess.check_output(cmd, stderr=subprocess.STDOUT)) + except: + print ('calling {cmd} failed\n{trace}'.format(cmd=' '.join(cmd),trace=traceback.format_exc())) + + diff --git a/lib/capture/__init__.py b/lib/capture/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/capture/ant.py b/lib/capture/ant.py new file mode 100644 index 0000000..74bd95a --- /dev/null +++ b/lib/capture/ant.py @@ -0,0 +1,78 @@ +# Copyright (c) 2015 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import util +import generic + +MODULE_NAME = __name__ +MODULE_DESCRIPTION = '''Run analysis of code built with a command like: +ant [options] [target] + +Analysis examples: +capture_javac.py -- ant compile''' + +def gen_instance(cmd): + return AntCapture(cmd) + +# This creates an empty argparser for the module, which provides only +# description/usage information and no arguments. +create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) + + +class AntCapture(generic.GenericCapture): + + def __init__(self, cmd): + self.build_cmd = ['ant', '-verbose'] + cmd[1:] + + def is_interesting(self, content): + return self.is_quoted(content) or content.endswith('.java') + + def is_quoted(self, argument): + quote = '\'' + return len(argument) > 2 and argument[0] == quote\ + and argument[-1] == quote + + def remove_quotes(self, argument): + if self.is_quoted(argument): + return argument[1:-1] + else: + return argument + + def get_target_jars(self, verbose_output): + jar_pattern = '[jar] Building jar: ' + jars = [] + + for line in verbose_output: + if jar_pattern in line: + pos = line.index(jar_pattern) + len(jar_pattern) + jar = line[pos:].strip() + jars.append(jar) + + return jars + + def get_javac_commands(self, verbose_output): + javac_pattern = '[javac]' + argument_start_pattern = 'Compilation arguments' + javac_arguments = [] + javac_commands = [] + collect = False + for line in verbose_output: + if javac_pattern in line: + if argument_start_pattern in line: + collect = True + if javac_arguments != []: + javac_commands.append(javac_arguments) + javac_arguments = [] + if collect: + pos = line.index(javac_pattern) + len(javac_pattern) + content = line[pos:].strip() + if self.is_interesting(content): + arg = self.remove_quotes(content) + javac_arguments.append(arg) + if javac_arguments != []: + javac_commands.append(javac_arguments) + return map(self.javac_parse, javac_commands) diff --git a/lib/capture/generic.py b/lib/capture/generic.py new file mode 100644 index 0000000..95c350e --- /dev/null +++ b/lib/capture/generic.py @@ -0,0 +1,63 @@ +import util +import zipfile + +def is_switch(s): + return s != None and s.startswith('-') + +def get_entry_point(jar): + class_pattern = "Main-Class:" + + with zipfile.ZipFile(jar, 'r') as zip: + metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF")) + for line in metadata: + if class_pattern in line: + content = line[len(class_pattern):].strip() + return {"jar": jar, "main": content} + + return {"jar": jar} + +class GenericCapture: + def __init__(self, cmd): + self.build_cmd = cmd + + def get_javac_commands(self, verbose_output): + return [] + + def get_target_jars(self, verbose_output): + return [] + + def capture(self): + build_output = util.get_build_output(self.build_cmd) + javac_commands = self.get_javac_commands(build_output) + target_jars = self.get_target_jars(build_output) + jars_with_entry_points = map(get_entry_point, target_jars) + return [javac_commands, jars_with_entry_points] + + def javac_parse(self, javac_command): + files = [] + switches = {} + + prev_arg = None + + for a in javac_command: + possible_switch_arg = True + + if is_switch(a): + possible_switch_arg = False + + if a.endswith('.java'): + files.append(a) + possible_switch_arg = False + + if is_switch(prev_arg): + if possible_switch_arg: + switches[prev_arg[1:]] = a + else: + switches[prev_arg[1:]] = True + + if is_switch(a): + prev_arg = a + else: + prev_arg = None + + return dict(java_files=files, javac_switches=switches) diff --git a/lib/capture/gradle.py b/lib/capture/gradle.py new file mode 100644 index 0000000..a9d4340 --- /dev/null +++ b/lib/capture/gradle.py @@ -0,0 +1,41 @@ +# Copyright (c) 2015 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import util +import generic + +MODULE_NAME = __name__ +MODULE_DESCRIPTION = '''Run analysis of code built with a command like: +gradle [options] [task] + +Analysis examples: +capture_javac.py -- gradle build +capture_javac.py -- ./gradlew build''' + +def gen_instance(cmd): + return GradleCapture(cmd) + +# This creates an empty argparser for the module, which provides only +# description/usage information and no arguments. +create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) + + +class GradleCapture(generic.GenericCapture): + + def __init__(self, cmd): + self.build_cmd = [cmd[0], '--debug'] + cmd[1:] + + def get_javac_commands(self, verbose_output): + argument_start_pattern = ' Compiler arguments: ' + results = [] + + for line in verbose_output: + if argument_start_pattern in line: + content = line.partition(argument_start_pattern)[2].strip() + results.append(content.split(' ')) + + return map(self.javac_parse, results) diff --git a/lib/capture/javac.py b/lib/capture/javac.py new file mode 100644 index 0000000..7a2595c --- /dev/null +++ b/lib/capture/javac.py @@ -0,0 +1,33 @@ +# Copyright (c) 2015 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import util +import generic + +MODULE_NAME = __name__ +MODULE_DESCRIPTION = '''Run analysis of code built with a command like: +javac [options] + +Analysis examples: +do-like-javac.py -- javac srcfile.java''' + +def gen_instance(cmd): + return JavaCapture(cmd) + +# This creates an empty argparser for the module, which provides only +# description/usage information and no arguments. +create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) + + +class JavaCapture(generic.GenericCapture): + + def __init__(self, cmd): + self.build_cmd = cmd + self.cmd = cmd[1:] + + def get_javac_commands(self, verbose_output): + return map(self.javac_parse, [self.cmd]) diff --git a/lib/capture/mvn.py b/lib/capture/mvn.py new file mode 100644 index 0000000..ac59fc9 --- /dev/null +++ b/lib/capture/mvn.py @@ -0,0 +1,67 @@ +# Copyright (c) 2015 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import re +import util +import generic + +MODULE_NAME = __name__ +MODULE_DESCRIPTION = '''Run analysis of code built with a command like: +mvn [options] [task] + +Analysis examples: +capture_javac.py -- mvn build''' + +def gen_instance(cmd): + return MavenCapture(cmd) + +# This creates an empty argparser for the module, which provides only +# description/usage information and no arguments. +create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) + + +class MavenCapture(generic.GenericCapture): + def __init__(self, cmd): + self.build_cmd = ['mvn', '-X'] + cmd[1:] + + def get_target_jars(self, verbose_output): + jar_pattern = '[INFO] Building jar: ' + jars = [] + + for line in verbose_output: + if jar_pattern in line: + pos = line.index(jar_pattern) + len(jar_pattern) + jar = line[pos:].strip() + jars.append(jar) + + return jars + + def get_javac_commands(self, verbose_output): + file_pattern = r'\[DEBUG\] Stale source detected: ([^ ]*\.java)' + options_pattern = '[DEBUG] Command line options:' + + javac_commands = [] + files_to_compile = [] + options_next = False + + for line in verbose_output: + if options_next: + # line has format [Debug] + javac_args = line.split(' ')[1:] + files_to_compile + javac_commands.append(javac_args) + options_next = False + files_to_compile = [] + elif options_pattern in line: + # Next line will have javac options to run + options_next = True + + else: + found = re.match(file_pattern, line) + if found: + files_to_compile.append(found.group(1)) + + return map(self.javac_parse, javac_commands) diff --git a/lib/capture/util.py b/lib/capture/util.py new file mode 100644 index 0000000..45497a7 --- /dev/null +++ b/lib/capture/util.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 - present Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import argparse +import os +import logging +import subprocess +import traceback + +def get_build_output(build_cmd): + # TODO make it return generator to be able to handle large builds + proc = subprocess.Popen(build_cmd, stdout=subprocess.PIPE, stderr=open(os.devnull, 'w')) + (verbose_out_chars, _) = proc.communicate() + return verbose_out_chars.split('\n') + +def run_cmd_ignore_fail(cmd): + try: + return subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except: + return 'calling {cmd} failed\n{trace}'.format( + cmd=' '.join(cmd), + trace=traceback.format_exc()) + +def log_java_version(): + java_version = run_cmd_ignore_fail(['java', '-version']) + javac_version = run_cmd_ignore_fail(['javac', '-version']) + logging.info("java versions:\n%s%s", java_version, javac_version) + + +def base_argparser(description, module_name): + def _func(group_name=module_name): + """This creates an empty argparser for the module, which provides only + description/usage information and no arguments.""" + parser = argparse.ArgumentParser(add_help=False) + group = parser.add_argument_group( + "{grp} module".format(grp=group_name), + description=description, + ) + return parser + return _func