src_code_metrics.py

import re, csv, pytz, json, subprocess
from dateutil import parser
import pandas as pd
import get_bugs
from libmozdata import patchanalysis

# Execute a shell command
def shellCommand(command_str):
    cmd =subprocess.Popen(command_str.split(' '), stdout=subprocess.PIPE)
    cmd_out, cmd_err = cmd.communicate()
    return cmd_out

def loadReleaseDate():
    print 'Loading Relase date ...'
    rel_date_list = list()
    rel_list = list()
    with open('complexity_sna/data/release2commit.csv') as f:
        csvreader = csv.reader(f)
        for row in csvreader:
            rel_num = row[0]
            rel_date = re.sub(r'[^0-9]', '', row[2])
            rel_date_list.append([rel_date, rel_num])
            rel_list.append(rel_num)
    return rel_date_list, list(reversed(rel_list))

def loadCommitDate():
    print 'Loading commit date ...'
    commit_date_dict = dict()
    with open('commit_date.csv') as f:
        csvreader = csv.reader(f, delimiter='\t')
        for row in csvreader:
            commit_id = row[0]
            raw_time = row[1]
            datetime_obj = parser.parse(raw_time)
            time_str = datetime_obj.astimezone(pytz.utc).strftime('%Y%m%d')
            commit_date_dict[commit_id] = time_str
    return commit_date_dict

def correspondingRelease(commit_id, commit_date_dict, rel_date_list):
    if commit_id in commit_date_dict:
        commit_date = commit_date_dict[commit_id]
    else:
        for key in commit_date_dict:
            if commit_id in key:
                commit_date = commit_date_dict[key]
    for item in rel_date_list:
        if commit_date >= item[0]:
            return item[1]
    return rel_date_list[-1][1]

def removePrefix(path):
    return re.sub(r'^[\/\.]+', '', path)

def loadMetrics4Releases(category, release_list):
    rel_metric_dict = dict()
    metric_names = None
    for rel in release_list:
        metric_dict = dict()
        metric_file = 'complexity_sna/code_metrics/%s-%s.csv' %(category, rel.replace('.', '_'))
        with open(metric_file, 'r') as f:
            csvreader = csv.reader(f)
            metric_names = next(csvreader, None)[1:]
            for line in csvreader:
                key = removePrefix(line[0])
                metric_dict[key] = line[1:]
            rel_metric_dict[rel] = metric_dict
    return rel_metric_dict, metric_names

def extractSourceCodeMetrics(rel_date_list, rel_list, commit_date_dict, category):
    # load metrics
    rel_metric_dict, metric_names = loadMetrics4Releases(category, rel_list)
    # map and compute metric values
    result_list = list()
    i = 0

    bugs = get_bugs.get_all()

    for bug in bugs:
        if DEBUG and i > 5:
            break

        bug_id = bug['id']
        commits, _ = patchanalysis.get_commits_for_bug(bug)

        print bug_id

        # extract metrics
        raw_list = list()
        metric_list = list()
        for commit_id in commits:
            i += 1
            if DEBUG:
                print ' ', commit_id
            # corresponding (prior) release of a commit
            rel_num = correspondingRelease(commit_id, commit_date_dict, rel_date_list)
            # changed files in a commit
            shell_res = shellCommand('hg -R %s log -r %s --template {files}\t{diffstat}' %(HG_REPO_PATH, commit_id)).split('\t')
            raw_changed_files = shell_res[0]
            cpp_changed_files = re.findall(r'(\S+\.(?:c|cpp|cc|cxx|h|hpp|hxx)\b)', raw_changed_files)
            # map file/node to metrics
            for a_file in cpp_changed_files:
                metric_dict = rel_metric_dict[rel_num]
                for node in metric_dict:
                    if node in a_file:
                        metrics = metric_dict[node]
                        raw_list.append(metrics)
        # compute average/sum value for a specific attachment
        if len(raw_list):
            df = pd.DataFrame(raw_list, columns=metric_names).apply(pd.to_numeric)
            for metric_name in metric_names:
                metric_list.append(round(df[metric_name].mean(), 2))
            result_list.append([bug_id] + metric_list)
        else:
            result_list.append([bug_id] + [0]*len(metric_names))
    
    return pd.DataFrame(result_list, columns=['bug_id']+metric_names)

if __name__ == '__main__':
    DEBUG = False
    HG_REPO_PATH = '../firefox/'
    # load data
    rel_date_list, rel_list = loadReleaseDate()
    commit_date_dict = loadCommitDate()
    # extract metrics
    df_complexity = extractSourceCodeMetrics(rel_date_list, rel_list, commit_date_dict, 'complexity')
    df_sna = extractSourceCodeMetrics(rel_date_list, rel_list, commit_date_dict, 'sna')
    df_code = pd.merge(df_complexity, df_sna, on='bug_id')
    df_code.to_csv('independent_metrics/src_code_metrics.csv', index=False)
    if DEBUG:
        print df_code