RQ2_assess_function_calls.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/artist_44_660\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/title1_44_272\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/disc_number_44_1304\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/title2_44_433\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/year_44_982\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/genre_44_821\n",
      "Warning: multiple text input files found in ./data/eval_output/com.poupa.vinylmusicplayer/track_number_44_1143\n",
      "Warning: multiple text input files found in ./data/eval_output/My Files/search_src_text_137_106\n",
      "Warning: multiple text input files found in ./data/eval_output/OK_Social_Network_v22.6.1/text_login_66_448\n",
      "Warning: multiple text input files found in ./data/eval_output/OK_Social_Network_v22.6.1/password_text_66_607\n",
      "Warning: multiple text input files found in ./data/eval_output/org.totschnig.myexpenses/Payee_327_641\n",
      "Warning: multiple text input files found in ./data/eval_output/org.totschnig.myexpenses/Comment_327_1055\n",
      "Warning: multiple text input files found in ./data/eval_output/org.totschnig.myexpenses/AmountEditText_459_361\n",
      "Warning: multiple text input files found in ./data/eval_output/Instagram/full_name_77_391\n",
      "Warning: multiple text input files found in ./data/eval_output/Instagram/password_77_567\n",
      "Warning: multiple text input files found in ./data/eval_output/WhatsApp/registration_cc_182_480\n",
      "Warning: multiple text input files found in ./data/eval_output/WhatsApp/registration_phone_400_480\n"
     ]
    }
   ],
   "source": [
    "# High accurately are function calls invoked?\n",
    "\n",
    "import os \n",
    "import shutil\n",
    "import re\n",
    "import glob \n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "output_dir = './data/eval_output'\n",
    "\n",
    "rows = []\n",
    "\n",
    "for app_output_path in glob.glob(os.path.join(output_dir, '*')):\n",
    "    if not os.path.isdir(app_output_path):\n",
    "        continue\n",
    "\n",
    "    for tf_result_path in glob.glob(os.path.join(app_output_path, '*')):\n",
    "        gen_text_inputs = []\n",
    "        text_input_files = glob.glob(os.path.join(tf_result_path, 'text_inputs', 'text_input_*.json'))\n",
    "        if len(text_input_files) > 1:\n",
    "            print(f'Warning: multiple text input files found in {tf_result_path}')\n",
    "\n",
    "        with open(text_input_files[0], 'r') as f:\n",
    "            data = json.load(f)\n",
    "            assert len(data) == 1\n",
    "            for tf_id, tf_info in data.items():\n",
    "                gen_text_inputs.append(tf_info['default_tester'])\n",
    "\n",
    "        called_functions = []\n",
    "        prompt_files = glob.glob(os.path.join(tf_result_path, 'prompts', 'prompt_*.json'))\n",
    "        for prompt_file in prompt_files:\n",
    "            # only one prompt sample\n",
    "            with open(prompt_file, 'r') as f:\n",
    "                prompt = json.load(f)\n",
    "            \n",
    "            for user_message, assistant_message in prompt['prompt']['conversation']:\n",
    "                if isinstance(assistant_message, dict):\n",
    "                    called_functions.append(assistant_message['name'])\n",
    "\n",
    "            break\n",
    "\n",
    "        rows.append({\n",
    "            'app_name': os.path.basename(app_output_path),\n",
    "            'textfield_id': os.path.basename(tf_result_path),\n",
    "            'called_functions': called_functions,\n",
    "            'gen_text_inputs': gen_text_inputs\n",
    "        })\n",
    "\n",
    "    \n",
    "df = pd.DataFrame(rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "eval_df = pd.read_csv('./data/RQ1_DroidAgent-textinput-labelling-merged-cleaned.csv', header=0)[:-1]\n",
    "\n",
    "textfield_map = defaultdict(list)\n",
    "\n",
    "for _, row in eval_df.iterrows():\n",
    "    textfield_map[(row['app_name'], row['textfield_id'])].append(int(row['index']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['data_id'] = df.apply(lambda row: textfield_map[(row['app_name'], row['textfield_id'])], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('./data/RQ3_function_call_accuracy.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_id</th>\n",
       "      <th>app_name</th>\n",
       "      <th>textfield_id</th>\n",
       "      <th>expected</th>\n",
       "      <th>actual</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93</td>\n",
       "      <td>Settings</td>\n",
       "      <td>search_src_text_137_106</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>org.koitharu.kotatsu</td>\n",
       "      <td>searchView_198_119</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>95</td>\n",
       "      <td>Galaxy Store</td>\n",
       "      <td>search_src_text_153_118</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>94</td>\n",
       "      <td>Galaxy Store</td>\n",
       "      <td>coupon_edit_89_1912</td>\n",
       "      <td>[0, 0, 1]</td>\n",
       "      <td>[0, 0, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>96</td>\n",
       "      <td>Galaxy Store</td>\n",
       "      <td>name_86_725</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>127</td>\n",
       "      <td>Clock</td>\n",
       "      <td>worldclock_search_map_txt_find_158_140</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>124</td>\n",
       "      <td>Clock</td>\n",
       "      <td>numberpicker_input_144_408</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>128</td>\n",
       "      <td>Clock</td>\n",
       "      <td>hour_48_1787</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>130</td>\n",
       "      <td>Clock</td>\n",
       "      <td>second_354_1787</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>129</td>\n",
       "      <td>Clock</td>\n",
       "      <td>minute_201_1787</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     data_id              app_name                            textfield_id  \\\n",
       "0         93              Settings                 search_src_text_137_106   \n",
       "1          1  org.koitharu.kotatsu                      searchView_198_119   \n",
       "2         95          Galaxy Store                 search_src_text_153_118   \n",
       "3         94          Galaxy Store                     coupon_edit_89_1912   \n",
       "4         96          Galaxy Store                             name_86_725   \n",
       "..       ...                   ...                                     ...   \n",
       "115      127                 Clock  worldclock_search_map_txt_find_158_140   \n",
       "116      124                 Clock              numberpicker_input_144_408   \n",
       "117      128                 Clock                            hour_48_1787   \n",
       "118      130                 Clock                         second_354_1787   \n",
       "119      129                 Clock                         minute_201_1787   \n",
       "\n",
       "      expected     actual  \n",
       "0    [0, 0, 0]  [0, 1, 0]  \n",
       "1    [0, 0, 0]  [0, 0, 0]  \n",
       "2    [0, 0, 0]  [0, 0, 0]  \n",
       "3    [0, 0, 1]  [0, 0, 1]  \n",
       "4    [0, 0, 0]  [1, 0, 0]  \n",
       "..         ...        ...  \n",
       "115  [0, 0, 0]  [1, 0, 0]  \n",
       "116  [0, 0, 0]  [0, 1, 0]  \n",
       "117  [0, 0, 0]  [0, 1, 0]  \n",
       "118  [0, 0, 0]  [0, 0, 0]  \n",
       "119  [0, 0, 0]  [0, 0, 0]  \n",
       "\n",
       "[120 rows x 5 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_labelled = pd.read_csv('./data/RQ3_function_call_accuracy_labelled_5prompts.csv', header=0)\n",
    "\n",
    "result_rows = []\n",
    "\n",
    "function2index = {\n",
    "    'get_friend_profile': 0,\n",
    "    'get_samsung_product_info': 1,\n",
    "    'get_galaxy_store_coupon_code': 2,\n",
    "}\n",
    "\n",
    "for _, row in df_labelled.iterrows():\n",
    "    expected = [0, 0, 0]\n",
    "    actual = [0, 0, 0]\n",
    "\n",
    "    if row['required_function_call'] in function2index:\n",
    "        expected[function2index[row['required_function_call']]] = 1\n",
    "    \n",
    "    called_functions = json.loads(row['called_functions'].replace('\\'', '\"'))\n",
    "    data_ids = json.loads(row['data_id'].replace('\\'', '\"'))\n",
    "\n",
    "    for called_function in called_functions:\n",
    "        assert called_function in function2index\n",
    "        actual[function2index[called_function]] = 1\n",
    "\n",
    "    for i in data_ids:\n",
    "        result_rows.append({\n",
    "            'data_id': i,\n",
    "            'app_name': row['app_name'],\n",
    "            'textfield_id': row['textfield_id'],\n",
    "            'expected': expected,\n",
    "            'actual': actual\n",
    "        })\n",
    "        \n",
    "\n",
    "result_df = pd.DataFrame(result_rows)\n",
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                              precision    recall  f1-score   support\n",
      "\n",
      "          get_friend_profile       0.31      0.65      0.42        20\n",
      "    get_samsung_product_info       0.14      1.00      0.25         3\n",
      "get_galaxy_store_coupon_code       1.00      1.00      1.00         1\n",
      "\n",
      "                   micro avg       0.27      0.71      0.39        24\n",
      "                   macro avg       0.48      0.88      0.56        24\n",
      "                weighted avg       0.32      0.71      0.42        24\n",
      "                 samples avg       0.14      0.14      0.14        24\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(result_df['expected'].tolist(), result_df['actual'].tolist(), target_names=['get_friend_profile', 'get_samsung_product_info', 'get_galaxy_store_coupon_code'], zero_division=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sr-stem",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}