test_hyperparameter_simple.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# My library\n",
    "from molgraph.dataset import *\n",
    "from molgraph.simplemodel import *\n",
    "from molgraph.hyperparameter_simple import *\n",
    "from molgraph.testing_simple import *\n",
    "from molgraph.visualize import *\n",
    "from molgraph.experiment import *\n",
    "# General library\n",
    "import os\n",
    "import argparse\n",
    "import numpy as np\n",
    "# pytorch\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "# optuna\n",
    "import optuna\n",
    "from optuna.trial import TrialState\n",
    "from optuna.visualization import plot_param_importances\n",
    "\n",
    "# Ensure that all operations are deterministic on GPU (if used) for reproducibility\n",
    "torch.backends.cudnn.determinstic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Argument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = ArgumentParser()\n",
    "args = parser.getArgument('''--file bbbp\n",
    "                             --model Descriptor\n",
    "                             --schema A\n",
    "                             --reduced \n",
    "                             --mol_embedding 256\n",
    "                             --fold 5\n",
    "                             --seed 42'''.split())\n",
    "\n",
    "args"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = args.file\n",
    "smiles = args.smiles \n",
    "task = args.task\n",
    "splitting = args.splitting \n",
    "splitting_fold = args.fold\n",
    "splitting_seed = args.splitting_seed\n",
    "\n",
    "# get validated dataset\n",
    "datasets = getDataset(file, smiles, task, splitting)\n",
    "# compute positive weight for classification\n",
    "if args.graphtask == 'classification':\n",
    "    args.pos_weight = getPosWeight(datasets)\n",
    "    print('pos_weight:', args.pos_weight)\n",
    "# generate dataset splitting\n",
    "datasets_splitted = generateDatasetSplitting(file, splitting, splitting_fold, splitting_seed)\n",
    "# generate all graph dataset\n",
    "datasets_graph = generateGraphDataset(file)\n",
    "# generate all reduced graph dataset\n",
    "dict_reducedgraph = dict()\n",
    "for g in args.reduced:\n",
    "    if g == 'substructure':\n",
    "        for i in range(splitting_fold):\n",
    "            vocab_file = file+'_'+str(i)\n",
    "            if not os.path.exists('vocab/'+vocab_file+'.txt'):\n",
    "                generateVocabTrain(file, splitting_seed, splitting_fold, vocab_len=args.vocab_len)\n",
    "            dict_reducedgraph[g] = generateReducedGraphDict(file, g, vocab_file=vocab_file)\n",
    "    else:\n",
    "        dict_reducedgraph[g] = generateReducedGraphDict(file, g)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyper = Hyper(args)\n",
    "\n",
    "# storage_string = \"sqlite:///./test.db\"\n",
    "if args.graphtask == 'regression':\n",
    "    study = optuna.create_study(direction=\"minimize\")\n",
    "elif args.graphtask == 'classification':\n",
    "    study = optuna.create_study(direction=\"maximize\")\n",
    "\n",
    "t_start = time.time()\n",
    "study.optimize(hyper.objective, n_trials=5, timeout=3600)\n",
    "len(study.get_trials())\n",
    "print(\"Time: {:.3f}s\".format(time.time() - t_start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED])\n",
    "complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
    "\n",
    "print(\"Study statistics: \")\n",
    "print(\"  Number of finished trials: \", len(study.trials))\n",
    "print(\"  Number of pruned trials: \", len(pruned_trials))\n",
    "print(\"  Number of complete trials: \", len(complete_trials))\n",
    "\n",
    "print(\"Best trial:\")\n",
    "trial = study.best_trial\n",
    "\n",
    "print(\"  Value: \", trial.value)\n",
    "\n",
    "print(\"  Params: \")\n",
    "for key, value in trial.params.items():\n",
    "    if key == 'channels':\n",
    "        print(\"--in_channels {}\".format(value))\n",
    "        print(\"--hidden_channels {}\".format(value))\n",
    "        print(\"--out_channels {}\".format(value))\n",
    "    else:\n",
    "        print(\"--{} {}\".format(key, value))\n",
    "\n",
    "with open('dataset/{}/hyperparams.txt'.format(hyper.log_folder_name), 'w') as f:\n",
    "    for key, value in trial.params.items():\n",
    "        if key == 'channels':\n",
    "            f.write(\"--in_channels {}\".format(value))\n",
    "            f.write('\\n')\n",
    "            f.write(\"--hidden_channels {}\".format(value))\n",
    "            f.write('\\n')\n",
    "            f.write(\"--out_channels {}\".format(value))\n",
    "        else:\n",
    "            f.write(\"--{} {}\".format(key, value))\n",
    "        f.write('\\n')\n",
    "\n",
    "print(optuna.importance.get_param_importances(study))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args_test = dict()\n",
    "\n",
    "# Load model\n",
    "# ts = \"2022-Oct-06-23:57:53\"\n",
    "# args_test['log_folder_name'] = os.path.join(*[args.file, args.model+'_'+args.reduced+'_'+args.schema, f\"{ts}\"])\n",
    "args_test['log_folder_name'] = hyper.log_folder_name\n",
    "args_test['exp_name'] = args.experiment_number\n",
    "args_test['fold_number'] = 0\n",
    "args_test['seed'] = args.seed\n",
    "\n",
    "test_loader, datasets_test =  generateDataLoaderTesting(args.file, args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tester = Tester(args, args_test)\n",
    "tester.test(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_embed = tester.getXEmbed()\n",
    "y_test = tester.getYTest()\n",
    "path = 'results/'+hyper.log_folder_name\n",
    "legend = None\n",
    "\n",
    "if args.graphtask == 'regression':\n",
    "    min_value = np.min(y_test)\n",
    "    max_value = np.max(y_test)\n",
    "    \n",
    "    interval_num = 10\n",
    "    interval = (max_value-min_value)/interval_num\n",
    "    ranging = [(min_value+(interval*i), min_value+(interval*(i+1))) for i in range(interval_num)]\n",
    "\n",
    "    y_test_new = list()\n",
    "    for y in y_test:\n",
    "        for i, r in enumerate(ranging):\n",
    "            if r[0] <= y < r[1]:\n",
    "                y_test_new.append(i)\n",
    "                break\n",
    "            elif y == max_value:\n",
    "                y_test_new.append(interval_num-1)\n",
    "                break\n",
    "\n",
    "    y_test = np.array(y_test_new)\n",
    "\n",
    "    legend_new = list()\n",
    "    for i, r in enumerate(ranging):\n",
    "        if i != len(ranging)-1:\n",
    "            legend_new.append('['+str(\"{:.2f}\".format(r[0]))+','+str(\"{:.2f}\".format(r[1]))+')')\n",
    "        else:\n",
    "            legend_new.append('['+str(\"{:.2f}\".format(r[0]))+','+str(\"{:.2f}\".format(r[1]))+']')\n",
    "\n",
    "    legend = legend_new"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('myenv38')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "39c542c5c02b7a511e1ae79f0bb478cef0c12733a4ae7993722600208abb2b65"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}