humbi_textured_meshes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UYPtfQRVtAfX"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hJUx7vDcc6zZ"
   },
   "source": [
    "If you run this notebook in Colab, please change **drive_path** to the path leading to the directory on your Google Drive in which `smplx/` is stored."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Wad4JaQlaTuV"
   },
   "outputs": [],
   "source": [
    "### Mount google drive if available\n",
    "try:\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive')\n",
    "    colab = True\n",
    "except:\n",
    "    colab = False\n",
    "\n",
    "drive_path = '/content/drive/MyDrive/master_thesis/' if colab else ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cYKdnJ1MEsLD"
   },
   "source": [
    "IGNORE the cell below if the repo has already been cloned and the dependencies installed. If you run this notebook in Colab, RUN the cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xw1P1PSvaaSc"
   },
   "outputs": [],
   "source": [
    "### Clone repository and install dependencies\n",
    "\n",
    "!git clone https://github.com/maximeraafat/humbi_textured_meshes.git\n",
    "!pip install -r humbi_textured_meshes/requirements.txt\n",
    "%cd humbi_textured_meshes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4S4OWZZJNkQN"
   },
   "outputs": [],
   "source": [
    "# @title Provide the HUMBI data root URL\n",
    "\n",
    "# @markdown Proceed to the [HUMBI website](https://humbi-data.net), register and obtain the data root URL (_https://....amazonaws.com_), and modify the **HUMBI_ROOT_URL** variable below.\n",
    "\n",
    "HUMBI_ROOT_URL = '' # @param {type:\"string\"}\n",
    "\n",
    "file_object = open('humbi_root_url.txt', 'a')\n",
    "file_object.write(HUMBI_ROOT_URL)\n",
    "file_object.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tGIjs-HKa5GF"
   },
   "outputs": [],
   "source": [
    "### Install remaining dependencies (can take a few minutes)\n",
    "\n",
    "# pytorch3d\n",
    "import os\n",
    "import sys\n",
    "import torch\n",
    "import requests\n",
    "\n",
    "need_pytorch3d=False\n",
    "try:\n",
    "    import pytorch3d\n",
    "except ModuleNotFoundError:\n",
    "    need_pytorch3d=True\n",
    "\n",
    "if need_pytorch3d:\n",
    "    if torch.__version__.startswith(\"1.11.\") and sys.platform.startswith(\"linux\"):\n",
    "        # We try to install PyTorch3D via a released wheel.\n",
    "        pyt_version_str=torch.__version__.split(\"+\")[0].replace(\".\", \"\")\n",
    "        version_str=\"\".join([\n",
    "            f\"py3{sys.version_info.minor}_cu\",\n",
    "            torch.version.cuda.replace(\".\",\"\"),\n",
    "            f\"_pyt{pyt_version_str}\"\n",
    "        ])\n",
    "        !pip install fvcore iopath\n",
    "        !pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html\n",
    "    else:\n",
    "        # We try to install PyTorch3D from source.\n",
    "        !curl -LO https://github.com/NVIDIA/cub/archive/1.10.0.tar.gz\n",
    "        !tar xzf 1.10.0.tar.gz\n",
    "        os.environ[\"CUB_HOME\"] = os.getcwd() + \"/cub-1.10.0\"\n",
    "        !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'\n",
    "\n",
    "\n",
    "# detectron2\n",
    "need_detectron=False\n",
    "try:\n",
    "    import detectron2\n",
    "except ModuleNotFoundError:\n",
    "    need_detectron=True   \n",
    "\n",
    "if need_detectron:\n",
    "    !git clone https://github.com/facebookresearch/detectron2.git detectron2_repo\n",
    "    torch_version = \".\".join(torch.__version__.split(\".\")[:2])\n",
    "    cuda_version = torch.__version__.split(\"+\")[-1]\n",
    "    url = 'https://dl.fbaipublicfiles.com/detectron2/wheels/%s/torch%s/index.html' % (cuda_version, torch_version)\n",
    "    try:\n",
    "        # We try to install Detectron2 via a released wheel.\n",
    "        requests.get(url).raise_for_status()\n",
    "        !pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/$CUDA_VERSION/torch$TORCH_VERSION/index.html\n",
    "    except:\n",
    "        # We try to install Detectron2 from source.\n",
    "        !python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'\n",
    "\n",
    "\n",
    "# cleanup\n",
    "!rm -rf 1.10.0.tar.gz cub-1.10.0/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uwz-hxNLr8Kc"
   },
   "source": [
    "# Mesh reconstruction and UV color texture learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AyS9aa_deHvT"
   },
   "source": [
    "Optional flags\n",
    "\n",
    "1. `--subject $SUBJECTS` : list or range of subjects for which we construct a textured mesh. SUBJECTS is a list passed as a string, e.g. `\"[1,2,3]\"` or `\"range(1,10)\"`. If not passed, all HUMBI subjects are constructed, i.e., `\"range(1, 618)\"`\n",
    "\n",
    "2. `--poses $POSES` : list of poses (same length as SUBJECTS!). If e.g. `SUBJECTS = \"[1,2,3]\"` and `POSES = \"[1, 9, 25]\"`, then subject **1** will be reconstructed for pose **00000001**, subject **2** in pose **00000009** and subject **3** in pose **00000025** If not passed, each subject will be reconstructed in the default T-pose\n",
    "\n",
    "3. `--gdrive $drive_path` : path leading to the directory in which `smplx/` is stored, and in which the constructed textured meshes will be stored. By default it is set to the current working directory\n",
    "\n",
    "4. `--iters 30` : number of training epochs (one epoch = one iteration through all available cameras per subject in the considered pose)\n",
    "\n",
    "5. `--saveobj` : whether to store the training progress in an `.obj` file every 3 epochs (stored under `<drive_path>/humbi_output/humbi_smplx_objs`)\n",
    "\n",
    "6. `--smoothing` : whether to slightly smooth the reconstructed mesh after learning the vertex displacements\n",
    "\n",
    "7. `--nodisps` : disable storing the displacement maps (only rgb textures will be saved). By default, rgb and displacement will both be stored in 2 different texture maps\n",
    "\n",
    "8. `--val` : whether to perform validation on the 10% of the data (and leave 10% out for testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UCEpQA7tdBnj"
   },
   "outputs": [],
   "source": [
    "### Textured deformed SMPL-X mesh reconstruction\n",
    "\n",
    "# SUBJECTS is called as a string, and $SUBJECTS evaluates a string, therefore we need SUBJECTS to be a string within a string\n",
    "SUBJECTS = \" '[1, 70, 122]' \"\n",
    "\n",
    "if colab:\n",
    "    %run main.py --subjects $SUBJECTS --gdrive $drive_path --iters 4 --saveobj --smoothing\n",
    "else:\n",
    "    %run main.py --subjects $SUBJECTS --iters 4 --saveobj --smoothing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZgzVKq-8g_0f"
   },
   "source": [
    "Notice that the displacement textures stored under `<drive_path>/humbi_output/humbi_smplx_geom` are normalized, i.e., the color values are saturated to take advantage of all pixel intensities (or bits).\n",
    "\n",
    "In order to revert the normalization process and get the true displacement textures, execute the cells in the next section."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bY90HHCLsA8Q"
   },
   "source": [
    "# Get true displacement textures via inverse normalizaton"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jfyU8_zLwBS1"
   },
   "source": [
    "The true displacement textures will be stored for the selected subjects in `<drive_path>/humbi_output/humbi_smplx_true_geom/`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gpQeLaqRmXna"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from utils.normalize_disps import denormalize_disp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UYXcIMLoroTY"
   },
   "outputs": [],
   "source": [
    "### Inverse normalization on displacement maps\n",
    "\n",
    "SUBJECTS = [1, 70, 122]\n",
    "save_path_disps = drive_path + 'humbi_output/humbi_smplx_true_geom/'\n",
    "\n",
    "os.makedirs(save_path_disps, exist_ok=True)\n",
    "for subject in SUBJECTS:\n",
    "\n",
    "    img_path = drive_path + 'humbi_output/humbi_smplx_geom/disp_texture_%d.png' % subject\n",
    "\n",
    "    normalization = np.load(drive_path + 'humbi_output/humbi_smplx_npz/normalization.npz')\n",
    "    global_min = normalization['global_min'].item()\n",
    "    global_max = normalization['global_max'].item()\n",
    "\n",
    "    image = denormalize_disp(img_path, global_min, global_max)\n",
    "\n",
    "    plt.figure(figsize=(10, 10))\n",
    "    plt.imshow(image + 0.5)\n",
    "    plt.axis(\"off\")\n",
    "    plt.title('subject %d\\n' % subject)\n",
    "    plt.imsave(save_path_disps + 'disp_texture_%d.png' % subject, image + 0.5)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "humbi_textured_meshes.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}