Apply dependency changes for conda and pip

INSTALL.md

@@ -21,7 +21,7 @@ At a high level, project requires following system dependencies.
 - PyTorch>=1.4
 - torchvision (matching PyTorch install)
 - CUDA (must be a version supported by the pytorch version)
-- OpenCV
+- OpenCV (Optional)
 
 ## Installing VISSL from pre-built binaries
 

dev/packaging/vissl_conda/vissl/meta.yaml

@@ -16,19 +16,16 @@ requirements:
     - pytorch
     - apex
     - torchvision>=0.5
-    - tensorboard>=1.15
-    - opencv
     - scipy
     - scikit-learn
     - parameterized
     - numpy >=1.11
-    - faiss
+    - faiss-gpu
     - fvcore
     - iopath
     - importlib_resources
     - hydra-core
     - tabulate
-    - pycocotools>=2.0.1
 
 test:
   imports:

dev/packaging/vissl_pip/test/test.sh

@@ -14,7 +14,7 @@ conda install -y -c pytorch pytorch=1.5.1 cudatoolkit=10.1 torchvision
 pip install apex -f https://dl.fbaipublicfiles.com/vissl/packaging/apexwheels/py37_cu101_pyt151/download.html
 #pip install vissl --no-index -f https://dl.fbaipublicfiles.com/vissl/packaging/visslwheels/download.html
 pip install vissl
-python -c "import vissl, apex, cv2"
+python -c "import vissl, apex"
 cd loc1
 python -m unittest discover -v -s tests
 dev/run_quick_tests.sh

docs/requirements.txt

@@ -9,9 +9,9 @@ sphinx_rtd_theme
 sphinx_markdown_tables
 mock
 opencv-python
-tensorboard==1.15.0
+tensorboard
 hydra-core>=1.0
-faiss>=1.5.3
+faiss-gpu
 cython
 scikit-learn
 parameterized==0.7.4

docs/source/installation.rst

@@ -13,7 +13,7 @@ At a high level, project requires following system dependencies.
 - PyTorch>=1.4
 - torchvision (matching PyTorch install)
 - CUDA (must be a version supported by the pytorch version)
-- OpenCV
+- OpenCV (Optional)
 
 Installing VISSL from pre-built binaries
 -------------------------------------------

docs/source/visualization.rst

@@ -28,21 +28,27 @@ parameters exposed by VISSL for Tensorboard:
 
 .. code-block:: yaml
 
-    TENSORBOARD_SETUP:
-      # whether to use tensorboard for the visualization
-      USE_TENSORBOARD: False
-      # log directory for tensorboard events
-      LOG_DIR: "."
-      EXPERIMENT_LOG_DIR: "tensorboard"
-      # flush logs every n minutes
-      FLUSH_EVERY_N_MIN: 5
-      # whether to log the model parameters to tensorboard
-      LOG_PARAMS: True
-      # whether ttp log the model parameters gradients to tensorboard
-      LOG_PARAMS_GRADIENTS: True
-      # if we want to log the model parameters every few iterations, set the iteration
-      # frequency. -1 means the params will be logged only at the end of epochs.
-      LOG_PARAMS_EVERY_N_ITERS: 310
+    HOOKS:
+      TENSORBOARD_SETUP:
+        # whether to use tensorboard for the visualization
+        USE_TENSORBOARD: False
+        # log directory for tensorboard events
+        LOG_DIR: "."
+        EXPERIMENT_LOG_DIR: "tensorboard"
+        # flush logs every n minutes
+        FLUSH_EVERY_N_MIN: 5
+        # whether to log the model parameters to tensorboard
+        LOG_PARAMS: True
+        # whether ttp log the model parameters gradients to tensorboard
+        LOG_PARAMS_GRADIENTS: True
+        # if we want to log the model parameters every few iterations, set the iteration
+        # frequency. -1 means the params will be logged only at the end of epochs.
+        LOG_PARAMS_EVERY_N_ITERS: 310
+
+.. note::
+
+    Please install tensorboard manually: if pip environment: :code:`pip install tensorboard` or if using conda and you prefer conda install of tensorboard:  :code:`conda install -c conda-forge tensorboard`.
+
 
 Example usage
 ---------------

requirements.txt

@@ -1,11 +1,8 @@
-tensorboard==1.15.0
 hydra-core>=1.0
-faiss>=1.5.3
 cython
 scikit-learn
 parameterized==0.7.4
 numpy>=1.15
 tabulate
-pycocotools>=2.0.1
 fvcore
 fairscale

setup.py

@@ -55,6 +55,9 @@ def get_version():
             "pre-commit",
             "nbconvert",
             "bs4",
+            "faiss-gpu",
+            "pycocotools>=2.0.1",
+            "tensorboard>=1.15",
         ]
     },
 )

tools/cluster_features_and_label.py

@@ -10,7 +10,6 @@
 from argparse import Namespace
 from typing import Any, List
 
-import faiss
 import numpy as np
 from hydra.experimental import compose, initialize_config_module
 from run_distributed_engines import launch_distributed
@@ -21,7 +20,7 @@
 from vissl.utils.hydra_config import AttrDict, convert_to_attrdict, is_hydra_available
 from vissl.utils.io import save_file
 from vissl.utils.logger import setup_logging, shutdown_logging
-from vissl.utils.misc import merge_features, set_seeds
+from vissl.utils.misc import merge_features, set_seeds, is_faiss_available
 
 
 def get_data_features_and_images(cfg: AttrDict):
@@ -45,6 +44,13 @@ def get_data_features_and_images(cfg: AttrDict):
 
 
 def cluster_features_and_label(args: Namespace, cfg: AttrDict):
+    # faiss is an optional dependency for VISSL.
+    assert is_faiss_available(), (
+        "Please install faiss using conda install faiss-gpu -c pytorch "
+        "if using conda or pip install faiss-gpu"
+    )
+    import faiss
+
     cluster_backend = cfg.CLUSTERFIT.CLUSTER_BACKEND
     num_clusters = cfg.CLUSTERFIT.NUM_CLUSTERS
     data_split = cfg.CLUSTERFIT.FEATURES.DATA_PARTITION

vissl/data/datasets/coco.py

@@ -53,6 +53,7 @@ def get_valid_objs(entry, objs):
 
 
 def get_coco_imgs_labels_info(split, data_source_dir, args):
+    # pycocotools is an optional dependency for VISSL
     from pycocotools.coco import COCO
 
     json_file = f"{data_source_dir}/annotations/instances_{split}2014.json"

vissl/data/ssl_transforms/img_pil_to_lab_tensor.py

@@ -2,11 +2,11 @@
 
 from typing import Any, Dict
 
-import cv2
 import numpy as np
 import torch
 from classy_vision.dataset.transforms import register_transform
 from classy_vision.dataset.transforms.classy_transform import ClassyTransform
+from vissl.utils.misc import is_opencv_available
 
 
 @register_transform("ImgPil2LabTensor")
@@ -39,6 +39,12 @@ def __call__(self, image):
         return img_lab_tensor
 
     def _convertbgr2lab(self, img):
+        # opencv is not a hard dependency for VISSL so we do the import locally
+        assert (
+            is_opencv_available()
+        ), "Please install OpenCV using: pip install opencv-python"
+        import cv2
+
         # img is [0, 255] , HWC, BGR format, uint8 type
         assert len(img.shape) == 3, "Image should have dim H x W x 3"
         assert img.shape[2] == 3, "Image should have dim H x W x 3"

vissl/hooks/__init__.py

@@ -99,7 +99,12 @@ def default_hook_generator(cfg: AttrDict) -> List[ClassyHook]:
     if cfg.MODEL.MODEL_COMPLEXITY.COMPUTE_COMPLEXITY:
         hooks.extend([SSLModelComplexityHook()])
     if cfg.TENSORBOARD_SETUP.USE_TENSORBOARD:
-        assert is_tensorboard_available(), "Tensorboard must be installed to use it."
+        assert is_tensorboard_available(), (
+            "Tensorboard must be installed to use it. Please install tensorboard using:"
+            "If pip environment: `pip install tensorboard` "
+            "If using conda and you prefer conda install of tensorboard: "
+            "`conda install -c conda-forge tensorboard`"
+        )
         tb_hook = get_tensorboard_hook(cfg)
         hooks.extend([tb_hook])
 

vissl/utils/misc.py

@@ -25,6 +25,37 @@ def is_apex_available():
     return apex_available
 
 
+def is_faiss_available():
+    """
+    Check if faiss is available with simple python imports.
+    To install faiss, simply do:
+        If using PIP env: `pip install faiss-gpu`
+        If using conda env: `conda install faiss-gpu -c pytorch`
+    """
+    try:
+        import faiss  # NOQA
+
+        faiss_available = True
+    except ImportError:
+        faiss_available = False
+    return faiss_available
+
+
+def is_opencv_available():
+    """
+    Check if opencv is available with simple python imports.
+    To install opencv, simply do: `pip install opencv-python`
+    regardless of whether using conda or pip environment.
+    """
+    try:
+        import cv2  # NOQA
+
+        opencv_available = True
+    except ImportError:
+        opencv_available = False
+    return opencv_available
+
+
 def find_free_tcp_port():
     """
     Find the free port that can be used for Rendezvous on the local machine.