Merge pull request #74 from 3DOM-FBK/dev-upright

Merge Dev upright

config/aliked.yaml

@@ -0,0 +1,22 @@
+# User configuration file
+
+general:
+  tile_size: (2000, 2000)
+  geom_verification: pydegensac
+  min_inliers_per_pair: 10
+  min_inlier_ratio_per_pair: 0.15
+  gv_threshold: 1
+
+extractor:
+  name: "aliked"
+  model_name: aliked-n16rot
+  max_num_keypoints: 8000
+  detection_threshold: 0.2
+  nms_radius: 3
+
+matcher:
+  name: "lightglue"
+  n_layers: 9
+  depth_confidence: 0.95 # early stopping, disable with -1
+  width_confidence: 0.99 # point pruning, disable with -1
+  filter_threshold: 0.10 # match threshold

config/cameras.yaml

@@ -1,12 +1,12 @@
 general:
-  camera_model: "pinhole" # ["simple-pinhole", "pinhole", "simple-radial", "opencv"]
+  camera_model: "simple-radial" # ["simple-pinhole", "pinhole", "simple-radial", "opencv"]
   openmvg_camera_model: "pinhole_radial_k3" # ["pinhole", "pinhole_radial_k3", "pinhole_brown_t2"]
-  single_camera: True
+  single_camera: False
 
-cam0:
-  camera_model: "pinhole"
-  images : ""
+cam0: # Do not add additional space when listing images, just comma separated
+  camera_model: "opencv"
+  images : "DSC_6466.JPG,DSC_6467.JPG"
 
 cam1:
-  camera_model: "pinhole"
-  images : "DSC_6468.jpg,DSC_6468.jpg"
+  camera_model: "simple-pinhole"
+  images : "DSC_6468.JPG"

config/dedode.yaml

@@ -0,0 +1,18 @@
+# User configuration file
+
+general:
+  tile_size: (2500, 2500)
+  geom_verification: pydegensac
+  min_inliers_per_pair: 10
+  min_inlier_ratio_per_pair: 0.15
+  gv_threshold: 1
+
+extractor:
+  name: "dedode"
+  n_features: 8000
+  upright: False
+
+matcher:
+  name: "kornia_matcher"
+  match_mode: "smnn"
+  th: 0.95

config/keynet.yaml

@@ -0,0 +1,18 @@
+# User configuration file
+
+general:
+  tile_size: (2000, 2000)
+  geom_verification: pydegensac
+  min_inliers_per_pair: 10
+  min_inlier_ratio_per_pair: 0.15
+  gv_threshold: 1
+
+extractor:
+  name: keynetaffnethardnet
+  n_features: 8000
+  upright: True
+
+matcher:
+  name: "kornia_matcher"
+  match_mode: smnn
+  th: 0.95

config/loftr.yaml

@@ -0,0 +1,14 @@
+# User configuration file
+
+general:
+  tile_size: (1000, 1000)
+  geom_verification: ransac
+  min_inliers_per_pair: 10
+  min_inlier_ratio_per_pair: 0.05
+  gv_threshold: 1
+
+extractor:
+  name: "no_extractor"
+
+matcher:
+  name: "loftr"

config/roma.yaml

@@ -5,7 +5,8 @@ general:
   tile_overlap: 0 # It is suggested to use 0 overlap to not increase the number of tiles
   geom_verification: ransac # ransac is suggested with a large number of matches (pydegensac may have issues with very large datasets)
   min_inliers_per_pair: 10
-  min_inlier_ratio_per_pair: 0.25
+  min_inlier_ratio_per_pair: 0.05
+  gv_threshold: 1
 
 extractor:
   name: "no_extractor"

config/rotations.txt

@@ -0,0 +1,10 @@
+DSC_6466.JPG 0
+DSC_6467.JPG 0
+DSC_6468.JPG 180
+DSC_6469.JPG 0
+DSC_6470.JPG 0
+DSC_6471.JPG 90
+DSC_6472.JPG 0
+DSC_6473.JPG 0
+DSC_6474.JPG 0
+DSC_6475.JPG 0

config/sift.yaml

@@ -1,20 +1,21 @@
 # User configuration file
 
 general:
-  tile_size: (2400, 2000)
+  tile_size: (2500, 2500)
   geom_verification: pydegensac
   min_inliers_per_pair: 10
-  min_inlier_ratio_per_pair: 0.25
+  min_inlier_ratio_per_pair: 0.05
+  gv_threshold: 1
 
 extractor:
   name: "sift"
   n_features: 8000
   nOctaveLayers: 3
-  contrastThreshold: 0.04
+  contrastThreshold: 0.0004
   edgeThreshold: 10
   sigma: 1.6
 
 matcher:
   name: "kornia_matcher"
   match_mode: "smnn"
-  th: 0.85
+  th: 0.9

config/superpoint+lightglue.yaml

@@ -3,13 +3,13 @@
 general:
   tile_size: (2400, 2000)
   geom_verification: PYDEGENSAC # NONE, PYDEGENSAC, MAGSAC, RANSAC, LMEDS, RHO, USAC_DEFAULT, USAC_PARALLEL, USAC_FM_8PTS, USAC_FAST, USAC_ACCURATE, USAC_PROSAC, USAC_MAGSAC
-  gv_threshold: 4
   min_inliers_per_pair: 10
-  min_inlier_ratio_per_pair: 0.25
+  min_inlier_ratio_per_pair: 0.15
+  gv_threshold: 1
 
 extractor:
   name: "superpoint"
-  max_keypoints: 8000 # -1 no limits
+  max_keypoints: 2000 # -1 no limits
   nms_radius: 4
   keypoint_threshold: 0.005
   remove_borders: 4

docs/rotations.md

@@ -0,0 +1,11 @@
+# Managing rotations
+
+The majority of DL local features do not manage rotations. To manage this kind of datasets, the optional arg `--upright` followed by `custom`, `2clusters` or `exif` can be used. `Exif` apply rotations as written in the exif metadata (sperimental). If rotations respect a default position (e.g. horizonatl) are known, can be specified passing `cusom` option. Rotations must be declared in `./config/rotations.txt` in the format:
+
+```
+image0.txt 0
+image1.txt 0
+image2.txt 90
+image3.txt 180
+image4.txt 270
+```

main.py

@@ -1,7 +1,10 @@
+import os
 import logging
 from importlib import import_module
+from pathlib import Path
 
 import deep_image_matching as dim
+from deep_image_matching.utils.loftr_roma_to_multiview import LoftrRomaToMultiview
 import yaml
 
 logger = dim.setup_logger("dim")
@@ -30,6 +33,15 @@
     camera_config_path=config.general["camera_options"],
 )
 
+if matcher.matching in ["loftr", "se2loftr", "roma"]:
+    images = os.listdir(imgs_dir)
+    image_format = Path(images[0]).suffix
+    LoftrRomaToMultiview(
+        input_dir=feature_path.parent,
+        output_dir=feature_path.parent,
+        image_dir=imgs_dir, 
+        img_ext=image_format)
+
 # Visualize view graph
 if config.general["graph"]:
     try:
@@ -99,4 +111,4 @@
             options=reconst_opts,
             verbose=config.general["verbose"],
             refine_intrinsics=refine_intrinsics,
-        )
+        )

scripts/convert_images.py

@@ -5,11 +5,15 @@
 import rasterio
 from PIL import Image
 
-
+from deep_image_matching import logger
+import rasterio as rio
+
+
 def load_img(
     path_to_img: str,
     reader: str,
 ):
+
     """
     Load an image from the given path using the specified reader.
 

scripts/globaldes_kmeans.py

@@ -0,0 +1,53 @@
+# When working with huge datasets, using DL global descriptors and KMeans clustering, rextract keyframes from the dataset
+
+from pathlib import Path
+from src.deep_image_matching.thirdparty.hloc import extract_features, pairs_from_retrieval
+from sklearn.cluster import KMeans
+import h5py
+import argparse
+import numpy as np
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--imgs_dir', type=str, help='Path to the directory containing the images')
+    parser.add_argument('--outs_dir', type=str, help='Path to the output directory')
+    parser.add_argument('--retrieval_conf', type=str, default='netvlad', help='Configuration for retrieval, e.g. netvlad')
+    parser.add_argument('--kmeans_clusters', type=int, default=1000, help='Number of clusters for KMeans')
+
+    args = parser.parse_args()
+
+    imgs_dir = Path(args.imgs_dir)
+    outs_dir = Path(args.outs_dir)
+
+    retrieval_conf = extract_features.confs[args.retrieval_conf]
+    retrieval_path = extract_features.main(retrieval_conf, imgs_dir, outs_dir)
+
+    gfeat_path = outs_dir / "global-feats-netvlad.h5"
+    with h5py.File(gfeat_path, 'r') as f:
+        images = list(f.keys())
+        first_key = list(f.keys())[0]
+        des_len = len(f[first_key]['global_descriptor'][:])
+        total_vectors = len(f.keys())
+        matrix = np.empty((total_vectors, des_len))
+        for i, key in enumerate(f.keys()):
+            matrix[i, :] = f[key]['global_descriptor'][:]
+        #print(matrix.shape)
+        #print(images);quit()
+
+    X = args.kmeans_clusters  # Number of clusters you want
+    kmeans = KMeans(n_clusters=X, random_state=42)
+    kmeans.fit(matrix)
+    labels = kmeans.labels_
+
+    keyframes = {}
+    for index, cluster in enumerate(labels):
+        keyframes[cluster] = images[index]
+
+    with open(outs_dir / "keyframes.txt", "w") as f:
+        for kfrm in keyframes.values():
+            f.write(f"left/{kfrm}\n")
+            f.write(f"right/{kfrm}\n")
+
+
+    ## Get the cluster centers (centroids)
+    #centroids = kmeans.cluster_centers_