How to encode custom data into PaliGemma segmentation tokens ?

[VitoVoelker]

Hello,

I have a labeled segmentation dataset I want to encode into segmentation tokens to fine-tune PaliGemma. My segmentation masks are in polygons which I convert into a binary mask and into a bounding box to encode with the refcoco_mask2str() function. I'm working with the code below and no matter the mask I get the same segmentation tokens as output:

<loc0000><loc0000><loc0726><loc1018><seg078><seg078><seg114><seg078><seg078><seg078><seg114><seg078><seg078><seg078><seg114><seg078><seg078><seg078><seg114><seg012>

If I decode this mask again, I get a close to zero IoU score with the original mask. Is this the right approach to encode custom data, or am I missing something ? Thanks for any help in advance! :)

Code below:

import sys
import numpy as np
import tensorflow as tf
from skimage.draw import polygon
from PIL import Image, ImageDraw
from big_vision.pp.proj.paligemma.segmentation import get_refcoco_mask2str


json_data = {
    "key1": {
        "filename": "filename",
        "size": size,
        "regions": [
            {
                "shape_attributes": {
                    "name": "polygon",
                    "all_points_x": [
                        3511,
                        3529,
                        3877,
                        3903,
                        3939,
                        3954,
                        4094,
                        4094,
                        0,
                        0,
                        40,
                        142,
                        281,
                        419,
                        559,
                        740,
                        1049,
                        1211,
                        1304,
                        1350,
                        1395,
                        1408,
                        1395,
                        1439,
                        1458,
                        1472,
                        1494,
                        1501
                    ],
                    "all_points_y": [
                         2082,
                        2083,
                        2040,
                        2050,
                        2055,
                        2031,
                        2015,
                        1,
                        0,
                        1924,
                        1941,
                        1966,
                        1996,
                        2020,
                        2041,
                        2066,
                        2114,
                        2133,
                    ]
                },
                "region_attributes": {
                    "value": "",
                    "label_type": "label_type"
                }
            }
        ],
        "file_attributes": {
            "caption": "",
            "public_domain": "public_domain",
            "image_url": ""
        }
    }
}


# Function to get and resize image
def get_and_resize_image(image_path, new_size):
    with Image.open(image_path) as img:
        img_resized = img.resize(new_size)
        return np.array(img_resized), img.size, img_resized.size

# Function to convert polygon to mask
def polygon_to_mask(image_shape, all_points_x, all_points_y):
    mask = np.zeros(image_shape[:2], dtype=np.uint8)
    all_points_x = np.clip(all_points_x, 0, image_shape[1] - 1)
    all_points_y = np.clip(all_points_y, 0, image_shape[0] - 1)
    rr, cc = polygon(all_points_y, all_points_x)
    mask[rr, cc] = 1
    return mask

# Function to create bounding box from polygon points
def create_bbox(all_points_x, all_points_y):
    x_min = min(all_points_x)
    x_max = max(all_points_x)
    y_min = min(all_points_y)
    y_max = max(all_points_y)
    return [y_min, x_min, y_max, x_max]

# Function to scale coordinates
def scale_coordinates(coords, original_size, new_size):
    x_scale = new_size[0] / original_size[0]
    y_scale = new_size[1] / original_size[1]
    scaled_coords = [(int(x * x_scale), int(y * y_scale)) for x, y in coords]
    return zip(*scaled_coords)
    
    # Normalize the bounding box coordinates to be between 0 and 1
def normalize_bbox(bbox, image_size):
    y_min, x_min, y_max, x_max = bbox
    height, width = image_size
    y_min = y_min / height
    y_max = y_max / height
    x_min = x_min / width
    x_max = x_max / width
    return [y_min, x_min, y_max, x_max]


image_path = r'path/to/image'
new_size = (224,224)
image, original_size, resized_size = get_and_resize_image(image_path, new_size)

# Extract original polygon points
all_points_x = json_data["key1"]["regions"][0]["shape_attributes"]["all_points_x"]
all_points_y = json_data["key1"]["regions"][0]["shape_attributes"]["all_points_y"]

# Scale polygon points
scaled_points_x, scaled_points_y = scale_coordinates(zip(all_points_x, all_points_y), original_size, resized_size)

# Update the mask creation with the resized image shape
mask = polygon_to_mask(image_shape, list(scaled_points_x), list(scaled_points_y))
mask = np.repeat(mask[:, :, np.newaxis], 3, axis=2)  # Repeat the mask to create 3 channels

# Create and scale bounding box
bbox = create_bbox(list(scaled_points_x), list(scaled_points_y))
print("bbox:", bbox)


data = {
    'objects/mask': mask,
    'objects/refs/sentence': "This is a sentence",  # Example sentence
    'objects/bbox': bbox
}

# Convert data to TensorFlow tensors
data['objects/mask'] = tf.convert_to_tensor(data['objects/mask'], dtype=tf.uint8)
data['objects/refs/sentence'] = tf.convert_to_tensor(data['objects/refs/sentence'], dtype=tf.string)
data['objects/bbox'] = tf.convert_to_tensor(data['objects/bbox'], dtype=tf.float32)

normalized_bbox = normalize_bbox(bbox.numpy(), resized_size)
data['objects/bbox'] = tf.convert_to_tensor(normalized_bbox, dtype=tf.float32)

# Apply the refcoco_mask2str function
encoded_data = get_refcoco_mask2str()(data)

print(encoded_data)

[biswa-iitg]

I am also facing the same issue. Are you able to resolve this problem? @VitoVoelker

[VitoVoelker]

I have not found a solution so far, sadly. Have you ?

[biswa-iitg]

yes, in my case it was the bbox which was not correct. Please check the bbox. If it is wrong and covers empty region, the output will always be the same.

[biswa-iitg]

In your case, I guess, it is the mask, please make sure that mask[mask > 0] = 255, because,inside get_refcoco_mask2str function it will be divided by 255 and normalized in the range -1 to +1