libvmaf: simplify metadata propagation by deferring until vmaf scores…

… are ready

The metadata propagation system has been significantly refactored to
reduce complexity and threading issues:

- Changed from per-feature propagation to propagating metadata when VMAF
scores are complete
- Added frame index tracking in metadata handler to ensure ordered
processing
- Added test coverage for non-monotonic frame processing scenarios

This change eliminates the need for complex reordering logic at the
caller level and
reduces potential thread synchronization issues by only propagating
metadata when all
necessary metrics are calculated.

libvmaf/include/libvmaf/libvmaf.h

@@ -322,7 +322,7 @@ enum VmafMetadataFlags {
  * @return 0 on success, or < 0 (a negative errno code) on error.
  */
 
-int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg, uint64_t flags);
+int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg);
 
 /**
  * Get the number of registered metadata handlers.

libvmaf/src/feature/feature_collector.c

@@ -20,18 +20,23 @@
 #include <pthread.h>
 #include <stdarg.h>
 #include <stdbool.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "dict.h"
+#include "libvmaf/model.h"
 #include "metadata_handler.h"
 #include "feature_collector.h"
 #include "feature_name.h"
 #include "feature_extractor.h"
 #include "libvmaf/libvmaf.h"
+#include "feature/alias.h"
 #include "log.h"
 #include "predict.h"
 
+#define MAX(a,b) ((a) > (b) ? (a) : (b))
+
 static int aggregate_vector_init(AggregateVector *aggregate_vector)
 {
     if (!aggregate_vector) return -EINVAL;
@@ -367,48 +372,63 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector,
 
     int res = 0;
 
-    VmafCallbackItem *metadata_iter = feature_collector->metadata ?
-                                      feature_collector->metadata->head : NULL;
-    while (metadata_iter) {
-        // Check current feature name is the same as the metadata feature name
-        if (!strcmp(metadata_iter->metadata_cfg.feature_name, feature_name)) {
-
-            // Call the callback function with the metadata feature name
-            VmafMetadata data = {
-                .feature_name = metadata_iter->metadata_cfg.feature_name,
-                .picture_index = picture_index,
-                .score = score,
-            };
-            metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data);
-            // Move to the next metadata
-            goto next_metadata;
-        }
+    VmafPredictModel *model_iter = feature_collector->models;
 
-        VmafPredictModel *model_iter = feature_collector->models;
+while (model_iter) {
+    VmafModel *model = model_iter->model;
+    bool needs_computation = false;
 
-        // If metadata feature name is not the same as the current feature feature_name
-        // Check if metadata feature name is the predicted feature
-        while (model_iter) {
-            VmafModel *model = model_iter->model;
+    // Check if current score needs computation
+    pthread_mutex_unlock(&(feature_collector->lock));
+    res = vmaf_feature_collector_get_score(feature_collector, model->name, &score, picture_index);
+    needs_computation = (res != 0);
+    pthread_mutex_lock(&(feature_collector->lock));
 
-            pthread_mutex_unlock(&(feature_collector->lock));
-            res = vmaf_feature_collector_get_score(feature_collector,
-                    model->name, &score, picture_index);
-            pthread_mutex_lock(&(feature_collector->lock));
+    if (needs_computation) {
+        // Compute the current frame's score
+        pthread_mutex_unlock(&(feature_collector->lock));
+        res = vmaf_predict_score_at_index(model, feature_collector, picture_index, &score, true, true, 0);
+        pthread_mutex_lock(&(feature_collector->lock));
 
-            if (res) {
+        if (!res) {
+            // Process all pending frames up to current index in order
+            unsigned process_index = feature_collector->metadata->last_seen_lowest_index;
+            feature_collector->metadata->last_seen_highest_index = MAX(picture_index, feature_collector->metadata->last_seen_highest_index);
+
+            while (process_index <= feature_collector->metadata->last_seen_highest_index) {
+                bool frame_ready = true;
+
+                // First check if this frame's score is ready
                 pthread_mutex_unlock(&(feature_collector->lock));
-                res |= vmaf_predict_score_at_index(model, feature_collector,
-                        picture_index, &score, true, true, 0);
+                if (vmaf_feature_collector_get_score(feature_collector, model->name, &score, process_index) != 0) {
+                    frame_ready = false;
+                }
                 pthread_mutex_lock(&(feature_collector->lock));
+
+                if (!frame_ready) break;  // Stop at first unready frame
+
+                // Frame is ready, trigger callbacks for all features
+                for (unsigned j = 0; j < feature_collector->cnt; j++) {
+                    VmafMetadata data = {
+                        .feature_name = feature_collector->feature_vector[j]->name,
+                        .picture_index = process_index,
+                        .score = feature_collector->feature_vector[j]->score[process_index].value,
+                    };
+
+                    // Call all metadata callbacks
+                    feature_collector->metadata->head->metadata_cfg.callback(
+                        feature_collector->metadata->head->metadata_cfg.data, &data);
+                }
+
+                process_index++;
+                feature_collector->metadata->last_seen_lowest_index = process_index;
             }
-            model_iter = model_iter->next;
         }
-
-    next_metadata:
-        metadata_iter = metadata_iter->next;
     }
 
+    model_iter = model_iter->next;
+}
+
 unlock:
     feature_collector->timer.end = clock();
     pthread_mutex_unlock(&(feature_collector->lock));

libvmaf/src/libvmaf.c

@@ -744,25 +744,10 @@ int vmaf_read_pictures(VmafContext *vmaf, VmafPicture *ref, VmafPicture *dist,
     return err;
 }
 
-int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg, uint64_t flags)
+int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg)
 {
     if (!vmaf) return -EINVAL;
 
-    if (flags & VMAF_METADATA_FLAG_FEATURE) {
-        VmafFeatureExtractor *fex = vmaf_get_feature_extractor_by_name(cfg.feature_name);
-        if (!fex) return -EINVAL;
-        int err = 0;
-        for (unsigned i = 0; fex->provided_features[i] != NULL; i++) {
-            VmafMetadataConfiguration new_cfg = { 0 };
-            new_cfg.data = cfg.data;
-            new_cfg.callback = cfg.callback;
-            new_cfg.feature_name = strdup(fex->provided_features[i]);
-            err = vmaf_feature_collector_register_metadata(vmaf->feature_collector, new_cfg);
-            if (err) return err;
-        }
-        return 0;
-    }
-
     return vmaf_feature_collector_register_metadata(vmaf->feature_collector, cfg);
 }
 

libvmaf/src/metadata_handler.c

@@ -31,6 +31,8 @@ int vmaf_metadata_init(VmafCallbackList **const metadata)
     if (!metadata_s) goto fail;
 
     metadata_s->head = NULL;
+    metadata_s->last_seen_highest_index = 0;
+    metadata_s->last_seen_lowest_index = 0;
 
     return 0;
 

libvmaf/src/metadata_handler.h

@@ -23,13 +23,13 @@
 
 typedef struct VmafCallbackItem {
     VmafMetadataConfiguration metadata_cfg;
-    void (*callback)(void *, VmafMetadata *);
-    void *data;
     struct VmafCallbackItem *next;
 } VmafCallbackItem;
 
 typedef struct  VmafCallbackList{
     VmafCallbackItem *head;
+    unsigned last_seen_highest_index;
+    unsigned last_seen_lowest_index;
 } VmafCallbackList;
 
 int vmaf_metadata_init(VmafCallbackList **const metadata);

libvmaf/test/test_predict.c

@@ -26,13 +26,13 @@
 
 #include <libvmaf/model.h>
 #include <math.h>
+#include <string.h>
 
 typedef struct {
     VmafDictionary **metadata;
     int flags;
 } MetaStruct;
 
-
 static char *test_predict_score_at_index()
 {
     int err;
@@ -77,6 +77,25 @@ void set_meta(void *data, VmafMetadata *metadata)
     vmaf_dictionary_set(meta->metadata, key, value, meta->flags);
 }
 
+static int g_callback_order[4] = {-1, -1, -1, -1};
+static int g_callback_count = 0;
+
+static void test_non_monotonic_callback(void *data, VmafMetadata *m)
+{
+    if (!data) return;
+    MetaStruct *meta = data;
+
+    if (!strcmp("vmaf", m->feature_name))
+        // Track callback order
+        g_callback_order[m->picture_index] = g_callback_count++;
+
+    // Store in dictionary for verification
+    char key[32], value[32];
+    snprintf(key, sizeof(key), "vmaf_%d", m->picture_index);
+    snprintf(value, sizeof(value), "%f", m->score);
+    vmaf_dictionary_set(meta->metadata, key, value, meta->flags);
+}
+
 static char* test_propagate_metadata()
 {
     int err;
@@ -157,6 +176,91 @@ static char* test_propagate_metadata()
 
 }
 
+static char *test_propagate_metadata_non_monotonic()
+{
+    int err;
+
+    // Reset global counters
+    g_callback_count = 0;
+    for (int i = 0; i < 4; i++) {
+        g_callback_order[i] = -1;
+    }
+
+    // Setup dictionary to store callback results
+    VmafDictionary *dict = NULL;
+    MetaStruct meta_data = {
+        .metadata = &dict,
+        .flags    = 0,
+    };
+
+    VmafMetadataConfiguration m = {
+        .feature_name = strdup("vmaf"),
+        .callback = test_non_monotonic_callback,
+        .data = &meta_data,
+    };
+
+    // Initialize feature collector
+    VmafFeatureCollector *feature_collector;
+    err = vmaf_feature_collector_init(&feature_collector);
+    mu_assert("problem during vmaf_feature_collector_init", !err);
+
+    err = vmaf_feature_collector_register_metadata(feature_collector, m);
+    mu_assert("problem during vmaf_feature_collector_register_metadata", !err);
+
+    // Load VMAF model
+    VmafModel *model;
+    VmafModelConfig cfg = {
+        .name = "vmaf",
+        .flags = VMAF_MODEL_FLAGS_DEFAULT,
+    };
+    err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1");
+    mu_assert("problem during vmaf_model_load", !err);
+    err = vmaf_feature_collector_mount_model(feature_collector, model);
+    mu_assert("problem during vmaf_mount_model", !err);
+
+    // Simulate non-monotonic VMAF score computations
+    // Frame order: 3, 0, 2, 1
+    for (unsigned i = 0; i < model->n_features; i++) {
+        // Frame 3
+        err = vmaf_feature_collector_append(feature_collector,
+                                          model->feature[i].name, 60., 3);
+        mu_assert("problem appending frame 3", !err);
+
+        // Frame 0
+        err = vmaf_feature_collector_append(feature_collector,
+                                          model->feature[i].name, 70., 0);
+        mu_assert("problem appending frame 0", !err);
+
+        // Frame 2
+        err = vmaf_feature_collector_append(feature_collector,
+                                          model->feature[i].name, 80., 2);
+        mu_assert("problem appending frame 2", !err);
+
+        // Frame 1
+        err = vmaf_feature_collector_append(feature_collector,
+                                          model->feature[i].name, 90., 1);
+        mu_assert("problem appending frame 1", !err);
+    }
+
+    // Verify callback order is monotonic regardless of computation order
+    mu_assert("Frame 0 callback not first", g_callback_order[0] == 0);
+    mu_assert("Frame 1 callback not second", g_callback_order[1] == 1);
+    mu_assert("Frame 2 callback not third", g_callback_order[2] == 2);
+    mu_assert("Frame 3 callback not fourth", g_callback_order[3] == 3);
+
+    // Verify all frame scores were propagated
+    for (int i = 0; i < 4; i++) {
+        char key[32];
+        snprintf(key, sizeof(key), "vmaf_%d", i);
+        VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, key, 0);
+        mu_assert("Missing frame score in metadata", e != NULL);
+    }
+
+    vmaf_feature_collector_destroy(feature_collector);
+    vmaf_model_destroy(model);
+    return NULL;
+}
+
 static char *test_find_linear_function_parameters()
 {
     int err;
@@ -274,5 +378,6 @@ char *run_tests()
     mu_run_test(test_find_linear_function_parameters);
     mu_run_test(test_piecewise_linear_mapping);
     mu_run_test(test_propagate_metadata);
+    mu_run_test(test_propagate_metadata_non_monotonic);
     return NULL;
 }