diff --git a/libvmaf/include/libvmaf/libvmaf.h b/libvmaf/include/libvmaf/libvmaf.h index d723c2588..a35e6bdd9 100644 --- a/libvmaf/include/libvmaf/libvmaf.h +++ b/libvmaf/include/libvmaf/libvmaf.h @@ -322,7 +322,7 @@ enum VmafMetadataFlags { * @return 0 on success, or < 0 (a negative errno code) on error. */ -int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg, uint64_t flags); +int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg); /** * Get the number of registered metadata handlers. diff --git a/libvmaf/src/feature/feature_collector.c b/libvmaf/src/feature/feature_collector.c index 888b7814a..ac4892057 100644 --- a/libvmaf/src/feature/feature_collector.c +++ b/libvmaf/src/feature/feature_collector.c @@ -20,18 +20,23 @@ #include #include #include +#include #include #include #include "dict.h" +#include "libvmaf/model.h" #include "metadata_handler.h" #include "feature_collector.h" #include "feature_name.h" #include "feature_extractor.h" #include "libvmaf/libvmaf.h" +#include "feature/alias.h" #include "log.h" #include "predict.h" +#define MAX(a,b) ((a) > (b) ? (a) : (b)) + static int aggregate_vector_init(AggregateVector *aggregate_vector) { if (!aggregate_vector) return -EINVAL; @@ -367,48 +372,63 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector, int res = 0; - VmafCallbackItem *metadata_iter = feature_collector->metadata ? - feature_collector->metadata->head : NULL; - while (metadata_iter) { - // Check current feature name is the same as the metadata feature name - if (!strcmp(metadata_iter->metadata_cfg.feature_name, feature_name)) { - - // Call the callback function with the metadata feature name - VmafMetadata data = { - .feature_name = metadata_iter->metadata_cfg.feature_name, - .picture_index = picture_index, - .score = score, - }; - metadata_iter->metadata_cfg.callback(metadata_iter->metadata_cfg.data, &data); - // Move to the next metadata - goto next_metadata; - } + VmafPredictModel *model_iter = feature_collector->models; - VmafPredictModel *model_iter = feature_collector->models; +while (model_iter) { + VmafModel *model = model_iter->model; + bool needs_computation = false; - // If metadata feature name is not the same as the current feature feature_name - // Check if metadata feature name is the predicted feature - while (model_iter) { - VmafModel *model = model_iter->model; + // Check if current score needs computation + pthread_mutex_unlock(&(feature_collector->lock)); + res = vmaf_feature_collector_get_score(feature_collector, model->name, &score, picture_index); + needs_computation = (res != 0); + pthread_mutex_lock(&(feature_collector->lock)); - pthread_mutex_unlock(&(feature_collector->lock)); - res = vmaf_feature_collector_get_score(feature_collector, - model->name, &score, picture_index); - pthread_mutex_lock(&(feature_collector->lock)); + if (needs_computation) { + // Compute the current frame's score + pthread_mutex_unlock(&(feature_collector->lock)); + res = vmaf_predict_score_at_index(model, feature_collector, picture_index, &score, true, true, 0); + pthread_mutex_lock(&(feature_collector->lock)); - if (res) { + if (!res) { + // Process all pending frames up to current index in order + unsigned process_index = feature_collector->metadata->last_seen_lowest_index; + feature_collector->metadata->last_seen_highest_index = MAX(picture_index, feature_collector->metadata->last_seen_highest_index); + + while (process_index <= feature_collector->metadata->last_seen_highest_index) { + bool frame_ready = true; + + // First check if this frame's score is ready pthread_mutex_unlock(&(feature_collector->lock)); - res |= vmaf_predict_score_at_index(model, feature_collector, - picture_index, &score, true, true, 0); + if (vmaf_feature_collector_get_score(feature_collector, model->name, &score, process_index) != 0) { + frame_ready = false; + } pthread_mutex_lock(&(feature_collector->lock)); + + if (!frame_ready) break; // Stop at first unready frame + + // Frame is ready, trigger callbacks for all features + for (unsigned j = 0; j < feature_collector->cnt; j++) { + VmafMetadata data = { + .feature_name = feature_collector->feature_vector[j]->name, + .picture_index = process_index, + .score = feature_collector->feature_vector[j]->score[process_index].value, + }; + + // Call all metadata callbacks + feature_collector->metadata->head->metadata_cfg.callback( + feature_collector->metadata->head->metadata_cfg.data, &data); + } + + process_index++; + feature_collector->metadata->last_seen_lowest_index = process_index; } - model_iter = model_iter->next; } - - next_metadata: - metadata_iter = metadata_iter->next; } + model_iter = model_iter->next; +} + unlock: feature_collector->timer.end = clock(); pthread_mutex_unlock(&(feature_collector->lock)); diff --git a/libvmaf/src/libvmaf.c b/libvmaf/src/libvmaf.c index 95338b20e..bef8f85bb 100644 --- a/libvmaf/src/libvmaf.c +++ b/libvmaf/src/libvmaf.c @@ -744,25 +744,10 @@ int vmaf_read_pictures(VmafContext *vmaf, VmafPicture *ref, VmafPicture *dist, return err; } -int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg, uint64_t flags) +int vmaf_register_metadata_handler(VmafContext *vmaf, VmafMetadataConfiguration cfg) { if (!vmaf) return -EINVAL; - if (flags & VMAF_METADATA_FLAG_FEATURE) { - VmafFeatureExtractor *fex = vmaf_get_feature_extractor_by_name(cfg.feature_name); - if (!fex) return -EINVAL; - int err = 0; - for (unsigned i = 0; fex->provided_features[i] != NULL; i++) { - VmafMetadataConfiguration new_cfg = { 0 }; - new_cfg.data = cfg.data; - new_cfg.callback = cfg.callback; - new_cfg.feature_name = strdup(fex->provided_features[i]); - err = vmaf_feature_collector_register_metadata(vmaf->feature_collector, new_cfg); - if (err) return err; - } - return 0; - } - return vmaf_feature_collector_register_metadata(vmaf->feature_collector, cfg); } diff --git a/libvmaf/src/metadata_handler.c b/libvmaf/src/metadata_handler.c index 8fe3a069a..ce51dda5c 100644 --- a/libvmaf/src/metadata_handler.c +++ b/libvmaf/src/metadata_handler.c @@ -31,6 +31,8 @@ int vmaf_metadata_init(VmafCallbackList **const metadata) if (!metadata_s) goto fail; metadata_s->head = NULL; + metadata_s->last_seen_highest_index = 0; + metadata_s->last_seen_lowest_index = 0; return 0; diff --git a/libvmaf/src/metadata_handler.h b/libvmaf/src/metadata_handler.h index f5f781fa4..c6439ec71 100644 --- a/libvmaf/src/metadata_handler.h +++ b/libvmaf/src/metadata_handler.h @@ -23,13 +23,13 @@ typedef struct VmafCallbackItem { VmafMetadataConfiguration metadata_cfg; - void (*callback)(void *, VmafMetadata *); - void *data; struct VmafCallbackItem *next; } VmafCallbackItem; typedef struct VmafCallbackList{ VmafCallbackItem *head; + unsigned last_seen_highest_index; + unsigned last_seen_lowest_index; } VmafCallbackList; int vmaf_metadata_init(VmafCallbackList **const metadata); diff --git a/libvmaf/test/test_predict.c b/libvmaf/test/test_predict.c index e3bc65378..04752caf0 100644 --- a/libvmaf/test/test_predict.c +++ b/libvmaf/test/test_predict.c @@ -26,13 +26,13 @@ #include #include +#include typedef struct { VmafDictionary **metadata; int flags; } MetaStruct; - static char *test_predict_score_at_index() { int err; @@ -77,6 +77,25 @@ void set_meta(void *data, VmafMetadata *metadata) vmaf_dictionary_set(meta->metadata, key, value, meta->flags); } +static int g_callback_order[4] = {-1, -1, -1, -1}; +static int g_callback_count = 0; + +static void test_non_monotonic_callback(void *data, VmafMetadata *m) +{ + if (!data) return; + MetaStruct *meta = data; + + if (!strcmp("vmaf", m->feature_name)) + // Track callback order + g_callback_order[m->picture_index] = g_callback_count++; + + // Store in dictionary for verification + char key[32], value[32]; + snprintf(key, sizeof(key), "vmaf_%d", m->picture_index); + snprintf(value, sizeof(value), "%f", m->score); + vmaf_dictionary_set(meta->metadata, key, value, meta->flags); +} + static char* test_propagate_metadata() { int err; @@ -157,6 +176,91 @@ static char* test_propagate_metadata() } +static char *test_propagate_metadata_non_monotonic() +{ + int err; + + // Reset global counters + g_callback_count = 0; + for (int i = 0; i < 4; i++) { + g_callback_order[i] = -1; + } + + // Setup dictionary to store callback results + VmafDictionary *dict = NULL; + MetaStruct meta_data = { + .metadata = &dict, + .flags = 0, + }; + + VmafMetadataConfiguration m = { + .feature_name = strdup("vmaf"), + .callback = test_non_monotonic_callback, + .data = &meta_data, + }; + + // Initialize feature collector + VmafFeatureCollector *feature_collector; + err = vmaf_feature_collector_init(&feature_collector); + mu_assert("problem during vmaf_feature_collector_init", !err); + + err = vmaf_feature_collector_register_metadata(feature_collector, m); + mu_assert("problem during vmaf_feature_collector_register_metadata", !err); + + // Load VMAF model + VmafModel *model; + VmafModelConfig cfg = { + .name = "vmaf", + .flags = VMAF_MODEL_FLAGS_DEFAULT, + }; + err = vmaf_model_load(&model, &cfg, "vmaf_v0.6.1"); + mu_assert("problem during vmaf_model_load", !err); + err = vmaf_feature_collector_mount_model(feature_collector, model); + mu_assert("problem during vmaf_mount_model", !err); + + // Simulate non-monotonic VMAF score computations + // Frame order: 3, 0, 2, 1 + for (unsigned i = 0; i < model->n_features; i++) { + // Frame 3 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 60., 3); + mu_assert("problem appending frame 3", !err); + + // Frame 0 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 70., 0); + mu_assert("problem appending frame 0", !err); + + // Frame 2 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 80., 2); + mu_assert("problem appending frame 2", !err); + + // Frame 1 + err = vmaf_feature_collector_append(feature_collector, + model->feature[i].name, 90., 1); + mu_assert("problem appending frame 1", !err); + } + + // Verify callback order is monotonic regardless of computation order + mu_assert("Frame 0 callback not first", g_callback_order[0] == 0); + mu_assert("Frame 1 callback not second", g_callback_order[1] == 1); + mu_assert("Frame 2 callback not third", g_callback_order[2] == 2); + mu_assert("Frame 3 callback not fourth", g_callback_order[3] == 3); + + // Verify all frame scores were propagated + for (int i = 0; i < 4; i++) { + char key[32]; + snprintf(key, sizeof(key), "vmaf_%d", i); + VmafDictionaryEntry *e = vmaf_dictionary_get(&dict, key, 0); + mu_assert("Missing frame score in metadata", e != NULL); + } + + vmaf_feature_collector_destroy(feature_collector); + vmaf_model_destroy(model); + return NULL; +} + static char *test_find_linear_function_parameters() { int err; @@ -274,5 +378,6 @@ char *run_tests() mu_run_test(test_find_linear_function_parameters); mu_run_test(test_piecewise_linear_mapping); mu_run_test(test_propagate_metadata); + mu_run_test(test_propagate_metadata_non_monotonic); return NULL; }