From 117c64aa98f0874ade2f383c651513bba6d34756 Mon Sep 17 00:00:00 2001 From: bghira Date: Thu, 21 Sep 2023 11:02:39 -0700 Subject: [PATCH] Randomise bucket processing order for multi-node processing --- helpers/caching/vae.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/caching/vae.py b/helpers/caching/vae.py index cdd96d9e..b449514d 100644 --- a/helpers/caching/vae.py +++ b/helpers/caching/vae.py @@ -1,4 +1,5 @@ import os, torch, logging +from random import shuffle from tqdm import tqdm from pathlib import Path from PIL import Image @@ -190,7 +191,7 @@ def process_buckets(self, bucket_manager): ) aspect_bucket_cache = bucket_manager.read_cache().copy() - + shuffle(aspect_bucket_cache) for bucket in aspect_bucket_cache: relevant_files = [ f