From 78f0e225074a66a6d43296bd3b51e327c577eab6 Mon Sep 17 00:00:00 2001 From: sigmasternchen Date: Sun, 12 Jan 2025 22:14:00 +0100 Subject: [PATCH] fix: Deduplicate content list before processing --- grimoiressg/content_files.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/grimoiressg/content_files.py b/grimoiressg/content_files.py index 854d6af..d9c722d 100644 --- a/grimoiressg/content_files.py +++ b/grimoiressg/content_files.py @@ -25,6 +25,18 @@ def handle_file(filename): return results +def deduplicate(candidates): + names = set() + results = [] + + for candidate in candidates: + if candidate["relative_filename"] not in names: + names.add(candidate["relative_filename"]) + results.append(candidate) + + return results + + def recursively_read_files(context): data = [] @@ -33,6 +45,8 @@ def recursively_read_files(context): for filename in context["filenames"]: data.extend(for_each_glob(filename, handle_file)) + data = deduplicate(data) + logger.info(f"Read %d files in total.", len(data)) return data