diff --git a/grimoiressg/content_files.py b/grimoiressg/content_files.py index 854d6af..d9c722d 100644 --- a/grimoiressg/content_files.py +++ b/grimoiressg/content_files.py @@ -25,6 +25,18 @@ def handle_file(filename): return results +def deduplicate(candidates): + names = set() + results = [] + + for candidate in candidates: + if candidate["relative_filename"] not in names: + names.add(candidate["relative_filename"]) + results.append(candidate) + + return results + + def recursively_read_files(context): data = [] @@ -33,6 +45,8 @@ def recursively_read_files(context): for filename in context["filenames"]: data.extend(for_each_glob(filename, handle_file)) + data = deduplicate(data) + logger.info(f"Read %d files in total.", len(data)) return data