diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index d7cf69050..8226b49a4 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -235,7 +235,6 @@ class Library: library_dir: Path | None = None engine: Engine | None = None folder: Folder | None = None - included_files: set[Path] = set() def __init__(self) -> None: self.dupe_entries_count: int = -1 # NOTE: For internal management. @@ -248,7 +247,6 @@ def close(self): self.engine.dispose() self.library_dir = None self.folder = None - self.included_files = set() self.dupe_entries_count = -1 self.dupe_files_count = -1 @@ -1169,6 +1167,10 @@ def has_entry_with_path(self, path: Path) -> bool: with Session(self.engine) as session: return session.query(exists().where(Entry.path == path)).scalar() + def all_paths(self) -> Iterable[tuple[int, Path]]: + with Session(self.engine) as session: + return ((i, p) for i, p in session.execute(select(Entry.id, Entry.path)).all()) + def get_paths(self, limit: int = -1) -> list[str]: path_strings: list[str] = [] with Session(self.engine) as session: @@ -1505,6 +1507,23 @@ def update_entry_path(self, entry_id: int | Entry, path: Path) -> bool: session.commit() return True + def update_entry_paths(self, paths: Iterable[tuple[int, Path]]) -> bool: + """Set the path field of many entries. + + Returns True if the action succeeded and False if any path already exists. + """ + with Session(self.engine) as session: + for entry_id, new_path in paths: + stmt = update(Entry).where(Entry.id == entry_id).values(path=new_path) + session.execute(stmt) + try: + session.commit() + except IntegrityError as e: + logger.error(e) + session.rollback() + return False + return True + def remove_tag(self, tag_id: int) -> bool: with Session(self.engine, expire_on_commit=False) as session: try: diff --git a/src/tagstudio/core/library/alchemy/registries/unlinked_registry.py b/src/tagstudio/core/library/alchemy/registries/unlinked_registry.py deleted file mode 100644 index 0e876ac5c..000000000 --- a/src/tagstudio/core/library/alchemy/registries/unlinked_registry.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: (c) TagStudio Contributors -# SPDX-License-Identifier: GPL-3.0-only - - -from collections.abc import Iterator -from dataclasses import dataclass, field -from pathlib import Path - -import structlog -from wcmatch import pathlib - -from tagstudio.core.library.alchemy.library import Library -from tagstudio.core.library.alchemy.models import Entry -from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, Ignore, ignore_to_glob -from tagstudio.core.utils.types import unwrap - -logger = structlog.get_logger() - - -@dataclass -class UnlinkedRegistry: - """State tracker for unlinked entries.""" - - lib: Library - files_fixed_count: int = 0 - unlinked_entries: list[Entry] = field(default_factory=list) - - @property - def unlinked_entries_count(self) -> int: - return len(self.unlinked_entries) - - def reset(self): - self.unlinked_entries.clear() - - def refresh_unlinked_files(self) -> Iterator[int]: - """Track the number of entries that point to an invalid filepath.""" - logger.info("[UnlinkedRegistry] Refreshing unlinked files...") - - self.unlinked_entries = [] - for i, entry in enumerate(self.lib.all_entries()): - yield i - full_path = unwrap(self.lib.library_dir) / entry.path - if not full_path.exists() or not full_path.is_file(): - self.unlinked_entries.append(entry) - - def match_unlinked_file_entry(self, match_entry: Entry) -> list[Path]: - """Try and match unlinked file entries with matching results in the library directory. - - Works if files were just moved to different subfolders and don't have duplicate names. - """ - library_dir = unwrap(self.lib.library_dir) - matches: list[Path] = [] - - # NOTE: ignore_to_glob() is needed for wcmatch, not ripgrep. - ignore_patterns = ignore_to_glob(Ignore.get_patterns(library_dir)) - for path in pathlib.Path(str(library_dir)).glob( - f"***/{match_entry.path.name}", - flags=PATH_GLOB_FLAGS, - exclude=ignore_patterns, - ): - if path.is_dir(): - continue - if path.name == match_entry.path.name: - new_path = Path(path).relative_to(library_dir) - matches.append(new_path) - - logger.info("[UnlinkedRegistry] Matches", matches=matches) - return matches - - def fix_unlinked_entries(self) -> Iterator[int]: - """Attempt to fix unlinked file entries by finding a match in the library directory.""" - self.files_fixed_count = 0 - matched_entries: list[Entry] = [] - for i, entry in enumerate(self.unlinked_entries): - yield i - item_matches = self.match_unlinked_file_entry(entry) - if len(item_matches) == 1: - logger.info( - "[UnlinkedRegistry]", - entry=entry.path.as_posix(), - item_matches=item_matches[0].as_posix(), - ) - if not self.lib.update_entry_path(entry.id, item_matches[0]): - try: - match = unwrap(self.lib.get_entry_full_by_path(item_matches[0])) - entry_full = unwrap(self.lib.get_entry_full(entry.id)) - self.lib.merge_entries(entry_full, match) - except AttributeError: - continue - self.files_fixed_count += 1 - matched_entries.append(entry) - - for entry in matched_entries: - self.unlinked_entries.remove(entry) - - def remove_unlinked_entries(self) -> None: - self.lib.remove_entries(list(map(lambda unlinked: unlinked.id, self.unlinked_entries))) - self.unlinked_entries = [] diff --git a/src/tagstudio/core/library/ignore.py b/src/tagstudio/core/library/ignore.py index 63b7ab18e..0a2f6dcd1 100644 --- a/src/tagstudio/core/library/ignore.py +++ b/src/tagstudio/core/library/ignore.py @@ -14,7 +14,7 @@ logger = structlog.get_logger() -PATH_GLOB_FLAGS = glob.GLOBSTARLONG | glob.DOTGLOB | glob.NEGATE | pathlib.MATCHBASE +PATH_GLOB_FLAGS = glob.GLOBSTARLONG | glob.DOTGLOB | glob.NEGATE | pathlib.MATCHBASE | pathlib.NODIR GLOBAL_IGNORE = [ diff --git a/src/tagstudio/core/library/refresh.py b/src/tagstudio/core/library/refresh.py index 824cae527..214ebb005 100644 --- a/src/tagstudio/core/library/refresh.py +++ b/src/tagstudio/core/library/refresh.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: GPL-3.0-only -import shutil +import subprocess +import unicodedata from collections.abc import Iterator from dataclasses import dataclass, field from datetime import datetime as dt @@ -10,12 +11,13 @@ from time import time import structlog -from wcmatch import pathlib +from wcmatch import glob from tagstudio.core.library.alchemy.library import Library from tagstudio.core.library.alchemy.models import Entry from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, Ignore, ignore_to_glob -from tagstudio.core.utils.silent_subprocess import silent_run # pyright: ignore +from tagstudio.core.utils.ripgrep_status import RipgrepStatus +from tagstudio.core.utils.silent_subprocess import silent_popen from tagstudio.core.utils.types import unwrap logger = structlog.get_logger(__name__) @@ -24,32 +26,106 @@ @dataclass class RefreshTracker: library: Library - files_not_in_library: list[Path] = field(default_factory=list) + + _paths_to_id: dict[Path, int] = field(default_factory=dict) + _expected_paths: set[Path] = field(default_factory=set) + + _missing_paths: dict[Path, int] = field(default_factory=dict) + _new_paths: set[Path] = field(default_factory=set) + + @staticmethod + def _normalize(path: Path) -> Path: + return Path(unicodedata.normalize("NFD", str(path))) + + @property + def missing_files_count(self) -> int: + return len(self._missing_paths) @property - def files_count(self) -> int: - return len(self.files_not_in_library) + def new_files_count(self) -> int: + return len(self._new_paths) + + def _add_path(self, entry_id: int, path: Path): + self._paths_to_id[path] = entry_id + self._expected_paths.add(path) + + def _del_path(self, path: Path): + self._paths_to_id.pop(path) + self._expected_paths.remove(path) def save_new_files(self) -> Iterator[int]: """Save the list of files that are not in the library.""" batch_size = 200 - index = 0 - while index < len(self.files_not_in_library): - yield index - end = min(len(self.files_not_in_library), index + batch_size) - entries = [ - Entry( - path=entry_path, - folder=unwrap(self.library.folder), - fields=[], - date_added=dt.now(), - ) - for entry_path in self.files_not_in_library[index:end] - ] - self.library.add_entries(entries) - index = end - self.files_not_in_library = [] + all_entries = [ + Entry( + path=entry_path, + folder=unwrap(self.library.folder), + fields=[], + date_added=dt.now(), + ) + for entry_path in self._new_paths + ] + for start in range(0, len(all_entries), batch_size): + yield start + end = start + batch_size + entries = all_entries[start:end] + entry_ids = self.library.add_entries(entries) + + assert len(entry_ids) == len(entries) + for i in range(len(entry_ids)): + id = entry_ids[i] + path = entries[i].path + self._add_path(id, path) + + self._new_paths.clear() + + def fix_unlinked_entries(self): + """Attempt to fix unlinked file entries by finding a match in the library directory.""" + new_paths: dict[Path, list[Path]] = {} + new_normalized_paths: dict[Path, list[Path]] = {} + for path in self._new_paths: + name = Path(path.name) + new_paths.setdefault(name, []).append(path) + normalized_name = self._normalize(name) + if normalized_name != name: + new_normalized_paths.setdefault(normalized_name, []).append(name) + + fixed: list[tuple[int, Path, Path]] = [] + for ( + path, + entry_id, + ) in self._missing_paths.items(): + name = Path(path.name) + if name not in new_paths: + name = self._normalize(name) + if (names := new_normalized_paths.get(name)) and len(names) == 1: + name = names[0] + + if name not in new_paths or len(new_paths[name]) != 1: + continue + new_path = new_paths.pop(name)[0] + self._new_paths.remove(new_path) + + fixed.append((entry_id, path, new_path)) + + paths = ((id, new_path) for (id, _path, new_path) in fixed) + if not self.library.update_entry_paths(paths): + return + + for entry_id, path, new_path in fixed: + self._del_path(path) + self._add_path(entry_id, new_path) + self._missing_paths.pop(path) + + def remove_unlinked_entries(self) -> None: + to_remove = [] + for path, id in self._missing_paths.items(): + to_remove.append(id) + self._del_path(path) + self._missing_paths.clear() + + self.library.remove_entries(to_remove) def refresh_dir(self, library_dir: Path, force_internal_tools: bool = False) -> Iterator[int]: """Scan a directory for files, and add those relative filenames to internal variables. @@ -62,144 +138,102 @@ def refresh_dir(self, library_dir: Path, force_internal_tools: bool = False) -> if self.library.library_dir is None: raise ValueError("No library directory set.") - ignore_patterns = Ignore.get_patterns(library_dir) + start_time = time() + self._paths_to_id = dict((p, i) for i, p in self.library.all_paths()) + self._expected_paths = set(self._paths_to_id.keys()) + logger.info( + "[Refresh]: Fetch entry paths", + duration=(time() - start_time), + ) - if force_internal_tools: - return self.__wc_add(library_dir, ignore_to_glob(ignore_patterns)) + ignore_patterns = Ignore.get_patterns(library_dir) - dir_list: list[str] | None = self.__get_dir_list(library_dir, ignore_patterns) + yield 0 + raw_paths = None + if not force_internal_tools: + rg_path = RipgrepStatus.which() + if rg_path is None: + logger.warning("[Refresh: ripgrep not found on system]") + else: + raw_paths = self.__rg(rg_path, library_dir, ignore_patterns) # Use ripgrep if it was found and working, else fallback to wcmatch. - if dir_list is not None: - return self.__rg_add(library_dir, dir_list) - else: - return self.__wc_add(library_dir, ignore_to_glob(ignore_patterns)) - - def __get_dir_list(self, library_dir: Path, ignore_patterns: list[str]) -> list[str] | None: - """Use ripgrep to return a list of matched directories and files. - - Return `None` if ripgrep not found on system. - """ - rg_path = shutil.which("rg") - # Use ripgrep if found on system - if rg_path is not None: - logger.info("[Refresh: Using ripgrep for scanning]") - - compiled_ignore_path = library_dir / ".TagStudio" / ".compiled_ignore" - - # Write compiled ignore patterns (built-in + user) to a temp file to pass to ripgrep - with open(compiled_ignore_path, "w") as pattern_file: - pattern_file.write("\n".join(ignore_patterns)) - - result = silent_run( - " ".join( - [ - "rg", - "--files", - "--follow", - "--hidden", - "--ignore-file", - f'"{str(compiled_ignore_path)}"', - ] - ), - cwd=library_dir, - capture_output=True, - shell=True, - encoding="UTF-8", - ) - compiled_ignore_path.unlink() - - if result.stderr: - logger.error(result.stderr) - - return result.stdout.splitlines() # pyright: ignore [reportReturnType] - - logger.warning("[Refresh: ripgrep not found on system]") - return None - - def __rg_add(self, library_dir: Path, dir_list: list[str]) -> Iterator[int]: - start_time_total = time() - start_time_loop = time() - dir_file_count = 0 - self.files_not_in_library = [] - - for r in dir_list: - f = pathlib.Path(r) - - end_time_loop = time() - # Yield output every 1/30 of a second - if (end_time_loop - start_time_loop) > 0.034: - yield dir_file_count - start_time_loop = time() - - # Skip if the file/path is already mapped in the Library - if f in self.library.included_files: - dir_file_count += 1 - continue - - # Ignore if the file is a directory - if f.is_dir(): - continue + if raw_paths is None: + raw_paths = self.__wc(library_dir, ignore_patterns) - dir_file_count += 1 - self.library.included_files.add(f) + start_time = time() + last_update = time() - if not self.library.has_entry_with_path(f): - self.files_not_in_library.append(f) + paths = set() + for raw_path in raw_paths: + paths.add(Path(raw_path)) + if (time() - last_update) >= 0.033: + last_update = time() + yield len(paths) - end_time_total = time() - yield dir_file_count logger.info( - "[Refresh]: Directory scan time", - path=library_dir, - duration=(end_time_total - start_time_total), - files_scanned=dir_file_count, - tool_used="ripgrep (system)", + "[Refresh]: Library scan time", + duration=(time() - start_time), ) - - def __wc_add(self, library_dir: Path, ignore_patterns: list[str]) -> Iterator[int]: - start_time_total = time() - start_time_loop = time() - dir_file_count = 0 - self.files_not_in_library = [] - + yield len(paths) + + self.__add(library_dir, paths) + + def __rg(self, rg_path: str, library_dir: Path, ignore_patterns: list[str]) -> Iterator[str]: + """Use ripgrep to return a list of matched directories and files.""" + logger.info("[Refresh: Using ripgrep for scanning]") + + compiled_ignore_path = library_dir / ".TagStudio" / ".compiled_ignore" + + # Write compiled ignore patterns (built-in + user) to a temp file to pass to ripgrep + with open(compiled_ignore_path, "w") as pattern_file: + pattern_file.write("\n".join(ignore_patterns)) + + with silent_popen( + " ".join( + [ + f'"{rg_path}"', + "--files", + "--follow", + "--hidden", + "--ignore-file", + f'"{str(compiled_ignore_path)}"', + ] + ), + cwd=library_dir, + shell=True, + stdout=subprocess.PIPE, + text=True, + encoding="utf-8", + ) as proc: + assert proc.stdout + for line in proc.stdout: + yield line[:-1] # exclude newline + + compiled_ignore_path.unlink() + + def __wc(self, library_dir: Path, ignore_patterns: list[str]) -> Iterator[str]: logger.info("[Refresh]: Falling back to wcmatch for scanning") - try: - for f in pathlib.Path(str(library_dir)).glob( - "***/*", flags=PATH_GLOB_FLAGS, exclude=ignore_patterns - ): - end_time_loop = time() - # Yield output every 1/30 of a second - if (end_time_loop - start_time_loop) > 0.034: - yield dir_file_count - start_time_loop = time() - - # Skip if the file/path is already mapped in the Library - if f in self.library.included_files: - dir_file_count += 1 - continue - - # Ignore if the file is a directory - if f.is_dir(): - continue + ignore_patterns = ignore_to_glob(ignore_patterns) + return glob.iglob( + "***/*", root_dir=library_dir, flags=PATH_GLOB_FLAGS, exclude=ignore_patterns + ) - dir_file_count += 1 - self.library.included_files.add(f) + def __add(self, library_dir: Path, paths: set[Path]): + start_time_total = time() - relative_path = f.relative_to(library_dir) + self._new_paths = paths.difference(self._expected_paths) - if not self.library.has_entry_with_path(relative_path): - self.files_not_in_library.append(relative_path) - except ValueError: - logger.info("[Refresh]: ValueError when refreshing directory with wcmatch!") + missing = self._expected_paths.difference(paths) + self._missing_paths = dict((p, self._paths_to_id[p]) for p in missing) end_time_total = time() - yield dir_file_count logger.info( - "[Refresh]: Directory scan time", + "[Refresh]: Calculate new/missing paths", path=library_dir, duration=(end_time_total - start_time_total), - files_scanned=dir_file_count, - tool_used="wcmatch (internal)", + files_scanned=len(paths), + missing=len(self._missing_paths), + new=len(self._new_paths), ) diff --git a/src/tagstudio/qt/controllers/library_scanner_controller.py b/src/tagstudio/qt/controllers/library_scanner_controller.py new file mode 100644 index 000000000..1cb178bba --- /dev/null +++ b/src/tagstudio/qt/controllers/library_scanner_controller.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: (c) TagStudio Contributors +# SPDX-License-Identifier: GPL-3.0-only + + +from pathlib import Path +from typing import TYPE_CHECKING + +from PySide6.QtCore import QThreadPool +from PySide6.QtWidgets import QWidget + +from tagstudio.core.library.refresh import RefreshTracker +from tagstudio.core.utils.types import unwrap +from tagstudio.qt.mixed.progress_bar import ProgressWidget +from tagstudio.qt.mixed.unlinked_entries_modal import UnlinkedEntriesModal +from tagstudio.qt.translations import Translations +from tagstudio.qt.utils.custom_runnable import CustomRunnable +from tagstudio.qt.utils.function_iterator import FunctionIterator + +if TYPE_CHECKING: + from tagstudio.core.library.alchemy.library import Library + from tagstudio.qt.ts_qt import QtDriver + + +class LibraryScannerController(QWidget): + def __init__(self, driver: "QtDriver", lib: "Library"): + super().__init__() + self.driver = driver + self.lib = lib + self.tracker = RefreshTracker(lib) + self.unlinked_modal = UnlinkedEntriesModal(self.driver, self) + + @property + def unlinked_entries_count(self) -> int: + return self.tracker.missing_files_count + + @property + def new_files_count(self) -> int: + return self.tracker.new_files_count + + @property + def unlinked_paths(self) -> list[Path]: + return list(self.tracker._missing_paths.keys()) + + def _progress_bar(self, pw: ProgressWidget, iterator, on_update, on_finish): + pw.show() + iterator = FunctionIterator(iterator) + iterator.value.connect(on_update) + r = CustomRunnable(iterator.run) + r.done.connect(lambda: (pw.hide(), pw.deleteLater(), on_finish())) + QThreadPool.globalInstance().start(r) + + def scan(self, on_finish=None): + pw = ScanProgressWidget() + + def on_finish_(): + self.driver.on_library_scan() + + if on_finish is not None: + on_finish() + elif self.tracker.missing_files_count > 0: + self.open_unlinked_view() + else: + self.save_new_files() + + library_dir = unwrap(self.lib.library_dir) + self._progress_bar( + pw, + iterator=lambda: self.tracker.refresh_dir(library_dir), + on_update=lambda i: pw.on_update(i), + on_finish=on_finish_, + ) + + def open_unlinked_view(self): + self.unlinked_modal.show() + + def fix_unlinked_entries(self): + self.tracker.fix_unlinked_entries() + self.driver.on_library_scan() + + def remove_unlinked_entries(self): + self.tracker.remove_unlinked_entries() + self.driver.on_library_scan() + + def save_new_files(self): + files_to_save = self.tracker.new_files_count + if files_to_save == 0: + return + + def on_finish(): + self.driver.on_library_scan() + self.driver.update_browsing_state() + + pw = SaveNewProgressWidget(files_to_save) + self._progress_bar( + pw, + iterator=self.tracker.save_new_files, + on_update=lambda i: pw.on_update(i), + on_finish=on_finish, + ) + + +class ScanProgressWidget(ProgressWidget): + def __init__(self): + super().__init__( + cancel_button_text=None, + minimum=0, + maximum=0, + window_title=Translations["library.refresh.title"], + label_text=Translations["library.refresh.scanning_preparing"], + ) + + def on_update(self, files_searched: int): + self.update_label( + Translations.format( + "library.refresh.scanning.singular" + if files_searched == 1 + else "library.refresh.scanning.plural", + searched_count=f"{files_searched:n}", + found_count=0, # New files are found after scan in single step so no progress + ) + ) + + +class SaveNewProgressWidget(ProgressWidget): + def __init__(self, files_to_save: int): + super().__init__( + cancel_button_text=None, + minimum=0, + maximum=files_to_save, + window_title=Translations["entries.running.dialog.title"], + label_text=Translations.format("library.refresh.scanning_preparing", total=0), + ) + + def on_update(self, files_saved: int): + self.update_progress(files_saved) + self.update_label( + Translations.format("entries.running.dialog.new_entries", total=f"{files_saved:n}") + ) diff --git a/src/tagstudio/qt/mixed/fix_unlinked.py b/src/tagstudio/qt/mixed/fix_unlinked.py deleted file mode 100644 index f96bf2d08..000000000 --- a/src/tagstudio/qt/mixed/fix_unlinked.py +++ /dev/null @@ -1,165 +0,0 @@ -# SPDX-FileCopyrightText: (c) TagStudio Contributors -# SPDX-License-Identifier: GPL-3.0-only - - -from typing import TYPE_CHECKING, override - -from PySide6 import QtCore, QtGui -from PySide6.QtCore import Qt -from PySide6.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget - -from tagstudio.core.library.alchemy.library import Library -from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry -from tagstudio.qt.mixed.merge_dupe_entries import MergeDuplicateEntries -from tagstudio.qt.mixed.progress_bar import ProgressWidget -from tagstudio.qt.mixed.relink_entries_modal import RelinkUnlinkedEntries -from tagstudio.qt.mixed.remove_unlinked_modal import RemoveUnlinkedEntriesModal -from tagstudio.qt.translations import Translations -from tagstudio.qt.views.stylesheets.stylesheets import header - -# Only import for type checking/autocompletion, will not be imported at runtime. -if TYPE_CHECKING: - from tagstudio.qt.ts_qt import QtDriver - - -# TODO: Break up into MVC classes, similar to fix_ignored_modal -class FixUnlinkedEntriesModal(QWidget): - def __init__(self, library: "Library", driver: "QtDriver"): - super().__init__() - self.lib = library - self.driver = driver - - self.tracker = UnlinkedRegistry(lib=self.lib) - - self.unlinked_count = -1 - self.dupe_count = -1 - self.setWindowTitle(Translations["entries.unlinked.title"]) - self.setWindowModality(Qt.WindowModality.ApplicationModal) - self.setMinimumSize(400, 300) - self.root_layout = QVBoxLayout(self) - self.root_layout.setContentsMargins(6, 6, 6, 6) - - self.unlinked_desc_widget = QLabel(Translations["entries.unlinked.description"]) - self.unlinked_desc_widget.setObjectName("unlinkedDescriptionLabel") - self.unlinked_desc_widget.setWordWrap(True) - self.unlinked_desc_widget.setStyleSheet("text-align:left;") - - self.unlinked_count_label = QLabel() - self.unlinked_count_label.setObjectName("unlinkedCountLabel") - self.unlinked_count_label.setAlignment(Qt.AlignmentFlag.AlignCenter) - - self.dupe_count_label = QLabel() - self.dupe_count_label.setObjectName("dupeCountLabel") - self.dupe_count_label.setAlignment(Qt.AlignmentFlag.AlignCenter) - - self.refresh_unlinked_button = QPushButton(Translations["entries.generic.refresh_alt"]) - self.refresh_unlinked_button.clicked.connect(self.refresh_unlinked) - - self.merge_class = MergeDuplicateEntries(self.lib, self.driver) - self.relink_class = RelinkUnlinkedEntries(self.tracker) - - self.search_button = QPushButton(Translations["entries.unlinked.search_and_relink"]) - self.relink_class.done.connect( - # refresh the grid - lambda: ( - self.driver.update_browsing_state(), - self.refresh_unlinked(), - ) - ) - self.search_button.clicked.connect(self.relink_class.repair_entries) - - self.manual_button = QPushButton(Translations["entries.unlinked.relink.manual"]) - self.manual_button.setHidden(True) - - self.remove_button = QPushButton(Translations["entries.unlinked.remove_alt"]) - self.remove_modal = RemoveUnlinkedEntriesModal(self.driver, self.tracker) - self.remove_modal.done.connect( - lambda: ( - self.set_unlinked_count(), - # refresh the grid - self.driver.update_browsing_state(), - self.refresh_unlinked(), - ) - ) - self.remove_button.clicked.connect(self.remove_modal.show) - - self.button_container = QWidget() - self.button_layout = QHBoxLayout(self.button_container) - self.button_layout.setContentsMargins(6, 6, 6, 6) - self.button_layout.addStretch(1) - - self.done_button = QPushButton(Translations["generic.done_alt"]) - self.done_button.setDefault(True) - self.done_button.clicked.connect(self.hide) - self.button_layout.addWidget(self.done_button) - - self.root_layout.addWidget(self.unlinked_count_label) - self.root_layout.addWidget(self.unlinked_desc_widget) - self.root_layout.addWidget(self.refresh_unlinked_button) - self.root_layout.addWidget(self.search_button) - self.root_layout.addWidget(self.manual_button) - self.root_layout.addWidget(self.remove_button) - self.root_layout.addStretch(1) - self.root_layout.addStretch(2) - self.root_layout.addWidget(self.button_container) - - self.update_unlinked_count() - - def refresh_unlinked(self): - pw = ProgressWidget( - cancel_button_text=None, - minimum=0, - maximum=self.lib.entries_count, - ) - pw.setWindowTitle(Translations["library.scan_library.title"]) - pw.update_label(Translations["entries.unlinked.scanning"]) - - def update_driver_widgets(): - if ( - hasattr(self.driver, "library_info_window") - and self.driver.library_info_window.isVisible() - ): - self.driver.library_info_window.update_cleanup() - - pw.from_iterable_function( - self.tracker.refresh_unlinked_files, - None, - self.set_unlinked_count, - self.update_unlinked_count, - self.remove_modal.refresh_list, - update_driver_widgets, - ) - - def set_unlinked_count(self): - """Sets the unlinked_entries_count in the Library to the tracker's value.""" - self.lib.unlinked_entries_count = self.tracker.unlinked_entries_count - - def update_unlinked_count(self): - """Updates the UI to reflect the Library's current unlinked_entries_count.""" - # Indicates that the library is new compared to the last update. - # NOTE: Make sure set_unlinked_count() is called before this! - if self.tracker.unlinked_entries_count > 0 and self.lib.unlinked_entries_count < 0: - self.tracker.reset() - - count: int = self.lib.unlinked_entries_count - - self.search_button.setDisabled(count < 1) - self.remove_button.setDisabled(count < 1) - - count_text: str = Translations.format( - "entries.unlinked.unlinked_count", count=count if count >= 0 else "—" - ) - self.unlinked_count_label.setText(header(count_text, 3)) - - @override - def showEvent(self, event: QtGui.QShowEvent) -> None: - self.update_unlinked_count() - return super().showEvent(event) - - @override - def keyPressEvent(self, event: QtGui.QKeyEvent) -> None: # noqa N802 - if event.key() == QtCore.Qt.Key.Key_Escape: - self.done_button.click() - else: # Other key presses - pass - return super().keyPressEvent(event) diff --git a/src/tagstudio/qt/mixed/relink_entries_modal.py b/src/tagstudio/qt/mixed/relink_entries_modal.py deleted file mode 100644 index 8acd7c1bb..000000000 --- a/src/tagstudio/qt/mixed/relink_entries_modal.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: (c) TagStudio Contributors -# SPDX-License-Identifier: GPL-3.0-only - - -from PySide6.QtCore import QObject, Signal - -from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry -from tagstudio.qt.mixed.progress_bar import ProgressWidget -from tagstudio.qt.translations import Translations - - -class RelinkUnlinkedEntries(QObject): - done = Signal() - - def __init__(self, tracker: UnlinkedRegistry): - super().__init__() - self.tracker = tracker - - def repair_entries(self): - def displayed_text(x): - return Translations.format( - "entries.unlinked.relink.attempting", - index=x, - unlinked_count=self.tracker.unlinked_entries_count, - fixed_count=self.tracker.files_fixed_count, - ) - - pw = ProgressWidget( - label_text="", - cancel_button_text=None, - minimum=0, - maximum=self.tracker.unlinked_entries_count, - ) - pw.setWindowTitle(Translations["entries.unlinked.relink.title"]) - pw.from_iterable_function(self.tracker.fix_unlinked_entries, displayed_text, self.done.emit) diff --git a/src/tagstudio/qt/mixed/remove_unlinked_modal.py b/src/tagstudio/qt/mixed/remove_unlinked_modal.py index 0b4e6b6b0..8e7c5257d 100644 --- a/src/tagstudio/qt/mixed/remove_unlinked_modal.py +++ b/src/tagstudio/qt/mixed/remove_unlinked_modal.py @@ -16,23 +16,23 @@ QWidget, ) -from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry from tagstudio.qt.mixed.progress_bar import ProgressWidget from tagstudio.qt.translations import Translations from tagstudio.qt.utils.custom_runnable import CustomRunnable # Only import for type checking/autocompletion, will not be imported at runtime. if TYPE_CHECKING: + from tagstudio.qt.controllers.library_scanner_controller import LibraryScannerController from tagstudio.qt.ts_qt import QtDriver class RemoveUnlinkedEntriesModal(QWidget): done = Signal() - def __init__(self, driver: "QtDriver", tracker: UnlinkedRegistry): + def __init__(self, driver: "QtDriver", scanner: "LibraryScannerController"): super().__init__() self.driver = driver - self.tracker = tracker + self.scanner = scanner self.setWindowTitle(Translations["entries.unlinked.remove"]) self.setWindowModality(Qt.WindowModality.ApplicationModal) self.setMinimumSize(500, 400) @@ -42,7 +42,7 @@ def __init__(self, driver: "QtDriver", tracker: UnlinkedRegistry): self.desc_widget = QLabel( Translations.format( "entries.remove.plural.confirm", - count=self.tracker.unlinked_entries_count, + count=self.scanner.unlinked_entries_count, ) ) self.desc_widget.setObjectName("descriptionLabel") @@ -75,13 +75,14 @@ def __init__(self, driver: "QtDriver", tracker: UnlinkedRegistry): def refresh_list(self): self.desc_widget.setText( Translations.format( - "entries.remove.plural.confirm", count=self.tracker.unlinked_entries_count + "entries.remove.plural.confirm", count=self.scanner.unlinked_entries_count ) ) self.model.clear() - for i in self.tracker.unlinked_entries: - item = QStandardItem(str(i.path)) + unlinked = sorted(self.scanner.unlinked_paths) + for path in unlinked: + item = QStandardItem(str(path)) item.setEditable(False) self.model.appendRow(item) @@ -94,12 +95,12 @@ def remove_entries(self): pw.setWindowTitle(Translations["entries.generic.remove.removing"]) pw.update_label( Translations.format( - "entries.generic.remove.removing_count", count=self.tracker.unlinked_entries_count + "entries.generic.remove.removing_count", count=self.scanner.unlinked_entries_count ) ) pw.show() - r = CustomRunnable(self.tracker.remove_unlinked_entries) + r = CustomRunnable(self.scanner.remove_unlinked_entries) QThreadPool.globalInstance().start(r) r.done.connect( lambda: ( diff --git a/src/tagstudio/qt/mixed/unlinked_entries_modal.py b/src/tagstudio/qt/mixed/unlinked_entries_modal.py new file mode 100644 index 000000000..b8d161cf9 --- /dev/null +++ b/src/tagstudio/qt/mixed/unlinked_entries_modal.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: (c) TagStudio Contributors +# SPDX-License-Identifier: GPL-3.0-only + + +from typing import TYPE_CHECKING, override + +from PySide6 import QtCore, QtGui +from PySide6.QtCore import Qt +from PySide6.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget + +from tagstudio.qt.mixed.remove_unlinked_modal import RemoveUnlinkedEntriesModal +from tagstudio.qt.translations import Translations +from tagstudio.qt.views.stylesheets.stylesheets import header + +if TYPE_CHECKING: + from tagstudio.qt.controllers.library_scanner_controller import LibraryScannerController + from tagstudio.qt.ts_qt import QtDriver + + +class UnlinkedEntriesModal(QWidget): + def __init__(self, driver: "QtDriver", scanner: "LibraryScannerController"): + super().__init__() + self.driver = driver + self.scanner = scanner + self.setWindowTitle(Translations["entries.unlinked.title"]) + self.setWindowModality(Qt.WindowModality.ApplicationModal) + self.setMinimumSize(400, 300) + self.root_layout = QVBoxLayout(self) + self.root_layout.setContentsMargins(6, 6, 6, 6) + + self.unlinked_desc_widget = QLabel(Translations["entries.unlinked.description"]) + self.unlinked_desc_widget.setObjectName("unlinkedDescriptionLabel") + self.unlinked_desc_widget.setWordWrap(True) + self.unlinked_desc_widget.setStyleSheet("text-align:left;") + + self.unlinked_count_label = QLabel() + self.unlinked_count_label.setObjectName("unlinkedCountLabel") + self.unlinked_count_label.setAlignment(Qt.AlignmentFlag.AlignCenter) + + self.refresh_unlinked_button = QPushButton(Translations["entries.generic.refresh_alt"]) + self.refresh_unlinked_button.clicked.connect(self._on_refresh) + + self.auto_relink_button = QPushButton(Translations["entries.unlinked.search_and_relink"]) + self.auto_relink_button.clicked.connect(self._on_auto_relink) + + self.remove_button = QPushButton(Translations["entries.unlinked.remove_alt"]) + self.remove_modal = RemoveUnlinkedEntriesModal(driver, scanner) + self.remove_button.clicked.connect(self._on_remove) + self.remove_modal.done.connect(self.update_unlinked_count) + + self.button_container = QWidget() + self.button_layout = QHBoxLayout(self.button_container) + self.button_layout.setContentsMargins(6, 6, 6, 6) + self.button_layout.addStretch(1) + + self.done_button = QPushButton(Translations["generic.done_alt"]) + self.done_button.setDefault(True) + self.done_button.clicked.connect(self._on_done) + self.button_layout.addWidget(self.done_button) + + self.root_layout.addWidget(self.unlinked_count_label) + self.root_layout.addWidget(self.unlinked_desc_widget) + self.root_layout.addWidget(self.refresh_unlinked_button) + self.root_layout.addWidget(self.auto_relink_button) + self.root_layout.addWidget(self.remove_button) + self.root_layout.addStretch(1) + self.root_layout.addStretch(2) + self.root_layout.addWidget(self.button_container) + + def update_unlinked_count(self): + count = self.scanner.unlinked_entries_count + + self.auto_relink_button.setDisabled(count < 1) + self.remove_button.setDisabled(count < 1) + + count_text: str = Translations.format( + "entries.unlinked.unlinked_count", count=count if count >= 0 else "—" + ) + self.unlinked_count_label.setText(header(count_text, 3)) + + def _on_refresh(self): + self.scanner.scan(on_finish=self.update_unlinked_count) + + def _on_auto_relink(self): + self.scanner.fix_unlinked_entries() + self.update_unlinked_count() + + def _on_remove(self): + self.remove_modal.refresh_list() + self.remove_modal.show() + + def _on_done(self): + self.hide() + if self.scanner.new_files_count: + self.scanner.save_new_files() + else: + self.driver.update_browsing_state() + + @override + def showEvent(self, event: QtGui.QShowEvent) -> None: + self.update_unlinked_count() + return super().showEvent(event) + + @override + def keyPressEvent(self, event: QtGui.QKeyEvent) -> None: # noqa N802 + if event.key() == QtCore.Qt.Key.Key_Escape: + self._on_done() + else: + return super().keyPressEvent(event) diff --git a/src/tagstudio/qt/ts_qt.py b/src/tagstudio/qt/ts_qt.py index f18ff1acb..220ecdefc 100644 --- a/src/tagstudio/qt/ts_qt.py +++ b/src/tagstudio/qt/ts_qt.py @@ -25,7 +25,7 @@ import structlog from humanfriendly import format_size, format_timespan # pyright: ignore[reportUnknownVariableType] -from PySide6.QtCore import QObject, QSettings, Qt, QThread, QThreadPool, QTimer, Signal +from PySide6.QtCore import QObject, QSettings, Qt, QThread, QTimer, Signal from PySide6.QtGui import ( QColor, QDragEnterEvent, @@ -47,7 +47,6 @@ from tagstudio.core.library.alchemy.library import Library, LibraryStatus from tagstudio.core.library.alchemy.models import Entry from tagstudio.core.library.ignore import Ignore -from tagstudio.core.library.refresh import RefreshTracker from tagstudio.core.media_types import MediaCategories from tagstudio.core.query_lang.util import ParsingError from tagstudio.core.ts_core import TagStudioCore @@ -63,6 +62,7 @@ from tagstudio.qt.controllers.fix_ignored_modal_controller import FixIgnoredEntriesModal from tagstudio.qt.controllers.ignore_modal_controller import IgnoreModal from tagstudio.qt.controllers.library_info_window_controller import LibraryInfoWindow +from tagstudio.qt.controllers.library_scanner_controller import LibraryScannerController from tagstudio.qt.controllers.tag_search_panel_controller import TagSearchModal from tagstudio.qt.controllers.update_available_message_box import UpdateAvailableMessageBox from tagstudio.qt.global_settings import DEFAULT_GLOBAL_SETTINGS_PATH, GlobalSettings, Theme @@ -70,20 +70,16 @@ from tagstudio.qt.mixed.build_tag import BuildTagPanel from tagstudio.qt.mixed.drop_import_modal import DropImportModal from tagstudio.qt.mixed.fix_dupe_files import FixDupeFilesModal -from tagstudio.qt.mixed.fix_unlinked import FixUnlinkedEntriesModal from tagstudio.qt.mixed.folders_to_tags import FoldersToTagsModal from tagstudio.qt.mixed.item_thumb import BadgeType from tagstudio.qt.mixed.migration_modal import JsonMigrationModal -from tagstudio.qt.mixed.progress_bar import ProgressWidget from tagstudio.qt.mixed.settings_panel import SettingsPanel from tagstudio.qt.mixed.tag_color_manager import TagColorManager from tagstudio.qt.models.palette import ColorType, UiColor, get_ui_color from tagstudio.qt.platform_strings import trash_term from tagstudio.qt.resource_manager import ResourceManager from tagstudio.qt.translations import Translations -from tagstudio.qt.utils.custom_runnable import CustomRunnable from tagstudio.qt.utils.file_deleter import delete_file -from tagstudio.qt.utils.function_iterator import FunctionIterator from tagstudio.qt.views.field_template_search_panel_view import FieldTemplateSearchPanelView from tagstudio.qt.views.main_window import MainWindow from tagstudio.qt.views.panel_modal import PanelModal @@ -177,7 +173,6 @@ class QtDriver(DriverMixin, QObject): add_field_modal: PanelModal | None = None folders_modal: FoldersToTagsModal about_modal: AboutModal - unlinked_modal: FixUnlinkedEntriesModal ignored_modal: FixIgnoredEntriesModal dupe_modal: FixDupeFilesModal library_info_window: LibraryInfoWindow @@ -185,7 +180,8 @@ class QtDriver(DriverMixin, QObject): applied_theme: Theme lib: Library - cache_manager: CacheManager | None + cache_manager: CacheManager | None = None + library_scanner: LibraryScannerController | None = None browsing_history: History[BrowsingState] @@ -532,13 +528,8 @@ def on_increase_thumbnail_size_action(): # region Tools Menu =========================================================== - def create_fix_unlinked_entries_modal(): - if not hasattr(self, "unlinked_modal"): - self.unlinked_modal = FixUnlinkedEntriesModal(self.lib, self) - self.unlinked_modal.show() - self.main_window.menu_bar.fix_unlinked_entries_action.triggered.connect( - create_fix_unlinked_entries_modal + lambda: unwrap(self.library_scanner).open_unlinked_view() ) def create_ignored_entries_modal(): @@ -779,6 +770,7 @@ def close_library(self, is_shutdown: bool = False): self.lib.close() self.cache_manager = None + self.library_scanner = None self.thumb_job_queue.queue.clear() if is_shutdown: @@ -1039,82 +1031,17 @@ def delete_file_confirmation(self, count: int, filename: Path | None = None) -> return msg.exec() + def on_library_scan(self): + scanner = unwrap(self.library_scanner) + unlinked_entries = scanner.unlinked_entries_count + self.lib.unlinked_entries_count = unlinked_entries + if hasattr(self, "library_info_window") and self.library_info_window.isVisible(): + self.library_info_window.update_cleanup() + def add_new_files_callback(self): """Run when user initiates adding new files to the Library.""" - tracker = RefreshTracker(self.lib) - - pw = ProgressWidget( - cancel_button_text=None, - minimum=0, - maximum=0, - ) - pw.setWindowTitle(Translations["library.refresh.title"]) - pw.update_label(Translations["library.refresh.scanning_preparing"]) - pw.show() - - iterator = FunctionIterator(lambda lib=self.lib.library_dir: tracker.refresh_dir(lib)) - iterator.value.connect( - lambda x: ( - pw.update_progress(x + 1), - pw.update_label( - Translations.format( - "library.refresh.scanning.plural" - if x + 1 != 1 - else "library.refresh.scanning.singular", - searched_count=f"{x + 1:n}", - found_count=f"{tracker.files_count:n}", - ) - ), - ) - ) - r = CustomRunnable(iterator.run) - r.done.connect( - lambda: ( - pw.hide(), - pw.deleteLater(), - self.add_new_files_runnable(tracker), - ) - ) - QThreadPool.globalInstance().start(r) - - def add_new_files_runnable(self, tracker: RefreshTracker): - """Adds any known new files to the library and run default macros on them. - - Threaded method. - """ - files_count = tracker.files_count - - iterator = FunctionIterator(tracker.save_new_files) - pw = ProgressWidget( - cancel_button_text=None, - minimum=0, - maximum=0, - ) - pw.setWindowTitle(Translations["entries.running.dialog.title"]) - pw.update_label( - Translations.format("entries.running.dialog.new_entries", total=f"{files_count:n}") - ) - pw.show() - - iterator.value.connect( - lambda _count: ( - pw.update_label( - Translations.format( - "entries.running.dialog.new_entries", total=f"{files_count:n}" - ) - ), - ) - ) - r = CustomRunnable(iterator.run) - r.done.connect( - lambda: ( - pw.hide(), - pw.deleteLater(), - # refresh the library only when new items are added - files_count and self.update_browsing_state(), - ) - ) - QThreadPool.globalInstance().start(r) + if scanner := self.library_scanner: + scanner.scan() def new_file_macros_runnable(self, new_ids): """Threaded method that runs macros on a set of Entry IDs.""" @@ -1622,6 +1549,7 @@ def open_library(self, path: Path) -> None: logger.info( f"[Config] Thumbnail Cache Size: {format_size(cache_size)}", ) + self.library_scanner = LibraryScannerController(self, self.lib) # Migration is required if open_status.json_migration_req: diff --git a/tests/conftest.py b/tests/conftest.py index 28b1d5777..a82baf673 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -162,7 +162,7 @@ class Args: open = library_dir ci = True - with patch("tagstudio.qt.ts_qt.Consumer"), patch("tagstudio.qt.ts_qt.CustomRunnable"): + with patch("tagstudio.qt.ts_qt.Consumer"): driver = QtDriver(Args()) # pyright: ignore[reportArgumentType] driver.app = Mock() diff --git a/tests/macros/test_missing_files.py b/tests/macros/test_missing_files.py index 3ea463a65..a8d9e2bb9 100644 --- a/tests/macros/test_missing_files.py +++ b/tests/macros/test_missing_files.py @@ -9,7 +9,7 @@ from tagstudio.core.library.alchemy.enums import BrowsingState from tagstudio.core.library.alchemy.library import Library -from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry +from tagstudio.core.library.refresh import RefreshTracker from tagstudio.core.utils.types import unwrap CWD = Path(__file__).parent @@ -18,19 +18,24 @@ # NOTE: Does this test actually work? @pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True) def test_refresh_missing_files(library: Library): - registry = UnlinkedRegistry(lib=library) + library_dir = unwrap(library.library_dir) + tracker = RefreshTracker(library) # touch the file `one/two/bar.md` but in wrong location to simulate a moved file - (unwrap(library.library_dir) / "bar.md").touch() + (library_dir / "bar.md").touch() # no files actually exist, so it should return all entries - assert list(registry.refresh_unlinked_files()) == [0, 1] + list(tracker.refresh_dir(library_dir, force_internal_tools=True)) + assert sorted(tracker._missing_paths.values()) == [1, 2] # neither of the library entries exist - assert len(registry.unlinked_entries) == 2 + assert tracker.missing_files_count == 2 # iterate through two files - assert list(registry.fix_unlinked_entries()) == [0, 1] + bar = Path("one/two/bar.md") + assert bar in tracker._missing_paths + tracker.fix_unlinked_entries() + assert bar not in tracker._missing_paths # `bar.md` should be relinked to new correct path results = library.search_library(BrowsingState.from_path("bar.md"), page_size=500) diff --git a/tests/macros/test_refresh_dir.py b/tests/macros/test_refresh_dir.py index 01578aad8..1e6961302 100644 --- a/tests/macros/test_refresh_dir.py +++ b/tests/macros/test_refresh_dir.py @@ -20,22 +20,20 @@ def test_refresh_new_files(library: Library, exclude_mode: bool): library_dir = unwrap(library.library_dir) # Given - registry = RefreshTracker(library=library) - library.included_files.clear() + tracker = RefreshTracker(library=library) (library_dir / "FOO.MD").touch() (library_dir / IGNORE_NAME).write_text("*.md" if exclude_mode else "*\n!*.md") # Test if the single file was added - list(registry.refresh_dir(library_dir, force_internal_tools=True)) - assert set(registry.files_not_in_library) == set([Path(IGNORE_NAME), Path("FOO.MD")]) + list(tracker.refresh_dir(library_dir, force_internal_tools=True)) + assert set(tracker._new_paths) == set([Path(IGNORE_NAME), Path("FOO.MD")]) @pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True) def test_refresh_multi_byte_filenames(library: Library): library_dir = unwrap(library.library_dir) # Given - registry = RefreshTracker(library=library) - library.included_files.clear() + tracker = RefreshTracker(library=library) (library_dir / ".TagStudio").mkdir() (library_dir / "こんにちは.txt").touch() (library_dir / "em–dash.txt").touch() @@ -43,8 +41,8 @@ def test_refresh_multi_byte_filenames(library: Library): (library_dir / "umlaute äöü.txt").touch() # Test if all files were added with their correct names and without exceptions - list(registry.refresh_dir(library_dir)) - assert Path("こんにちは.txt") in registry.files_not_in_library - assert Path("em–dash.txt") in registry.files_not_in_library - assert Path("apostrophe’.txt") in registry.files_not_in_library - assert Path("umlaute äöü.txt") in registry.files_not_in_library + list(tracker.refresh_dir(library_dir)) + assert Path("こんにちは.txt") in tracker._new_paths + assert Path("em–dash.txt") in tracker._new_paths + assert Path("apostrophe’.txt") in tracker._new_paths + assert Path("umlaute äöü.txt") in tracker._new_paths