From 80073c0bee5b8e1de26db44717021c989dfa4ab2 Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 16:47:09 -0700 Subject: [PATCH] chore: shrink Python footprint after Rust migration (refs #59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project is now Rust-first; this pass cuts the Python dep list from 8 to 4 runtime deps and drops modules whose work the Rust crates now own. Removed modules - src/fastled/paths.py — HERE/PROJECT_ROOT constants, zero importers. - src/fastled/filewatcher.py — file watching is now owned by the Rust CLI. Comments in compile_native.py already say so. The only inbound reference was the unit test below. - tests/unit/test_filechanger.py — cascading; only consumer of FileWatcherProcess. Dropped dependencies (pyproject.toml) - watchdog — only used by filewatcher.py. - watchfiles — declared but never imported anywhere. - disklru — never imported. - zccache — never imported. - progress — replaced by a stdlib itertools.cycle-based spinner. - rapidfuzz — replaced by difflib.SequenceMatcher with a token-sort wrapper that matches the previous fuzz.token_sort_ratio shape. All 23 string-diff unit tests pass (including the 600-line comprehensive suite). Dropped from requirements.testing.txt - fastapi — declared but never imported anywhere in the repo. Kept (still live, with non-trivial callers) - httpx — 6 callsites for downloads and local-server probes. - fasteners — InterProcessLock for installer concurrency in 2 places. - clang-tool-chain — emscripten installer integration. - zcmds_win32 — Windows-only git-bash bundle for debug symbol resolution. Linters and the full unit-test suite are green after this change (118 passed, 1 skipped). --- pyproject.toml | 6 - requirements.testing.txt | 1 - src/fastled/filewatcher.py | 307 --------------------------------- src/fastled/paths.py | 4 - src/fastled/spinner.py | 17 +- src/fastled/string_diff.py | 12 +- tests/unit/test_filechanger.py | 57 ------ 7 files changed, 21 insertions(+), 383 deletions(-) delete mode 100644 src/fastled/filewatcher.py delete mode 100644 src/fastled/paths.py delete mode 100644 tests/unit/test_filechanger.py diff --git a/pyproject.toml b/pyproject.toml index 55280049..4c2dc481 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,15 +13,9 @@ license = { text = "BSD 3-Clause License" } classifiers = ["Programming Language :: Python :: 3"] dependencies = [ "httpx>=0.28.1", - "watchdog>=6.0.0", - "watchfiles>=1.0.5", - "disklru>=2.0.4", "fasteners>=0.20", "clang-tool-chain>=1.2.6", - "rapidfuzz", - "progress", "zcmds_win32; sys_platform == 'win32'", - "zccache>=1.1.20", ] [tool.maturin] diff --git a/requirements.testing.txt b/requirements.testing.txt index 24331405..25cb7d74 100644 --- a/requirements.testing.txt +++ b/requirements.testing.txt @@ -6,5 +6,4 @@ ruff pytest-xdist pyright httpx -fastapi playwright diff --git a/src/fastled/filewatcher.py b/src/fastled/filewatcher.py deleted file mode 100644 index 56c3b84c..00000000 --- a/src/fastled/filewatcher.py +++ /dev/null @@ -1,307 +0,0 @@ -"""File system watcher implementation using watchdog""" - -import hashlib -import os -import queue -import threading -import time -from contextlib import redirect_stdout -from multiprocessing import Process, Queue -from pathlib import Path -from queue import Empty -from typing import Dict, Set - -from watchdog.events import FileSystemEvent, FileSystemEventHandler -from watchdog.observers import Observer -from watchdog.observers.api import BaseObserver - -from fastled.interrupts import handle_keyboard_interrupt -from fastled.settings import FILE_CHANGED_DEBOUNCE_SECONDS - -_WATCHER_TIMEOUT = 0.1 - - -def file_watcher_enabled() -> bool: - """Check if watchdog is enabled""" - return os.getenv("NO_FILE_WATCHING", "0") != "1" - - -def file_watcher_set(enabled: bool) -> None: - """Set the file watcher enabled state""" - os.environ["NO_FILE_WATCHING"] = "1" if not enabled else "0" - - -class MyEventHandler(FileSystemEventHandler): - def __init__( - self, - change_queue: queue.Queue, - excluded_patterns: Set[str], - file_hashes: Dict[str, str], - ) -> None: - super().__init__() - self.change_queue = change_queue - self.excluded_patterns = excluded_patterns - self.file_hashes = file_hashes - - def _get_file_hash(self, filepath: str) -> str: - try: - with open(filepath, "rb") as f: - return hashlib.md5(f.read()).hexdigest() - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - raise - except Exception: # pylint: disable=broad-except - return "" - - def on_modified(self, event: FileSystemEvent) -> None: - if not event.is_directory: - # Convert src_path to str if it's bytes - src_path = ( - event.src_path.decode() - if isinstance(event.src_path, bytes) - else event.src_path - ) - path = Path(src_path) - # Check if any part of the path matches excluded patterns - if not any(part in self.excluded_patterns for part in path.parts): - new_hash = self._get_file_hash(src_path) - if new_hash and new_hash != self.file_hashes.get(src_path): - self.file_hashes[src_path] = new_hash - self.change_queue.put(src_path) - - -class FileChangedNotifier(threading.Thread): - """Watches a directory for file changes and queues notifications""" - - def __init__( - self, - path: str, - debounce_seconds: float = FILE_CHANGED_DEBOUNCE_SECONDS, - excluded_patterns: list[str] | None = None, - ) -> None: - """Initialize the notifier with a path to watch - - Args: - path: Directory path to watch for changes - debounce_seconds: Minimum time between notifications for the same file - excluded_patterns: List of directory/file patterns to exclude from watching - """ - super().__init__(daemon=True) - self.path = path - self.observer: BaseObserver | None = None - self.event_handler: MyEventHandler | None = None - - # Combine default and user-provided patterns - self.excluded_patterns = ( - set(excluded_patterns) if excluded_patterns is not None else set() - ) - self.stopped = False - self.change_queue: queue.Queue = queue.Queue() - self.last_notification: Dict[str, float] = {} - self.file_hashes: Dict[str, str] = {} - self.debounce_seconds = debounce_seconds - - def stop(self) -> None: - """Stop watching for changes""" - print("watcher stop") - self.stopped = True - if self.observer: - self.observer.stop() - self.observer.join() - self.observer = None - self.event_handler = None - - def run(self) -> None: - """Thread main loop - starts watching for changes""" - self.event_handler = MyEventHandler( - self.change_queue, self.excluded_patterns, self.file_hashes - ) - self.observer = Observer() - self.observer.schedule(self.event_handler, self.path, recursive=True) - self.observer.start() - - try: - while not self.stopped: - time.sleep(0.1) - except KeyboardInterrupt as ki: - print("File watcher stopped by user.") - handle_keyboard_interrupt(ki) - finally: - self.stop() - - def get_next_change(self, timeout: float = _WATCHER_TIMEOUT) -> str | None: - """Get the next file change event from the queue - - Args: - timeout: How long to wait for next change in seconds - - Returns: - Changed filepath or None if no change within timeout - """ - if not file_watcher_enabled(): - time.sleep(timeout) - return None - try: - filepath = self.change_queue.get(timeout=timeout) - current_time = time.time() - - # Check if we've seen this file recently - last_time = self.last_notification.get(filepath, 0) - if current_time - last_time < self.debounce_seconds: - return None - - self.last_notification[filepath] = current_time - return filepath - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - except queue.Empty: - return None - - def get_all_changes(self, timeout: float = _WATCHER_TIMEOUT) -> list[str]: - """Get all file change events from the queue - - Args: - timeout: How long to wait for next change in seconds - - Returns: - List of changed filepaths - """ - changed_files = [] - while True: - changed_file = self.get_next_change(timeout=timeout) - if changed_file is None: - break - changed_files.append(changed_file) - # clear all the changes from the queue - self.change_queue.queue.clear() - return changed_files - - -def _process_wrapper(root: Path, excluded_patterns: list[str], queue: Queue): - with open(os.devnull, "w") as fnull: # Redirect to /dev/null - with redirect_stdout(fnull): - watcher = FileChangedNotifier( - str(root), excluded_patterns=excluded_patterns - ) - watcher.start() - while True: - try: - changed_files = watcher.get_all_changes() - for file in changed_files: - queue.put(file) - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - break - watcher.stop() - - -class ProcessWraperTask: - def __init__(self, root: Path, excluded_patterns: list[str], queue: Queue) -> None: - self.root = root - self.excluded_patterns = excluded_patterns - self.queue = queue - - def run(self): - _process_wrapper(self.root, self.excluded_patterns, self.queue) - - -class FileWatcherProcess: - def __init__(self, root: Path, excluded_patterns: list[str]) -> None: - self.queue: Queue = Queue() - task = ProcessWraperTask(root, excluded_patterns, self.queue) - self.process = Process( - target=task.run, - daemon=True, - ) - self.process.start() - self.global_debounce = FILE_CHANGED_DEBOUNCE_SECONDS - - def stop(self): - self.process.terminate() - self.process.join() - self.queue.close() - self.queue.join_thread() - - def get_all_changes(self, timeout: float | None = None) -> list[str]: - changed_files = [] - block = timeout is not None - - while True: - try: - changed_file = self.queue.get(block=block, timeout=timeout) - changed_files.append(changed_file) - except Empty: - break - return changed_files - - -class DebouncedFileWatcherProcess(threading.Thread): - """ - Wraps a FileWatcherProcess to batch rapid-fire change events - and only emit them once the debounce interval has passed. - Runs in its own thread, polling every 0.1 s. - """ - - def __init__( - self, - watcher: FileWatcherProcess, - debounce_seconds: float = FILE_CHANGED_DEBOUNCE_SECONDS, - ) -> None: - super().__init__(daemon=True) - self.watcher = watcher - self.debounce_seconds = debounce_seconds - - # internal buffer & timing - self._buffer: list[str] = [] - self._last_time: float = 0.0 - - # queue of flushed batches - self._out_queue: queue.Queue[list[str]] = queue.Queue() - self._stop_event = threading.Event() - - # record timestamp of last event for external inspection - self.last_event_time: float | None = None - - self.start() - - def run(self) -> None: - try: - while not self._stop_event.is_set(): - now = time.time() - # non-blocking poll of raw watcher events - new = self.watcher.get_all_changes(timeout=0) - if new: - self._buffer.extend(new) - self._last_time = now - self.last_event_time = now - - # if buffer exists and debounce interval elapsed, flush it - if self._buffer and (now - self._last_time) > self.debounce_seconds: - batch = sorted(set(self._buffer)) - self._buffer.clear() - self._last_time = now - self._out_queue.put(batch) - - time.sleep(0.1) - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - - def get_all_changes(self, timeout: float | None = None) -> list[str]: - """ - Return the next debounced batch of paths, or [] if none arrives - within `timeout` seconds. - """ - try: - if timeout is None: - # non-blocking - return self._out_queue.get_nowait() - else: - return self._out_queue.get(timeout=timeout) - except queue.Empty: - return [] - - def stop(self) -> None: - """Stop the polling thread and tear down the underlying watcher.""" - self._stop_event.set() - self.join() - self.watcher.stop() diff --git a/src/fastled/paths.py b/src/fastled/paths.py deleted file mode 100644 index 2743d988..00000000 --- a/src/fastled/paths.py +++ /dev/null @@ -1,4 +0,0 @@ -from pathlib import Path - -HERE = Path(__file__).resolve().parent -PROJECT_ROOT = HERE.parent.parent diff --git a/src/fastled/spinner.py b/src/fastled/spinner.py index 927f63e2..88ce99db 100644 --- a/src/fastled/spinner.py +++ b/src/fastled/spinner.py @@ -1,23 +1,30 @@ import _thread +import itertools +import sys import threading import time import warnings -from progress.spinner import Spinner as SpinnerImpl - class Spinner: + _FRAMES = "|/-\\" + def __init__(self, message: str = ""): - self.spinner = SpinnerImpl(message) + self.message = message self.event = threading.Event() self.thread = threading.Thread(target=self._spin, daemon=True) self.thread.start() def _spin(self) -> None: try: - while not self.event.is_set(): - self.spinner.next() + for frame in itertools.cycle(self._FRAMES): + if self.event.is_set(): + break + sys.stderr.write(f"\r{self.message} {frame}") + sys.stderr.flush() time.sleep(0.1) + sys.stderr.write("\r" + " " * (len(self.message) + 2) + "\r") + sys.stderr.flush() except KeyboardInterrupt: _thread.interrupt_main() except Exception as e: diff --git a/src/fastled/string_diff.py b/src/fastled/string_diff.py index f0a99990..46ad850e 100644 --- a/src/fastled/string_diff.py +++ b/src/fastled/string_diff.py @@ -1,6 +1,12 @@ +from difflib import SequenceMatcher from pathlib import Path -from rapidfuzz import fuzz + +def _token_sort_ratio(a: str, b: str) -> float: + """Return a 0..100 similarity score after sorting whitespace-split tokens.""" + a_sorted = " ".join(sorted(a.split())) + b_sorted = " ".join(sorted(b.split())) + return SequenceMatcher(None, a_sorted, b_sorted).ratio() * 100.0 def _filter_out_obvious_bad_choices( @@ -207,7 +213,7 @@ def normalize(s: str) -> str: # Calculate distances distances: list[float] = [] for s in string_list: - dist = fuzz.token_sort_ratio(normalize(input_string), normalize(s)) + dist = _token_sort_ratio(normalize(input_string), normalize(s)) distances.append(1.0 / (dist + 1.0)) # Handle case where no strings remain after filtering @@ -219,7 +225,7 @@ def normalize(s: str) -> str: distances = [] for s in string_list: - dist = fuzz.token_sort_ratio(normalize(input_string), normalize(s)) + dist = _token_sort_ratio(normalize(input_string), normalize(s)) distances.append(1.0 / (dist + 1.0)) min_distance = min(distances) diff --git a/tests/unit/test_filechanger.py b/tests/unit/test_filechanger.py deleted file mode 100644 index 2fc8130b..00000000 --- a/tests/unit/test_filechanger.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Unit test for file change notification. -""" - -import os -import tempfile -import time -import unittest -from pathlib import Path - -from fastled.filewatcher import FileWatcherProcess -from fastled.interrupts import handle_keyboard_interrupt - - -class FileChangeProcessTester(unittest.TestCase): - """Tests for process-based file watcher.""" - - def test_process_file_change_detection(self) -> None: - """Test that file changes are detected in a separate process.""" - # Create a temporary directory - with tempfile.TemporaryDirectory() as temp_dir: - # Create a test file - test_file = Path(temp_dir) / "test.txt" - test_file.write_text("initial content") - - # Start watching the directory in a separate process - proc = FileWatcherProcess(Path(temp_dir), []) - time.sleep(0.5) # Give the watcher time to start - - try: - # Make a change to the file - # time.sleep(2) # Give the watcher time to start - test_file.write_text("new content") - - # Wait for and verify the change - # changed_file = queue.get(timeout=5) - changed_files = proc.get_all_changes(timeout=5) - # self.assertIsNotNone(changed_file, "No file change detected") - self.assertNotEqual(len(changed_files), 0, "No file change detected") - self.assertEqual( - os.path.basename(changed_files[0]), os.path.basename(str(test_file)) - ) - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - except Exception as e: - type_str_of_exception = str(type(e)) - print(f"Got exception: {type_str_of_exception}") - if "queue.Empty" in type_str_of_exception: - self.fail("No file change detected") - print(f"Got exception: {e}") - raise - finally: - proc.stop() - - -if __name__ == "__main__": - unittest.main()