diff --git a/.env.beta b/.env.beta index 9f551226..4aef655d 100644 --- a/.env.beta +++ b/.env.beta @@ -2,6 +2,6 @@ SCIDK_CHANNEL=beta # Optional explicit overrides (channel already applies these if unset): # SCIDK_PROVIDERS=local_fs,mounted_fs,rclone -# SCIDK_RCLONE_MOUNTS=1 # SCIDK_FILES_VIEWER=rocrate # SCIDK_FEATURE_FILE_INDEX=1 +# Note: rclone mounts are now core functionality and always enabled diff --git a/.env.dev b/.env.dev index 8f2b6ef8..10b765f5 100644 --- a/.env.dev +++ b/.env.dev @@ -2,6 +2,6 @@ SCIDK_CHANNEL=dev # Optional explicit overrides (channel already applies these if unset): # SCIDK_PROVIDERS=local_fs,mounted_fs,rclone -# SCIDK_RCLONE_MOUNTS=1 # SCIDK_FILES_VIEWER=rocrate # SCIDK_FEATURE_FILE_INDEX=1 +# Note: rclone mounts are now core functionality and always enabled diff --git a/.env.stable b/.env.stable index 40059b23..cc8b8d60 100644 --- a/.env.stable +++ b/.env.stable @@ -2,6 +2,6 @@ SCIDK_CHANNEL=stable # In stable, advanced features are off unless explicitly set by operator. # SCIDK_PROVIDERS=local_fs,mounted_fs -# SCIDK_RCLONE_MOUNTS=0 # SCIDK_FILES_VIEWER=classic # SCIDK_FEATURE_FILE_INDEX=0 +# Note: rclone mounts are now core functionality and always enabled diff --git a/Makefile b/Makefile index 68328f27..b518bfeb 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,28 @@ # Convenience Makefile for docs/tools -.PHONY: flags-index docs-check unit integration check e2e-install-browsers e2e e2e-headed e2e-parallel e2e-debug +.PHONY: flags-index docs-check unit integration check e2e-install-browsers e2e e2e-headed e2e-parallel e2e-debug clean-test-artifacts flags-index: python -m dev.tools.feature_flags_index --write +# Clean up test artifacts (pytest sessions, cache, temp files) +# Keeps the 3 most recent pytest sessions +clean-test-artifacts: + @echo "Cleaning test artifacts..." + @# Remove pytest cache + @rm -rf .pytest_cache + @# Remove Python bytecode + @find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + @find . -type f -name "*.pyc" -delete 2>/dev/null || true + @# Keep last 3 pytest sessions, remove older ones + @if [ -d "dev/test-runs/tmp/pytest-of-patch" ]; then \ + cd dev/test-runs/tmp/pytest-of-patch && \ + ls -t | grep '^pytest-[0-9]*$$' | tail -n +4 | xargs -r rm -rf ; \ + fi + @# Remove playwright reports + @rm -rf playwright-report test-results + @echo "✓ Test artifacts cleaned (kept last 3 pytest sessions)" + # docs-check: run generator and diff; non-zero exit if mismatched # Note: this target assumes Unix tools (diff) docs-check: diff --git a/README.md b/README.md index 33a99c58..309e5c1d 100644 --- a/README.md +++ b/README.md @@ -146,11 +146,11 @@ Rclone provider (optional): - If rclone is not installed or a remote is misconfigured, API returns a clear error message with HTTP 500 and {"error": "..."}. - Optional FUSE mount flow with safe defaults: see docs/rclone/mount-examples.md and dev/ops/rclone/systemd/rclone-mount@.service. -### Rclone Mount Manager (MVP, feature-flagged) -- Enable the feature: set `SCIDK_RCLONE_MOUNTS=1` (or `SCIDK_FEATURE_RCLONE_MOUNTS=1`). When enabled, the rclone provider is auto-enabled for remote validation even if not listed in `SCIDK_PROVIDERS`. -- UI: Settings → Rclone Mounts section appears. Create a mount by entering `remote`, optional `subpath`, a `name`, and submit (read-only by default). +### Rclone Mount Manager (MVP) +- The rclone provider is always enabled for remote validation. +- UI: Settings → Rclone Mounts section. Create a mount by entering `remote`, optional `subpath`, a `name`, and submit (read-only by default). - Safety: Mountpoints are restricted under `./data/mounts/`; remotes are validated against `rclone listremotes` output. -- Endpoints (enabled only when the feature flag is set): +- Endpoints: - GET `/api/rclone/mounts` — list managed mounts - POST `/api/rclone/mounts` with JSON `{ remote, subpath, name, read_only }` — starts `rclone mount` targeting `./data/mounts/` - DELETE `/api/rclone/mounts/` — unmounts and stops the process diff --git a/dev b/dev index ae891d1a..2dac9d46 160000 --- a/dev +++ b/dev @@ -1 +1 @@ -Subproject commit ae891d1abbac862f5985fcaa63162bd483436efd +Subproject commit 2dac9d46136179f2f0d14bb9795f162f76cb2884 diff --git a/scidk/app.py b/scidk/app.py index c82aafa9..a5e0b33d 100644 --- a/scidk/app.py +++ b/scidk/app.py @@ -1,68 +1,47 @@ -from flask import Flask, Blueprint, jsonify, request, render_template, redirect, url_for +"""SciDK Flask application factory. + +This module provides the create_app() function that initializes the Flask application +with all necessary extensions, services, and route blueprints. + +Most initialization logic has been extracted to separate modules in scidk/core/ +and scidk/services/ to keep this file lean and maintainable. +""" + +from flask import Flask from pathlib import Path import os -from typing import Optional -import time -import json -from .core.graph import InMemoryGraph +# Core components from .core.filesystem import FilesystemManager from .core.registry import InterpreterRegistry from .interpreters import register_all as register_interpreters -from .core.providers import ProviderRegistry as FsProviderRegistry, LocalFSProvider, MountedFSProvider, RcloneProvider -from .web.helpers import commit_to_neo4j_batched - - -def _apply_channel_defaults(): - """Apply channel-based defaults for feature flags when unset. - Channels: stable (default), dev, beta. - Explicit env values always win; we only set defaults if unset. - Also soft-disable rclone provider by removing it from SCIDK_PROVIDERS if rclone binary is missing, - unless SCIDK_FORCE_RCLONE is truthy. Only perform soft-disable when SCIDK_PROVIDERS was not explicitly set by user. - """ - import shutil - ch = (os.environ.get('SCIDK_CHANNEL') or 'stable').strip().lower() - had_prov_env = 'SCIDK_PROVIDERS' in os.environ - def setdefault_env(name: str, value: str): - if os.environ.get(name) is None: - os.environ[name] = value - if ch in ('dev', 'beta'): - # Providers default: include rclone - if os.environ.get('SCIDK_PROVIDERS') is None: - os.environ['SCIDK_PROVIDERS'] = 'local_fs,mounted_fs,rclone' - # Mounts UI - setdefault_env('SCIDK_RCLONE_MOUNTS', '1') - # Files viewer mode - setdefault_env('SCIDK_FILES_VIEWER', 'rocrate') - # File index work in progress - setdefault_env('SCIDK_FEATURE_FILE_INDEX', '1') - # Soft rclone detection: remove if missing and not forced, but only when we set providers implicitly - if not had_prov_env: - prov_env = os.environ.get('SCIDK_PROVIDERS') - if prov_env: - prov_list = [p.strip() for p in prov_env.split(',') if p.strip()] - if 'rclone' in prov_list and not shutil.which('rclone'): - force = (os.environ.get('SCIDK_FORCE_RCLONE') or '').strip().lower() in ('1','true','yes','y','on') - if not force: - prov_list = [p for p in prov_list if p != 'rclone'] - os.environ['SCIDK_PROVIDERS'] = ','.join(prov_list) - # Record effective channel for UI/debug - os.environ.setdefault('SCIDK_CHANNEL', ch or 'stable') - # Default: commit to graph should read from index unless explicitly disabled - if os.environ.get('SCIDK_COMMIT_FROM_INDEX') is None: - os.environ['SCIDK_COMMIT_FROM_INDEX'] = '1' +# Initialization modules (extracted from app.py) +from .core.channel_config import apply_channel_defaults +from .core.neo4j_config import create_graph_backend +from .core.interpreter_enablement import compute_enabled_interpreters +from .core.providers_init import initialize_fs_providers +from .core.telemetry_loader import load_last_scan_from_sqlite +from .core.rclone_settings import load_rclone_interpretation_settings +from .core.rclone_mounts_loader import rehydrate_rclone_mounts def create_app(): + """Create and configure the Flask application. + + Returns: + Flask: Configured Flask application instance with scidk extensions + """ # Apply channel-based defaults before reading env-driven config - _apply_channel_defaults() + apply_channel_defaults() + app = Flask(__name__, template_folder="ui/templates", static_folder="ui/static") + # Feature: selective dry-run UI flag (dev default) try: ch = (os.environ.get('SCIDK_CHANNEL') or 'stable').strip().lower() flag_env = (os.environ.get('SCIDK_FEATURE_SELECTIVE_DRYRUN') or '').strip().lower() - flag = flag_env in ('1','true','yes','y','on') + flag = flag_env in ('1', 'true', 'yes', 'y', 'on') if flag_env == '' and ch == 'dev': flag = True app.config['feature.selectiveDryRun'] = bool(flag) @@ -73,7 +52,7 @@ def create_app(): try: from .core import migrations as _migs _migs.migrate() - except Exception as _e: + except Exception: # Defer reporting to /api/health if needed via app.extensions pass @@ -86,117 +65,31 @@ def create_app(): state_backend = 'sqlite' app.config['state.backend'] = state_backend - # Core singletons (select backend) - backend = (os.environ.get('SCIDK_GRAPH_BACKEND') or 'memory').strip().lower() - if backend == 'neo4j': - try: - uri, user, pwd, database, auth_mode = _get_neo4j_params() - from .core.neo4j_graph import Neo4jGraph - auth = None if auth_mode == 'none' else (user, pwd) - graph = Neo4jGraph(uri=uri, auth=auth, database=database) - except Exception: - # Fallback to in-memory if neo4j params invalid - from .core.graph import InMemoryGraph as _IMG - graph = _IMG() - else: - graph = InMemoryGraph() - registry = InterpreterRegistry() - # Load persisted interpreter toggle settings (optional) - try: - from .core.settings import InterpreterSettings - settings = InterpreterSettings(os.environ.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')) - enabled = settings.load_enabled_interpreters() - if enabled: - registry.enabled_interpreters = set(enabled) - except Exception: - settings = None + # Core singletons: graph backend (Neo4j or InMemory) + graph = create_graph_backend(app) - # Register interpreters with extensions and rules + # Interpreter registry + registry = InterpreterRegistry() register_interpreters(registry) - # Compute effective interpreter enablement (CLI envs > global settings > defaults) - testing_env = bool(os.environ.get('PYTEST_CURRENT_TEST')) or bool(os.environ.get('SCIDK_DISABLE_SETTINGS')) - try: - from .core.settings import InterpreterSettings - settings = None if testing_env else InterpreterSettings(db_path=str(Path(os.getcwd()) / 'scidk_settings.db')) - except Exception: - settings = None - # Defaults from interpreter attributes (fallback True) - all_ids = list(registry.by_id.keys()) - default_enabled_ids = set([iid for iid in all_ids if bool(getattr(registry.by_id[iid], 'default_enabled', True))]) - # CLI overrides via env - # CLI overrides via env (case-insensitive); ignore unknown ids to avoid surprises - en_raw = [s.strip() for s in (os.environ.get('SCIDK_ENABLE_INTERPRETERS') or '').split(',') if s.strip()] - dis_raw = [s.strip() for s in (os.environ.get('SCIDK_DISABLE_INTERPRETERS') or '').split(',') if s.strip()] - # Normalize to lowercase (registry ids are lowercase) - en_list = [s.lower() for s in en_raw] - dis_list = [s.lower() for s in dis_raw] - source = 'default' - if en_list or dis_list: - known_ids = set(all_ids) - unknown_en = [x for x in en_list if x not in known_ids] - unknown_dis = [x for x in dis_list if x not in known_ids] - # Start from defaults; remove DISABLE; add ENABLE; ENABLE wins on conflicts - enabled_set = set(default_enabled_ids) - for d in dis_list: - if d in known_ids: - enabled_set.discard(d) - for e in en_list: - if e in known_ids: - enabled_set.add(e) - source = 'cli' - # Do NOT persist CLI-derived sets to settings to avoid masking user intentions - try: - _ist = app.extensions.setdefault('scidk', {}).setdefault('interpreters', {}) - _ist['unknown_env'] = {'enable': unknown_en, 'disable': unknown_dis} - except Exception: - pass - else: - # Load global saved set if any - loaded = set() - try: - if settings: - loaded = set(settings.load_enabled_interpreters()) - except Exception: - loaded = set() - if loaded: - enabled_set = set(loaded) - source = 'global' - else: - enabled_set = set(default_enabled_ids) - source = 'default' - # Store effective on app - _interp_state = {'effective_enabled': enabled_set, 'source': source} - # Apply effective enabled set to registry for selection logic - try: - registry.enabled_interpreters = set(enabled_set) - except Exception: - pass + # Compute effective interpreter enablement (CLI > settings > defaults) + app.extensions = getattr(app, 'extensions', {}) + app.extensions['scidk'] = {} + enabled_set, source, settings = compute_enabled_interpreters(registry, app.extensions) + # FilesystemManager fs = FilesystemManager(graph=graph, registry=registry) - # Initialize filesystem providers (Phase 0) - prov_enabled = [p.strip() for p in (os.environ.get('SCIDK_PROVIDERS', 'local_fs,mounted_fs').split(',')) if p.strip()] - # If rclone mounts feature is enabled, ensure rclone provider is also enabled for listremotes validation - _ff_rc = (os.environ.get('SCIDK_RCLONE_MOUNTS') or os.environ.get('SCIDK_FEATURE_RCLONE_MOUNTS') or '').strip().lower() in ('1','true','yes','y','on') - if _ff_rc and 'rclone' not in prov_enabled: - prov_enabled.append('rclone') - fs_providers = FsProviderRegistry(enabled=prov_enabled) - p_local = LocalFSProvider(); p_local.initialize(app, {}) - p_mounted = MountedFSProvider(); p_mounted.initialize(app, {}) - p_rclone = RcloneProvider(); p_rclone.initialize(app, {}) - fs_providers.register(p_local) - fs_providers.register(p_mounted) - fs_providers.register(p_rclone) + # Initialize filesystem providers (local_fs, mounted_fs, rclone) + fs_providers = initialize_fs_providers(app) - # Store refs on app for easy access - app.extensions = getattr(app, 'extensions', {}) + # Store refs on app for easy access in routes app.extensions['scidk'] = { 'graph': graph, 'registry': registry, 'fs': fs, 'providers': fs_providers, - 'interpreters': _interp_state, + 'interpreters': {'effective_enabled': enabled_set, 'source': source}, # in-session registries 'scans': {}, # scan_id -> scan session dict 'directories': {}, # path -> aggregate info incl. scan_ids @@ -213,394 +106,37 @@ def create_app(): 'connected': False, 'last_error': None, }, - # rclone mounts runtime registry (feature-flagged API will use this) + # rclone mounts runtime registry 'rclone_mounts': {}, # id/name -> { id, remote, subpath, path, read_only, started_at, pid, log_file } 'settings': settings, } - # Hydrate telemetry.last_scan from SQLite settings on startup (best-effort) - try: - from .core import path_index_sqlite as pix - from .core import migrations as _migs - import json as _json - conn = pix.connect() - try: - _migs.migrate(conn) - cur = conn.cursor() - row = cur.execute("SELECT value FROM settings WHERE key = ?", ("telemetry.last_scan",)).fetchone() - if row and row[0]: - try: - last_scan = _json.loads(row[0]) - app.extensions.setdefault('scidk', {}).setdefault('telemetry', {})['last_scan'] = last_scan - except Exception: - pass - finally: - try: - conn.close() - except Exception: - pass - except Exception: - pass + # Hydrate telemetry.last_scan from SQLite settings on startup + last_scan = load_last_scan_from_sqlite() + if last_scan: + app.extensions['scidk']['telemetry']['last_scan'] = last_scan # Hydrate rclone interpretation settings (suggest mount threshold and batch size) - try: - def _env_int(name: str, dflt: int) -> int: - try: - v = os.environ.get(name) - return int(v) if v is not None and v != '' else dflt - except Exception: - return dflt - suggest_dflt = _env_int('SCIDK_RCLONE_INTERPRET_SUGGEST_MOUNT', 400) - max_batch_dflt = _env_int('SCIDK_RCLONE_INTERPRET_MAX_FILES', 1000) - max_batch_dflt = min(max(100, max_batch_dflt), 2000) - from .core import path_index_sqlite as pix - from .core import migrations as _migs - conn = pix.connect() - try: - _migs.migrate(conn) - cur = conn.cursor() - def _get_setting_int(key: str, dflt: int) -> int: - row = cur.execute("SELECT value FROM settings WHERE key= ?", (key,)).fetchone() - if row and row[0] not in (None, ''): - try: - return int(row[0]) - except Exception: - return dflt - return dflt - suggest_mount_threshold = _get_setting_int('rclone.interpret.suggest_mount_threshold', suggest_dflt) - max_files_per_batch = _get_setting_int('rclone.interpret.max_files_per_batch', max_batch_dflt) - max_files_per_batch = min(max(100, int(max_files_per_batch)), 2000) - app.config['rclone.interpret.suggest_mount_threshold'] = int(suggest_mount_threshold) - app.config['rclone.interpret.max_files_per_batch'] = int(max_files_per_batch) - finally: - try: - conn.close() - except Exception: - pass - except Exception: - # Defaults if hydration fails - app.config.setdefault('rclone.interpret.suggest_mount_threshold', 400) - app.config.setdefault('rclone.interpret.max_files_per_batch', 1000) - - # Feature flag for rclone mount manager (define before first use) - def _feature_rclone_mounts() -> bool: - val = (os.environ.get('SCIDK_RCLONE_MOUNTS') or os.environ.get('SCIDK_FEATURE_RCLONE_MOUNTS') or '').strip().lower() - return val in ('1', 'true', 'yes', 'y', 'on') + load_rclone_interpretation_settings(app) # Rehydrate rclone mounts metadata from SQLite on startup (no process attached) - if _feature_rclone_mounts(): - try: - from .core import path_index_sqlite as pix - from .core import migrations as _migs - import json as _json - conn = pix.connect() - try: - _migs.migrate(conn) - cur = conn.cursor() - cur.execute("SELECT id, provider, root, created, status, extra_json FROM provider_mounts WHERE provider='rclone'") - rows = cur.fetchall() or [] - rm = app.extensions['scidk'].setdefault('rclone_mounts', {}) - for (mid, provider, remote, created, status_persisted, extra) in rows: - try: - extra_obj = _json.loads(extra) if extra else {} - except Exception: - extra_obj = {} - rm[mid] = { - 'id': mid, - 'name': mid, - 'remote': remote, - 'subpath': extra_obj.get('subpath'), - 'path': extra_obj.get('path'), - 'read_only': extra_obj.get('read_only'), - 'started_at': created, - 'process': None, - 'pid': None, - 'log_file': extra_obj.get('log_file'), - } - finally: - try: - conn.close() - except Exception: - pass - except Exception: - pass - - # API routes - api = Blueprint('api', __name__, url_prefix='/api') - - # Import SQLite layer for selections/annotations lazily to avoid circular deps - from .core import annotations_sqlite as ann_db - - - # Helper to read Neo4j configuration, preferring in-app settings over environment - # Returns tuple: (uri, user, password, database, auth_mode) - # auth_mode: 'basic' (username+password) or 'none' (no authentication) - def _get_neo4j_params(): - cfg = app.extensions['scidk'].get('neo4j_config', {}) - uri = cfg.get('uri') or os.environ.get('NEO4J_URI') or os.environ.get('BOLT_URI') - user = cfg.get('user') or os.environ.get('NEO4J_USER') or os.environ.get('NEO4J_USERNAME') - pwd = cfg.get('password') or os.environ.get('NEO4J_PASSWORD') - database = cfg.get('database') or os.environ.get('SCIDK_NEO4J_DATABASE') or None - # Parse NEO4J_AUTH env var if provided (formats: "user/pass" or "none") - neo4j_auth = (os.environ.get('NEO4J_AUTH') or '').strip() - if neo4j_auth: - if neo4j_auth.lower() == 'none': - user = user or None - pwd = pwd or None - auth_mode = 'none' - else: - try: - # Expecting user/password - parts = neo4j_auth.split('/') - if len(parts) >= 2 and not (user and pwd): - user = user or parts[0] - pwd = pwd or '/'.join(parts[1:]) - except Exception: - pass - # If user/password still missing, try to parse from URI (bolt://user:pass@host:port) - auth_mode = 'basic' - try: - if uri and (not user or not pwd): - from urllib.parse import urlparse, unquote - parsed = urlparse(uri) - if parsed.username and parsed.password: - user = user or unquote(parsed.username) - pwd = pwd or unquote(parsed.password) - except Exception: - pass - # Determine auth mode: none only when explicitly set via NEO4J_AUTH=none - if (os.environ.get('NEO4J_AUTH') or '').strip().lower() == 'none': - auth_mode = 'none' - else: - auth_mode = 'basic' - return uri, user, pwd, database, auth_mode - - # Build rows for commit: files (rows) and standalone folders (folder_rows) - def build_commit_rows(scan, ds_map): - """Legacy builder from in-memory datasets.""" - try: - from .services.commit_service import CommitService - return CommitService().build_rows_legacy_from_datasets(scan, ds_map) - except Exception: - # Fallback to empty on unexpected import/runtime error - return [], [] - - # Execute Neo4j commit using simplified, idempotent Cypher - def commit_to_neo4j(rows, folder_rows, scan, neo4j_params): - # Support 4-tuple (backward compat) and 5-tuple with auth_mode - try: - uri, user, pwd, database, auth_mode = neo4j_params - except Exception: - uri, user, pwd, database = neo4j_params - auth_mode = 'basic' - result = {'attempted': False, 'written_files': 0, 'written_folders': 0, 'error': None} - if not uri: - return result - # Decide if we can attempt a connection - can_basic = bool(user and pwd) - can_connect = (auth_mode == 'none') or can_basic - if not can_connect: - return result - # Backoff on recent auth failures to avoid rate limiting - st = app.extensions['scidk'].setdefault('neo4j_state', {}) - import time as _t - now = _t.time() - next_after = float(st.get('next_connect_after') or 0) - if next_after and now < next_after: - result['error'] = f"neo4j connect backoff active; retry after {int(next_after-now)}s" - return result - result['attempted'] = True - try: - from .services.neo4j_client import Neo4jClient - client = Neo4jClient(uri, user, pwd, database, auth_mode).connect() - try: - client.ensure_constraints() - wres = client.write_scan(rows, folder_rows, scan) - result['written_files'] = wres.get('written_files', 0) - result['written_folders'] = wres.get('written_folders', 0) - vres = client.verify(scan.get('id')) - result.update(vres) - finally: - client.close() - except Exception as e: - msg = str(e) - result['error'] = msg - # On auth-related errors, set a backoff to avoid rate limiting - try: - emsg = msg.lower() - if ('unauthorized' in emsg) or ('authentication' in emsg): - # Exponential-ish backoff min 20s - prev = float(st.get('next_connect_after') or 0) - base = 20.0 - delay = base - if prev and now < prev: - # increase delay up to 120s - rem = prev - now - delay = min(max(base*2, rem*2), 120.0) - st['next_connect_after'] = now + delay - st['last_error'] = msg - except Exception: - pass - return result - - # Build or fetch per-scan filesystem index for snapshot navigation - def _get_or_build_scan_index(scan_id: str): - cache = app.extensions['scidk'].setdefault('scan_fs', {}) - if scan_id in cache: - return cache[scan_id] - scans = app.extensions['scidk'].get('scans', {}) - s = scans.get(scan_id) - if not s: - return None - checksums = s.get('checksums') or [] - ds_map = app.extensions['scidk']['graph'].datasets # checksum -> dataset - - from .core.path_utils import parse_remote_path, parent_remote_path - from pathlib import Path as _P - - folder_info = {} - children_files = {} - - def ensure_complete_parent_chain(path_str: str): - """Ensure all parent folders exist in folder_info for any given path""" - if not path_str or path_str in folder_info: - return - - info = parse_remote_path(path_str) - if info.get('is_remote'): - parent = parent_remote_path(path_str) - name = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or path_str) - else: - try: - p = _P(path_str) - parent = str(p.parent) - name = p.name or path_str - except Exception: - parent = '' - name = path_str - - folder_info[path_str] = { - 'path': path_str, - 'name': name, - 'parent': parent, - } - - if parent and parent != path_str: - ensure_complete_parent_chain(parent) - - # Seed scan base path (stable roots even on empty scans) - try: - base_path = s.get('path') or '' - if base_path: - ensure_complete_parent_chain(base_path) - except Exception: - pass - - # Process files and ensure their parent chains exist - for ch in checksums: - d = ds_map.get(ch) - if not d: - continue - file_path = d.get('path') - if not file_path: - continue - - info = parse_remote_path(file_path) - if info.get('is_remote'): - parent = parent_remote_path(file_path) - filename = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or file_path) - else: - try: - p = _P(file_path) - parent = str(p.parent) - filename = p.name or file_path - except Exception: - parent = '' - filename = file_path - - file_entry = { - 'id': d.get('id'), - 'path': file_path, - 'filename': d.get('filename') or filename, - 'extension': d.get('extension'), - 'size_bytes': int(d.get('size_bytes') or 0), - 'modified': float(d.get('modified') or 0), - 'mime_type': d.get('mime_type'), - 'checksum': d.get('checksum'), - } - children_files.setdefault(parent, []).append(file_entry) - - if parent: - ensure_complete_parent_chain(parent) - - # Process explicitly recorded folders - for f in (s.get('folders') or []): - path = f.get('path') - if path: - ensure_complete_parent_chain(path) - - # Build children_folders map - children_folders = {} - for fpath, info in folder_info.items(): - par = info.get('parent') - if par and par in folder_info: - children_folders.setdefault(par, []).append(fpath) - - # Find actual roots - roots = sorted([fp for fp, info in folder_info.items() - if not info.get('parent') or info.get('parent') not in folder_info]) - - # Prefer scan base as visible root and drop its ancestors - try: - base_path = s.get('path') or '' - if base_path and base_path in folder_info: - if base_path not in roots: - roots.append(base_path) - def _is_ancestor(candidate: str, child: str) -> bool: - if not candidate or candidate == child: - return False - cinf = parse_remote_path(candidate) - chinf = parse_remote_path(child) - if chinf.get('is_remote') and cinf.get('is_remote'): - return child.startswith(candidate.rstrip('/') + '/') - try: - return str(_P(child)).startswith(str(_P(candidate)) + '/') - except Exception: - return False - roots = [r for r in roots if not _is_ancestor(r, base_path) or r == base_path] - roots = sorted(list(dict.fromkeys(roots))) - except Exception: - pass - - # Sort children deterministically - for k in list(children_folders.keys()): - children_folders[k].sort(key=lambda p: folder_info.get(p, {}).get('name', '').lower()) - for k in list(children_files.keys()): - children_files[k].sort(key=lambda f: (f.get('filename') or '').lower()) - - idx = { - 'folder_info': folder_info, - 'children_folders': children_folders, - 'children_files': children_files, - 'roots': roots, - } - cache[scan_id] = idx - return idx + mounts = rehydrate_rclone_mounts() + app.extensions['scidk']['rclone_mounts'].update(mounts) # Feature flags for file indexing - _ff_index = (os.environ.get('SCIDK_FEATURE_FILE_INDEX') or '').strip().lower() in ('1','true','yes','y','on') - + _ff_index = (os.environ.get('SCIDK_FEATURE_FILE_INDEX') or '').strip().lower() in ( + '1', 'true', 'yes', 'y', 'on' + ) # Register all blueprints from web.routes package from .web.routes import register_blueprints register_blueprints(app) - # Note: Old UI routes (256 lines) have been moved to scidk/web/routes/ui.py - return app def main(): + """Run the Flask development server.""" app = create_app() # Read host/port from env for convenience host = os.environ.get('SCIDK_HOST', '127.0.0.1') diff --git a/scidk/app.py.pre-phase2-refactor b/scidk/app.py.pre-phase2-refactor new file mode 100644 index 00000000..dbfd974e --- /dev/null +++ b/scidk/app.py.pre-phase2-refactor @@ -0,0 +1,625 @@ +from flask import Flask, Blueprint, jsonify, request, render_template, redirect, url_for +from pathlib import Path +import os +from typing import Optional +import time +import json + +from .core.graph import InMemoryGraph +from .core.filesystem import FilesystemManager +from .core.registry import InterpreterRegistry +from .interpreters import register_all as register_interpreters +from .core.providers import ProviderRegistry as FsProviderRegistry, LocalFSProvider, MountedFSProvider, RcloneProvider +from .web.helpers import commit_to_neo4j_batched + + +def _apply_channel_defaults(): + """Apply channel-based defaults for feature flags when unset. + Channels: stable (default), dev, beta. + Explicit env values always win; we only set defaults if unset. + Also soft-disable rclone provider by removing it from SCIDK_PROVIDERS if rclone binary is missing, + unless SCIDK_FORCE_RCLONE is truthy. Only perform soft-disable when SCIDK_PROVIDERS was not explicitly set by user. + """ + import shutil + ch = (os.environ.get('SCIDK_CHANNEL') or 'stable').strip().lower() + had_prov_env = 'SCIDK_PROVIDERS' in os.environ + def setdefault_env(name: str, value: str): + if os.environ.get(name) is None: + os.environ[name] = value + if ch in ('dev', 'beta'): + # Providers default: include rclone + if os.environ.get('SCIDK_PROVIDERS') is None: + os.environ['SCIDK_PROVIDERS'] = 'local_fs,mounted_fs,rclone' + # Mounts UI + setdefault_env('SCIDK_RCLONE_MOUNTS', '1') + # Files viewer mode + setdefault_env('SCIDK_FILES_VIEWER', 'rocrate') + # File index work in progress + setdefault_env('SCIDK_FEATURE_FILE_INDEX', '1') + # Soft rclone detection: remove if missing and not forced, but only when we set providers implicitly + if not had_prov_env: + prov_env = os.environ.get('SCIDK_PROVIDERS') + if prov_env: + prov_list = [p.strip() for p in prov_env.split(',') if p.strip()] + if 'rclone' in prov_list and not shutil.which('rclone'): + force = (os.environ.get('SCIDK_FORCE_RCLONE') or '').strip().lower() in ('1','true','yes','y','on') + if not force: + prov_list = [p for p in prov_list if p != 'rclone'] + os.environ['SCIDK_PROVIDERS'] = ','.join(prov_list) + # Record effective channel for UI/debug + os.environ.setdefault('SCIDK_CHANNEL', ch or 'stable') + # Default: commit to graph should read from index unless explicitly disabled + if os.environ.get('SCIDK_COMMIT_FROM_INDEX') is None: + os.environ['SCIDK_COMMIT_FROM_INDEX'] = '1' + + + +def create_app(): + # Apply channel-based defaults before reading env-driven config + _apply_channel_defaults() + app = Flask(__name__, template_folder="ui/templates", static_folder="ui/static") + # Feature: selective dry-run UI flag (dev default) + try: + ch = (os.environ.get('SCIDK_CHANNEL') or 'stable').strip().lower() + flag_env = (os.environ.get('SCIDK_FEATURE_SELECTIVE_DRYRUN') or '').strip().lower() + flag = flag_env in ('1','true','yes','y','on') + if flag_env == '' and ch == 'dev': + flag = True + app.config['feature.selectiveDryRun'] = bool(flag) + except Exception: + app.config['feature.selectiveDryRun'] = False + + # Auto-migrate SQLite schema on boot (best effort) + try: + from .core import migrations as _migs + _migs.migrate() + except Exception as _e: + # Defer reporting to /api/health if needed via app.extensions + pass + + # State backend toggle (sqlite|memory) for app registries (reads) + try: + state_backend = (os.environ.get('SCIDK_STATE_BACKEND') or 'sqlite').strip().lower() + if state_backend not in ('sqlite', 'memory'): + state_backend = 'sqlite' + except Exception: + state_backend = 'sqlite' + app.config['state.backend'] = state_backend + + # Core singletons (select backend) + backend = (os.environ.get('SCIDK_GRAPH_BACKEND') or 'memory').strip().lower() + if backend == 'neo4j': + try: + uri, user, pwd, database, auth_mode = _get_neo4j_params() + if not uri: + raise ValueError("NEO4J_URI not configured") + if auth_mode != 'none' and (not user or not pwd): + raise ValueError("NEO4J credentials incomplete (NEO4J_USER/NEO4J_PASSWORD required)") + from .core.neo4j_graph import Neo4jGraph + auth = None if auth_mode == 'none' else (user, pwd) + graph = Neo4jGraph(uri=uri, auth=auth, database=database, auth_mode=auth_mode) + app.logger.info(f"Graph backend: neo4j (uri={uri}, database={database})") + except Exception as e: + # Fallback to in-memory if neo4j params invalid + app.logger.warning(f"SCIDK_GRAPH_BACKEND=neo4j but env incomplete or invalid ({e}); falling back to in-memory") + from .core.graph import InMemoryGraph as _IMG + graph = _IMG() + backend = 'memory' + else: + graph = InMemoryGraph() + if backend != 'memory': + app.logger.warning(f"Unknown SCIDK_GRAPH_BACKEND={backend}; using in-memory") + backend = 'memory' + else: + app.logger.info("Graph backend: in-memory") + registry = InterpreterRegistry() + # Load persisted interpreter toggle settings (optional) + try: + from .core.settings import InterpreterSettings + settings = InterpreterSettings(os.environ.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')) + enabled = settings.load_enabled_interpreters() + if enabled: + registry.enabled_interpreters = set(enabled) + except Exception: + settings = None + + # Register interpreters with extensions and rules + register_interpreters(registry) + + # Compute effective interpreter enablement (CLI envs > global settings > defaults) + testing_env = bool(os.environ.get('PYTEST_CURRENT_TEST')) or bool(os.environ.get('SCIDK_DISABLE_SETTINGS')) + try: + from .core.settings import InterpreterSettings + settings = None if testing_env else InterpreterSettings(db_path=str(Path(os.getcwd()) / 'scidk_settings.db')) + except Exception: + settings = None + # Defaults from interpreter attributes (fallback True) + all_ids = list(registry.by_id.keys()) + default_enabled_ids = set([iid for iid in all_ids if bool(getattr(registry.by_id[iid], 'default_enabled', True))]) + # CLI overrides via env + # CLI overrides via env (case-insensitive); ignore unknown ids to avoid surprises + en_raw = [s.strip() for s in (os.environ.get('SCIDK_ENABLE_INTERPRETERS') or '').split(',') if s.strip()] + dis_raw = [s.strip() for s in (os.environ.get('SCIDK_DISABLE_INTERPRETERS') or '').split(',') if s.strip()] + # Normalize to lowercase (registry ids are lowercase) + en_list = [s.lower() for s in en_raw] + dis_list = [s.lower() for s in dis_raw] + source = 'default' + if en_list or dis_list: + known_ids = set(all_ids) + unknown_en = [x for x in en_list if x not in known_ids] + unknown_dis = [x for x in dis_list if x not in known_ids] + # Start from defaults; remove DISABLE; add ENABLE; ENABLE wins on conflicts + enabled_set = set(default_enabled_ids) + for d in dis_list: + if d in known_ids: + enabled_set.discard(d) + for e in en_list: + if e in known_ids: + enabled_set.add(e) + source = 'cli' + # Do NOT persist CLI-derived sets to settings to avoid masking user intentions + try: + _ist = app.extensions.setdefault('scidk', {}).setdefault('interpreters', {}) + _ist['unknown_env'] = {'enable': unknown_en, 'disable': unknown_dis} + except Exception: + pass + else: + # Load global saved set if any + loaded = set() + try: + if settings: + loaded = set(settings.load_enabled_interpreters()) + except Exception: + loaded = set() + if loaded: + enabled_set = set(loaded) + source = 'global' + else: + enabled_set = set(default_enabled_ids) + source = 'default' + # Store effective on app + _interp_state = {'effective_enabled': enabled_set, 'source': source} + # Apply effective enabled set to registry for selection logic + try: + registry.enabled_interpreters = set(enabled_set) + except Exception: + pass + + fs = FilesystemManager(graph=graph, registry=registry) + + # Initialize filesystem providers (Phase 0) + prov_enabled = [p.strip() for p in (os.environ.get('SCIDK_PROVIDERS', 'local_fs,mounted_fs').split(',')) if p.strip()] + # If rclone mounts feature is enabled, ensure rclone provider is also enabled for listremotes validation + _ff_rc = (os.environ.get('SCIDK_RCLONE_MOUNTS') or os.environ.get('SCIDK_FEATURE_RCLONE_MOUNTS') or '').strip().lower() in ('1','true','yes','y','on') + if _ff_rc and 'rclone' not in prov_enabled: + prov_enabled.append('rclone') + fs_providers = FsProviderRegistry(enabled=prov_enabled) + p_local = LocalFSProvider(); p_local.initialize(app, {}) + p_mounted = MountedFSProvider(); p_mounted.initialize(app, {}) + p_rclone = RcloneProvider(); p_rclone.initialize(app, {}) + fs_providers.register(p_local) + fs_providers.register(p_mounted) + fs_providers.register(p_rclone) + + # Store refs on app for easy access + app.extensions = getattr(app, 'extensions', {}) + app.extensions['scidk'] = { + 'graph': graph, + 'registry': registry, + 'fs': fs, + 'providers': fs_providers, + 'interpreters': _interp_state, + # in-session registries + 'scans': {}, # scan_id -> scan session dict + 'directories': {}, # path -> aggregate info incl. scan_ids + 'telemetry': {}, + 'tasks': {}, # task_id -> task dict (background jobs like scans) + 'scan_fs': {}, # per-scan filesystem index cache for snapshot navigation + 'neo4j_config': { + 'uri': None, + 'user': None, + 'password': None, + 'database': None, + }, + 'neo4j_state': { + 'connected': False, + 'last_error': None, + }, + # rclone mounts runtime registry (feature-flagged API will use this) + 'rclone_mounts': {}, # id/name -> { id, remote, subpath, path, read_only, started_at, pid, log_file } + 'settings': settings, + } + + # Hydrate telemetry.last_scan from SQLite settings on startup (best-effort) + try: + from .core import path_index_sqlite as pix + from .core import migrations as _migs + import json as _json + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + row = cur.execute("SELECT value FROM settings WHERE key = ?", ("telemetry.last_scan",)).fetchone() + if row and row[0]: + try: + last_scan = _json.loads(row[0]) + app.extensions.setdefault('scidk', {}).setdefault('telemetry', {})['last_scan'] = last_scan + except Exception: + pass + finally: + try: + conn.close() + except Exception: + pass + except Exception: + pass + + # Hydrate rclone interpretation settings (suggest mount threshold and batch size) + try: + def _env_int(name: str, dflt: int) -> int: + try: + v = os.environ.get(name) + return int(v) if v is not None and v != '' else dflt + except Exception: + return dflt + suggest_dflt = _env_int('SCIDK_RCLONE_INTERPRET_SUGGEST_MOUNT', 400) + max_batch_dflt = _env_int('SCIDK_RCLONE_INTERPRET_MAX_FILES', 1000) + max_batch_dflt = min(max(100, max_batch_dflt), 2000) + from .core import path_index_sqlite as pix + from .core import migrations as _migs + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + def _get_setting_int(key: str, dflt: int) -> int: + row = cur.execute("SELECT value FROM settings WHERE key= ?", (key,)).fetchone() + if row and row[0] not in (None, ''): + try: + return int(row[0]) + except Exception: + return dflt + return dflt + suggest_mount_threshold = _get_setting_int('rclone.interpret.suggest_mount_threshold', suggest_dflt) + max_files_per_batch = _get_setting_int('rclone.interpret.max_files_per_batch', max_batch_dflt) + max_files_per_batch = min(max(100, int(max_files_per_batch)), 2000) + app.config['rclone.interpret.suggest_mount_threshold'] = int(suggest_mount_threshold) + app.config['rclone.interpret.max_files_per_batch'] = int(max_files_per_batch) + finally: + try: + conn.close() + except Exception: + pass + except Exception: + # Defaults if hydration fails + app.config.setdefault('rclone.interpret.suggest_mount_threshold', 400) + app.config.setdefault('rclone.interpret.max_files_per_batch', 1000) + + # Feature flag for rclone mount manager (define before first use) + def _feature_rclone_mounts() -> bool: + val = (os.environ.get('SCIDK_RCLONE_MOUNTS') or os.environ.get('SCIDK_FEATURE_RCLONE_MOUNTS') or '').strip().lower() + return val in ('1', 'true', 'yes', 'y', 'on') + + # Rehydrate rclone mounts metadata from SQLite on startup (no process attached) + if _feature_rclone_mounts(): + try: + from .core import path_index_sqlite as pix + from .core import migrations as _migs + import json as _json + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + cur.execute("SELECT id, provider, root, created, status, extra_json FROM provider_mounts WHERE provider='rclone'") + rows = cur.fetchall() or [] + rm = app.extensions['scidk'].setdefault('rclone_mounts', {}) + for (mid, provider, remote, created, status_persisted, extra) in rows: + try: + extra_obj = _json.loads(extra) if extra else {} + except Exception: + extra_obj = {} + rm[mid] = { + 'id': mid, + 'name': mid, + 'remote': remote, + 'subpath': extra_obj.get('subpath'), + 'path': extra_obj.get('path'), + 'read_only': extra_obj.get('read_only'), + 'started_at': created, + 'process': None, + 'pid': None, + 'log_file': extra_obj.get('log_file'), + } + finally: + try: + conn.close() + except Exception: + pass + except Exception: + pass + + # API routes + api = Blueprint('api', __name__, url_prefix='/api') + + # Import SQLite layer for selections/annotations lazily to avoid circular deps + from .core import annotations_sqlite as ann_db + + + # Helper to read Neo4j configuration, preferring in-app settings over environment + # Returns tuple: (uri, user, password, database, auth_mode) + # auth_mode: 'basic' (username+password) or 'none' (no authentication) + def _get_neo4j_params(): + cfg = app.extensions['scidk'].get('neo4j_config', {}) + uri = cfg.get('uri') or os.environ.get('NEO4J_URI') or os.environ.get('BOLT_URI') + user = cfg.get('user') or os.environ.get('NEO4J_USER') or os.environ.get('NEO4J_USERNAME') + pwd = cfg.get('password') or os.environ.get('NEO4J_PASSWORD') + database = cfg.get('database') or os.environ.get('SCIDK_NEO4J_DATABASE') or None + # Parse NEO4J_AUTH env var if provided (formats: "user/pass" or "none") + neo4j_auth = (os.environ.get('NEO4J_AUTH') or '').strip() + if neo4j_auth: + if neo4j_auth.lower() == 'none': + user = user or None + pwd = pwd or None + auth_mode = 'none' + else: + try: + # Expecting user/password + parts = neo4j_auth.split('/') + if len(parts) >= 2 and not (user and pwd): + user = user or parts[0] + pwd = pwd or '/'.join(parts[1:]) + except Exception: + pass + # If user/password still missing, try to parse from URI (bolt://user:pass@host:port) + auth_mode = 'basic' + try: + if uri and (not user or not pwd): + from urllib.parse import urlparse, unquote + parsed = urlparse(uri) + if parsed.username and parsed.password: + user = user or unquote(parsed.username) + pwd = pwd or unquote(parsed.password) + except Exception: + pass + # Determine auth mode: none only when explicitly set via NEO4J_AUTH=none + if (os.environ.get('NEO4J_AUTH') or '').strip().lower() == 'none': + auth_mode = 'none' + else: + auth_mode = 'basic' + return uri, user, pwd, database, auth_mode + + # Build rows for commit: files (rows) and standalone folders (folder_rows) + def build_commit_rows(scan, ds_map): + """Legacy builder from in-memory datasets.""" + try: + from .services.commit_service import CommitService + return CommitService().build_rows_legacy_from_datasets(scan, ds_map) + except Exception: + # Fallback to empty on unexpected import/runtime error + return [], [] + + # Execute Neo4j commit using simplified, idempotent Cypher + def commit_to_neo4j(rows, folder_rows, scan, neo4j_params): + # Support 4-tuple (backward compat) and 5-tuple with auth_mode + try: + uri, user, pwd, database, auth_mode = neo4j_params + except Exception: + uri, user, pwd, database = neo4j_params + auth_mode = 'basic' + result = {'attempted': False, 'written_files': 0, 'written_folders': 0, 'error': None} + if not uri: + return result + # Decide if we can attempt a connection + can_basic = bool(user and pwd) + can_connect = (auth_mode == 'none') or can_basic + if not can_connect: + return result + # Backoff on recent auth failures to avoid rate limiting + st = app.extensions['scidk'].setdefault('neo4j_state', {}) + import time as _t + now = _t.time() + next_after = float(st.get('next_connect_after') or 0) + if next_after and now < next_after: + result['error'] = f"neo4j connect backoff active; retry after {int(next_after-now)}s" + return result + result['attempted'] = True + try: + from .services.neo4j_client import Neo4jClient + client = Neo4jClient(uri, user, pwd, database, auth_mode).connect() + try: + client.ensure_constraints() + wres = client.write_scan(rows, folder_rows, scan) + result['written_files'] = wres.get('written_files', 0) + result['written_folders'] = wres.get('written_folders', 0) + vres = client.verify(scan.get('id')) + result.update(vres) + finally: + client.close() + except Exception as e: + msg = str(e) + result['error'] = msg + # On auth-related errors, set a backoff to avoid rate limiting + try: + emsg = msg.lower() + if ('unauthorized' in emsg) or ('authentication' in emsg): + # Exponential-ish backoff min 20s + prev = float(st.get('next_connect_after') or 0) + base = 20.0 + delay = base + if prev and now < prev: + # increase delay up to 120s + rem = prev - now + delay = min(max(base*2, rem*2), 120.0) + st['next_connect_after'] = now + delay + st['last_error'] = msg + except Exception: + pass + return result + + # Build or fetch per-scan filesystem index for snapshot navigation + def _get_or_build_scan_index(scan_id: str): + cache = app.extensions['scidk'].setdefault('scan_fs', {}) + if scan_id in cache: + return cache[scan_id] + scans = app.extensions['scidk'].get('scans', {}) + s = scans.get(scan_id) + if not s: + return None + checksums = s.get('checksums') or [] + ds_map = app.extensions['scidk']['graph'].datasets # checksum -> dataset + + from .core.path_utils import parse_remote_path, parent_remote_path + from pathlib import Path as _P + + folder_info = {} + children_files = {} + + def ensure_complete_parent_chain(path_str: str): + """Ensure all parent folders exist in folder_info for any given path""" + if not path_str or path_str in folder_info: + return + + info = parse_remote_path(path_str) + if info.get('is_remote'): + parent = parent_remote_path(path_str) + name = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or path_str) + else: + try: + p = _P(path_str) + parent = str(p.parent) + name = p.name or path_str + except Exception: + parent = '' + name = path_str + + folder_info[path_str] = { + 'path': path_str, + 'name': name, + 'parent': parent, + } + + if parent and parent != path_str: + ensure_complete_parent_chain(parent) + + # Seed scan base path (stable roots even on empty scans) + try: + base_path = s.get('path') or '' + if base_path: + ensure_complete_parent_chain(base_path) + except Exception: + pass + + # Process files and ensure their parent chains exist + for ch in checksums: + d = ds_map.get(ch) + if not d: + continue + file_path = d.get('path') + if not file_path: + continue + + info = parse_remote_path(file_path) + if info.get('is_remote'): + parent = parent_remote_path(file_path) + filename = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or file_path) + else: + try: + p = _P(file_path) + parent = str(p.parent) + filename = p.name or file_path + except Exception: + parent = '' + filename = file_path + + file_entry = { + 'id': d.get('id'), + 'path': file_path, + 'filename': d.get('filename') or filename, + 'extension': d.get('extension'), + 'size_bytes': int(d.get('size_bytes') or 0), + 'modified': float(d.get('modified') or 0), + 'mime_type': d.get('mime_type'), + 'checksum': d.get('checksum'), + } + children_files.setdefault(parent, []).append(file_entry) + + if parent: + ensure_complete_parent_chain(parent) + + # Process explicitly recorded folders + for f in (s.get('folders') or []): + path = f.get('path') + if path: + ensure_complete_parent_chain(path) + + # Build children_folders map + children_folders = {} + for fpath, info in folder_info.items(): + par = info.get('parent') + if par and par in folder_info: + children_folders.setdefault(par, []).append(fpath) + + # Find actual roots + roots = sorted([fp for fp, info in folder_info.items() + if not info.get('parent') or info.get('parent') not in folder_info]) + + # Prefer scan base as visible root and drop its ancestors + try: + base_path = s.get('path') or '' + if base_path and base_path in folder_info: + if base_path not in roots: + roots.append(base_path) + def _is_ancestor(candidate: str, child: str) -> bool: + if not candidate or candidate == child: + return False + cinf = parse_remote_path(candidate) + chinf = parse_remote_path(child) + if chinf.get('is_remote') and cinf.get('is_remote'): + return child.startswith(candidate.rstrip('/') + '/') + try: + return str(_P(child)).startswith(str(_P(candidate)) + '/') + except Exception: + return False + roots = [r for r in roots if not _is_ancestor(r, base_path) or r == base_path] + roots = sorted(list(dict.fromkeys(roots))) + except Exception: + pass + + # Sort children deterministically + for k in list(children_folders.keys()): + children_folders[k].sort(key=lambda p: folder_info.get(p, {}).get('name', '').lower()) + for k in list(children_files.keys()): + children_files[k].sort(key=lambda f: (f.get('filename') or '').lower()) + + idx = { + 'folder_info': folder_info, + 'children_folders': children_folders, + 'children_files': children_files, + 'roots': roots, + } + cache[scan_id] = idx + return idx + + # Feature flags for file indexing + _ff_index = (os.environ.get('SCIDK_FEATURE_FILE_INDEX') or '').strip().lower() in ('1','true','yes','y','on') + + + # Register all blueprints from web.routes package + from .web.routes import register_blueprints + register_blueprints(app) + + # Note: Old UI routes (256 lines) have been moved to scidk/web/routes/ui.py + + return app + + +def main(): + app = create_app() + # Read host/port from env for convenience + host = os.environ.get('SCIDK_HOST', '127.0.0.1') + port = int(os.environ.get('SCIDK_PORT', '5000')) + debug = os.environ.get('SCIDK_DEBUG', '1') == '1' + app.run(host=host, port=port, debug=debug) + + +if __name__ == "__main__": + main() diff --git a/scidk/core/channel_config.py b/scidk/core/channel_config.py new file mode 100644 index 00000000..5043f87e --- /dev/null +++ b/scidk/core/channel_config.py @@ -0,0 +1,63 @@ +"""Channel-based feature flag defaults configuration. + +This module applies channel-based defaults (stable, dev, beta) for feature flags +when environment variables are not explicitly set. It also handles soft-disabling +of rclone provider when the binary is not available. +""" + +import os +import shutil + + +def apply_channel_defaults(): + """Apply channel-based defaults for feature flags when unset. + + Channels: stable (default), dev, beta. + Explicit env values always win; we only set defaults if unset. + + Also soft-disable rclone provider by removing it from SCIDK_PROVIDERS + if rclone binary is missing, unless SCIDK_FORCE_RCLONE is truthy. + Only perform soft-disable when SCIDK_PROVIDERS was not explicitly set by user. + """ + ch = (os.environ.get('SCIDK_CHANNEL') or 'stable').strip().lower() + had_prov_env = 'SCIDK_PROVIDERS' in os.environ + + def setdefault_env(name: str, value: str): + """Set environment variable only if not already set.""" + if os.environ.get(name) is None: + os.environ[name] = value + + if ch in ('dev', 'beta'): + # Providers default: include rclone + if os.environ.get('SCIDK_PROVIDERS') is None: + os.environ['SCIDK_PROVIDERS'] = 'local_fs,mounted_fs,rclone' + + # Mounts UI + setdefault_env('SCIDK_RCLONE_MOUNTS', '1') + + # Files viewer mode + setdefault_env('SCIDK_FILES_VIEWER', 'rocrate') + + # File index work in progress + setdefault_env('SCIDK_FEATURE_FILE_INDEX', '1') + + # Soft rclone detection: remove if missing and not forced, + # but only when we set providers implicitly + if not had_prov_env: + prov_env = os.environ.get('SCIDK_PROVIDERS') + if prov_env: + prov_list = [p.strip() for p in prov_env.split(',') if p.strip()] + if 'rclone' in prov_list and not shutil.which('rclone'): + force = (os.environ.get('SCIDK_FORCE_RCLONE') or '').strip().lower() in ( + '1', 'true', 'yes', 'y', 'on' + ) + if not force: + prov_list = [p for p in prov_list if p != 'rclone'] + os.environ['SCIDK_PROVIDERS'] = ','.join(prov_list) + + # Record effective channel for UI/debug + os.environ.setdefault('SCIDK_CHANNEL', ch or 'stable') + + # Default: commit to graph should read from index unless explicitly disabled + if os.environ.get('SCIDK_COMMIT_FROM_INDEX') is None: + os.environ['SCIDK_COMMIT_FROM_INDEX'] = '1' diff --git a/scidk/core/graph.py b/scidk/core/graph.py index 81696813..d5d39769 100644 --- a/scidk/core/graph.py +++ b/scidk/core/graph.py @@ -282,20 +282,32 @@ def schema_triples(self, limit: int = 500) -> Dict: 'truncated': truncated, } - def commit_scan(self, scan: Dict): + def commit_scan(self, scan: Dict, rows: Optional[List[Dict]] = None, folder_rows: Optional[List[Dict]] = None) -> Dict: """Commit a scan session into the graph as a Scan node and SCANNED_IN edges. Expects scan to contain 'id' and 'checksums'. + + Args: + scan: Scan metadata dict + rows: Optional file rows (not used for InMemoryGraph) + folder_rows: Optional folder rows (not used for InMemoryGraph) + + Returns: + Dict with keys: {'db_scan_exists', 'db_files', 'db_folders', 'db_verified'} """ if not scan or not scan.get('id'): - return + return {'db_scan_exists': False, 'db_verified': False, 'db_files': 0, 'db_folders': 0} sid = scan['id'] # store a shallow copy self.scans[sid] = {k: scan[k] for k in scan.keys()} checksums = scan.get('checksums') or [] + file_count = 0 for ch in checksums: if ch in self.datasets: s = self.dataset_scans.setdefault(ch, set()) s.add(sid) + file_count += 1 + # Return in-memory stats + return {'db_scan_exists': True, 'db_verified': True, 'db_files': file_count, 'db_folders': 0} def delete_scan(self, scan_id: str): """Delete a committed scan node and unlink SCANNED_IN edges. Datasets remain intact.""" diff --git a/scidk/core/interpreter_enablement.py b/scidk/core/interpreter_enablement.py new file mode 100644 index 00000000..b0137df9 --- /dev/null +++ b/scidk/core/interpreter_enablement.py @@ -0,0 +1,103 @@ +"""Interpreter enablement logic for computing which interpreters should be active. + +This module handles the complex precedence rules: +1. CLI environment variables (SCIDK_ENABLE_INTERPRETERS, SCIDK_DISABLE_INTERPRETERS) +2. Global saved settings (from InterpreterSettings database) +3. Interpreter defaults (default_enabled attribute) +""" + +import os +from pathlib import Path +from typing import Set, Tuple, Dict, Any, Optional + + +def compute_enabled_interpreters( + registry, + app_extensions: Dict[str, Any] +) -> Tuple[Set[str], str, Optional[Any]]: + """Compute effective interpreter enablement with CLI > settings > defaults precedence. + + Args: + registry: InterpreterRegistry instance with by_id dict + app_extensions: app.extensions dict for storing unknown_env warnings + + Returns: + tuple: (enabled_set, source, settings_instance) + - enabled_set: Set of interpreter IDs that should be enabled + - source: 'cli' | 'global' | 'default' (where the config came from) + - settings_instance: InterpreterSettings object or None + """ + # Check if we're in testing environment (disable persistent settings) + testing_env = bool(os.environ.get('PYTEST_CURRENT_TEST')) or bool(os.environ.get('SCIDK_DISABLE_SETTINGS')) + + # Load settings instance (unless testing) + settings = None + if not testing_env: + try: + from .settings import InterpreterSettings + settings = InterpreterSettings(db_path=str(Path(os.getcwd()) / 'scidk_settings.db')) + except Exception: + settings = None + + # Compute defaults from interpreter attributes (fallback True) + all_ids = list(registry.by_id.keys()) + default_enabled_ids = set([ + iid for iid in all_ids + if bool(getattr(registry.by_id[iid], 'default_enabled', True)) + ]) + + # Parse CLI overrides (case-insensitive) + en_raw = [s.strip() for s in (os.environ.get('SCIDK_ENABLE_INTERPRETERS') or '').split(',') if s.strip()] + dis_raw = [s.strip() for s in (os.environ.get('SCIDK_DISABLE_INTERPRETERS') or '').split(',') if s.strip()] + en_list = [s.lower() for s in en_raw] + dis_list = [s.lower() for s in dis_raw] + + source = 'default' + + if en_list or dis_list: + # CLI overrides present + known_ids = set(all_ids) + unknown_en = [x for x in en_list if x not in known_ids] + unknown_dis = [x for x in dis_list if x not in known_ids] + + # Start from defaults; remove DISABLE; add ENABLE; ENABLE wins on conflicts + enabled_set = set(default_enabled_ids) + for d in dis_list: + if d in known_ids: + enabled_set.discard(d) + for e in en_list: + if e in known_ids: + enabled_set.add(e) + + source = 'cli' + + # Store unknown IDs for /api/interpreters to warn about + # Do NOT persist CLI-derived sets to settings to avoid masking user intentions + try: + _ist = app_extensions.setdefault('scidk', {}).setdefault('interpreters', {}) + _ist['unknown_env'] = {'enable': unknown_en, 'disable': unknown_dis} + except Exception: + pass + else: + # No CLI overrides; try loading from saved settings + loaded = set() + try: + if settings: + loaded = set(settings.load_enabled_interpreters()) + except Exception: + loaded = set() + + if loaded: + enabled_set = set(loaded) + source = 'global' + else: + enabled_set = set(default_enabled_ids) + source = 'default' + + # Apply to registry + try: + registry.enabled_interpreters = set(enabled_set) + except Exception: + pass + + return enabled_set, source, settings diff --git a/scidk/core/neo4j_config.py b/scidk/core/neo4j_config.py new file mode 100644 index 00000000..797b9bbf --- /dev/null +++ b/scidk/core/neo4j_config.py @@ -0,0 +1,106 @@ +"""Neo4j configuration and backend initialization. + +This module extracts Neo4j parameter parsing and graph backend creation +from app.py to keep application initialization modular. +""" + +import os +from typing import Tuple, Optional + + +def get_neo4j_params(app) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str], str]: + """Read Neo4j configuration, preferring in-app settings over environment. + + Args: + app: Flask application instance with extensions['scidk']['neo4j_config'] + + Returns: + tuple: (uri, user, password, database, auth_mode) + auth_mode: 'basic' (username+password) or 'none' (no authentication) + """ + cfg = app.extensions['scidk'].get('neo4j_config', {}) + uri = cfg.get('uri') or os.environ.get('NEO4J_URI') or os.environ.get('BOLT_URI') + user = cfg.get('user') or os.environ.get('NEO4J_USER') or os.environ.get('NEO4J_USERNAME') + pwd = cfg.get('password') or os.environ.get('NEO4J_PASSWORD') + database = cfg.get('database') or os.environ.get('SCIDK_NEO4J_DATABASE') or None + + # Parse NEO4J_AUTH env var if provided (formats: "user/pass" or "none") + neo4j_auth = (os.environ.get('NEO4J_AUTH') or '').strip() + if neo4j_auth: + if neo4j_auth.lower() == 'none': + user = user or None + pwd = pwd or None + auth_mode = 'none' + else: + try: + # Expecting user/password + parts = neo4j_auth.split('/') + if len(parts) >= 2 and not (user and pwd): + user = user or parts[0] + pwd = pwd or '/'.join(parts[1:]) + except Exception: + pass + + # If user/password still missing, try to parse from URI (bolt://user:pass@host:port) + auth_mode = 'basic' + try: + if uri and (not user or not pwd): + from urllib.parse import urlparse, unquote + parsed = urlparse(uri) + if parsed.username and parsed.password: + user = user or unquote(parsed.username) + pwd = pwd or unquote(parsed.password) + except Exception: + pass + + # Determine auth mode: none only when explicitly set via NEO4J_AUTH=none + if (os.environ.get('NEO4J_AUTH') or '').strip().lower() == 'none': + auth_mode = 'none' + else: + auth_mode = 'basic' + + return uri, user, pwd, database, auth_mode + + +def create_graph_backend(app): + """Create and configure the graph backend (Neo4j or InMemory). + + Args: + app: Flask application instance with logger + + Returns: + Graph backend instance (Neo4jGraph or InMemoryGraph) + Also sets backend name on app for reference + """ + from ..core.graph import InMemoryGraph + + backend = (os.environ.get('SCIDK_GRAPH_BACKEND') or 'memory').strip().lower() + + if backend == 'neo4j': + try: + uri, user, pwd, database, auth_mode = get_neo4j_params(app) + if not uri: + raise ValueError("NEO4J_URI not configured") + if auth_mode != 'none' and (not user or not pwd): + raise ValueError("NEO4J credentials incomplete (NEO4J_USER/NEO4J_PASSWORD required)") + + from ..core.neo4j_graph import Neo4jGraph + auth = None if auth_mode == 'none' else (user, pwd) + graph = Neo4jGraph(uri=uri, auth=auth, database=database, auth_mode=auth_mode) + app.logger.info(f"Graph backend: neo4j (uri={uri}, database={database})") + app.config['graph_backend'] = 'neo4j' + return graph + except Exception as e: + # Fallback to in-memory if neo4j params invalid + app.logger.warning(f"SCIDK_GRAPH_BACKEND=neo4j but env incomplete or invalid ({e}); falling back to in-memory") + graph = InMemoryGraph() + app.config['graph_backend'] = 'memory' + return graph + else: + graph = InMemoryGraph() + if backend != 'memory': + app.logger.warning(f"Unknown SCIDK_GRAPH_BACKEND={backend}; using in-memory") + else: + app.logger.info("Graph backend: in-memory") + app.config['graph_backend'] = 'memory' + return graph diff --git a/scidk/core/neo4j_graph.py b/scidk/core/neo4j_graph.py index e1cda589..9e0793c9 100644 --- a/scidk/core/neo4j_graph.py +++ b/scidk/core/neo4j_graph.py @@ -1,5 +1,8 @@ from typing import Dict, List, Optional from neo4j import GraphDatabase +import logging + +logger = logging.getLogger(__name__) class Neo4jGraph: @@ -7,9 +10,13 @@ class Neo4jGraph: Implements the subset of methods used by the web layer to avoid heavy in-memory state. """ - def __init__(self, uri: str, auth: Optional[tuple] = None, database: Optional[str] = None): + def __init__(self, uri: str, auth: Optional[tuple] = None, database: Optional[str] = None, auth_mode: str = "basic"): + self._uri = uri + self._auth = auth + self._auth_mode = auth_mode self._driver = GraphDatabase.driver(uri, auth=auth) if auth is not None else GraphDatabase.driver(uri) self._db = database + logger.info(f"Neo4jGraph initialized with backend=neo4j, uri={uri}, database={database}") def close(self): try: @@ -40,10 +47,46 @@ def list_datasets(self) -> List[Dict]: return [] # Scan lifecycle - def commit_scan(self, scan: Dict): + def commit_scan(self, scan: Dict, rows: Optional[List[Dict]] = None, folder_rows: Optional[List[Dict]] = None) -> Dict: + """Commit a scan with optional file and folder data. + + If rows and folder_rows are provided, writes them to Neo4j along with the scan node. + Returns verification dict with counts: {'db_scan_exists', 'db_files', 'db_folders', 'db_verified'}. + + Args: + scan: Scan metadata dict with id, path, started, ended, etc. + rows: Optional list of file dicts to write + folder_rows: Optional list of folder dicts to write + + Returns: + Dict with verification results including counts + """ if not scan or not scan.get('id'): - return + return {'db_scan_exists': False, 'db_verified': False, 'db_files': 0, 'db_folders': 0} + sid = scan.get('id') + + # If rows/folders provided, use full write_scan flow via Neo4jClient + if rows is not None or folder_rows is not None: + try: + from ..services.neo4j_client import Neo4jClient + client = Neo4jClient(self._uri, self._auth[0] if self._auth else None, + self._auth[1] if self._auth else None, + self._db, self._auth_mode).connect() + try: + client.ensure_constraints() + wres = client.write_scan(rows or [], folder_rows or [], scan) + vres = client.verify(sid) + logger.info(f"Neo4j commit completed: {wres.get('written_files', 0)} files, " + f"{wres.get('written_folders', 0)} folders for scan {sid}") + return vres + finally: + client.close() + except Exception as e: + logger.error(f"Neo4j commit_scan failed: {e}") + return {'db_scan_exists': False, 'db_verified': False, 'db_files': 0, 'db_folders': 0, 'error': str(e)} + + # Otherwise just create scan node (backward compat with old behavior) with self._session() as s: s.run( "MERGE (sc:Scan {id:$id}) SET sc.started=$started, sc.ended=$ended, sc.path=$path, " @@ -54,6 +97,8 @@ def commit_scan(self, scan: Dict): root_id=scan.get('root_id'), root_label=scan.get('root_label'), scan_source=scan.get('source') ).consume() + return {'db_scan_exists': True, 'db_verified': False, 'db_files': 0, 'db_folders': 0} + def delete_scan(self, scan_id: str): if not scan_id: return diff --git a/scidk/core/providers_init.py b/scidk/core/providers_init.py new file mode 100644 index 00000000..45a2f5d6 --- /dev/null +++ b/scidk/core/providers_init.py @@ -0,0 +1,53 @@ +"""Filesystem provider initialization. + +This module handles the initialization of all filesystem providers +(local_fs, mounted_fs, rclone) based on environment configuration. +""" + +import os + + +def initialize_fs_providers(app): + """Initialize and register all filesystem providers. + + Args: + app: Flask application instance + + Returns: + FsProviderRegistry with all providers registered + """ + from ..core.providers import ( + ProviderRegistry as FsProviderRegistry, + LocalFSProvider, + MountedFSProvider, + RcloneProvider + ) + + # Parse enabled providers from environment + prov_enabled = [ + p.strip() + for p in (os.environ.get('SCIDK_PROVIDERS', 'local_fs,mounted_fs').split(',')) + if p.strip() + ] + + # Ensure rclone provider is always available for listremotes validation + if 'rclone' not in prov_enabled: + prov_enabled.append('rclone') + + # Create registry with enabled providers + fs_providers = FsProviderRegistry(enabled=prov_enabled) + + # Initialize and register all providers + p_local = LocalFSProvider() + p_local.initialize(app, {}) + fs_providers.register(p_local) + + p_mounted = MountedFSProvider() + p_mounted.initialize(app, {}) + fs_providers.register(p_mounted) + + p_rclone = RcloneProvider() + p_rclone.initialize(app, {}) + fs_providers.register(p_rclone) + + return fs_providers diff --git a/scidk/core/rclone_mounts_loader.py b/scidk/core/rclone_mounts_loader.py new file mode 100644 index 00000000..af1719f0 --- /dev/null +++ b/scidk/core/rclone_mounts_loader.py @@ -0,0 +1,58 @@ +"""Rclone mounts metadata loader. + +This module rehydrates rclone mount metadata from SQLite on startup. +Note: Process handles are not restored (set to None). +""" + +import json +from typing import Dict, List, Any + + +def rehydrate_rclone_mounts() -> Dict[str, Dict[str, Any]]: + """Load rclone mount metadata from SQLite provider_mounts table. + + Returns: + dict mapping mount_id -> mount metadata dict + Returns empty dict on error + """ + try: + from . import path_index_sqlite as pix + from . import migrations as _migs + + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + cur.execute( + "SELECT id, provider, root, created, status, extra_json FROM provider_mounts WHERE provider='rclone'" + ) + rows = cur.fetchall() or [] + + mounts = {} + for (mid, provider, remote, created, status_persisted, extra) in rows: + try: + extra_obj = json.loads(extra) if extra else {} + except Exception: + extra_obj = {} + + mounts[mid] = { + 'id': mid, + 'name': mid, + 'remote': remote, + 'subpath': extra_obj.get('subpath'), + 'path': extra_obj.get('path'), + 'read_only': extra_obj.get('read_only'), + 'started_at': created, + 'process': None, # Process handles not restored + 'pid': None, + 'log_file': extra_obj.get('log_file'), + } + + return mounts + finally: + try: + conn.close() + except Exception: + pass + except Exception: + return {} diff --git a/scidk/core/rclone_settings.py b/scidk/core/rclone_settings.py new file mode 100644 index 00000000..4e571181 --- /dev/null +++ b/scidk/core/rclone_settings.py @@ -0,0 +1,92 @@ +"""Rclone interpretation settings loader. + +This module loads rclone-specific settings from SQLite on startup, +including suggest_mount_threshold and max_files_per_batch. +""" + +import os +from typing import Dict + + +def load_rclone_interpretation_settings(app) -> Dict[str, int]: + """Load rclone interpretation settings from SQLite with env overrides. + + Args: + app: Flask application instance to set config on + + Returns: + dict with 'suggest_mount_threshold' and 'max_files_per_batch' + + Side effects: + Sets app.config['rclone.interpret.suggest_mount_threshold'] + Sets app.config['rclone.interpret.max_files_per_batch'] + """ + def _env_int(name: str, dflt: int) -> int: + """Parse integer from environment with fallback.""" + try: + v = os.environ.get(name) + return int(v) if v is not None and v != '' else dflt + except Exception: + return dflt + + # Environment defaults (can override SQLite) + suggest_dflt = _env_int('SCIDK_RCLONE_INTERPRET_SUGGEST_MOUNT', 400) + max_batch_dflt = _env_int('SCIDK_RCLONE_INTERPRET_MAX_FILES', 1000) + max_batch_dflt = min(max(100, max_batch_dflt), 2000) + + try: + from . import path_index_sqlite as pix + from . import migrations as _migs + + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + + def _get_setting_int(key: str, dflt: int) -> int: + """Fetch integer setting from SQLite settings table.""" + row = cur.execute( + "SELECT value FROM settings WHERE key = ?", + (key,) + ).fetchone() + if row and row[0] not in (None, ''): + try: + return int(row[0]) + except Exception: + return dflt + return dflt + + # Load from SQLite, fallback to env defaults + suggest_mount_threshold = _get_setting_int( + 'rclone.interpret.suggest_mount_threshold', + suggest_dflt + ) + max_files_per_batch = _get_setting_int( + 'rclone.interpret.max_files_per_batch', + max_batch_dflt + ) + + # Clamp max_files_per_batch to sane range + max_files_per_batch = min(max(100, int(max_files_per_batch)), 2000) + + app.config['rclone.interpret.suggest_mount_threshold'] = int(suggest_mount_threshold) + app.config['rclone.interpret.max_files_per_batch'] = int(max_files_per_batch) + + return { + 'suggest_mount_threshold': int(suggest_mount_threshold), + 'max_files_per_batch': int(max_files_per_batch), + } + finally: + try: + conn.close() + except Exception: + pass + except Exception: + # Defaults if hydration fails + app.config.setdefault('rclone.interpret.suggest_mount_threshold', 400) + app.config.setdefault('rclone.interpret.max_files_per_batch', 1000) + + return { + 'suggest_mount_threshold': 400, + 'max_files_per_batch': 1000, + } diff --git a/scidk/core/telemetry_loader.py b/scidk/core/telemetry_loader.py new file mode 100644 index 00000000..6930fad4 --- /dev/null +++ b/scidk/core/telemetry_loader.py @@ -0,0 +1,42 @@ +"""Telemetry data loader for hydrating last_scan info from SQLite on startup. + +This module provides best-effort loading of telemetry.last_scan from the +path index SQLite database to restore session state across restarts. +""" + +import json +from typing import Optional, Dict, Any + + +def load_last_scan_from_sqlite() -> Optional[Dict[str, Any]]: + """Load telemetry.last_scan from SQLite settings table. + + Returns: + dict with last scan info, or None if not found or on error + """ + try: + from . import path_index_sqlite as pix + from . import migrations as _migs + + conn = pix.connect() + try: + _migs.migrate(conn) + cur = conn.cursor() + row = cur.execute( + "SELECT value FROM settings WHERE key = ?", + ("telemetry.last_scan",) + ).fetchone() + + if row and row[0]: + try: + return json.loads(row[0]) + except Exception: + return None + return None + finally: + try: + conn.close() + except Exception: + pass + except Exception: + return None diff --git a/scidk/services/config.py b/scidk/services/config.py index 4090b4cd..978cd669 100644 --- a/scidk/services/config.py +++ b/scidk/services/config.py @@ -16,18 +16,15 @@ def setdefault_env(name: str, value: str): # Defaults by channel (can be overridden by explicit env) if channel == 'dev': - setdefault_env('SCIDK_FEATURE_RCLONE_MOUNTS', '1') setdefault_env('SCIDK_FILES_VIEWER', 'rocrate') setdefault_env('SCIDK_FEATURE_FILE_INDEX', '1') setdefault_env('SCIDK_COMMIT_FROM_INDEX', '1') if os.environ.get('SCIDK_PROVIDERS') is None: os.environ['SCIDK_PROVIDERS'] = 'local_fs,mounted_fs,rclone' elif channel == 'beta': - setdefault_env('SCIDK_FEATURE_RCLONE_MOUNTS', '0') setdefault_env('SCIDK_COMMIT_FROM_INDEX', '1') else: # stable defaults - setdefault_env('SCIDK_FEATURE_RCLONE_MOUNTS', '0') setdefault_env('SCIDK_COMMIT_FROM_INDEX', '1') # Soft-disable rclone provider if binary missing and providers not explicitly set diff --git a/scidk/services/scan_index_service.py b/scidk/services/scan_index_service.py new file mode 100644 index 00000000..27bd40ef --- /dev/null +++ b/scidk/services/scan_index_service.py @@ -0,0 +1,162 @@ +"""Service for building and caching per-scan filesystem indexes for snapshot navigation. + +This module extracts the scan index building logic from app.py to keep the application +initialization lean. It builds hierarchical folder structures from scan data. +""" + +from pathlib import Path as _P + + +def get_or_build_scan_index(app, scan_id: str): + """Build or fetch per-scan filesystem index for snapshot navigation. + + Args: + app: Flask application instance with extensions['scidk'] configured + scan_id: Unique identifier for the scan + + Returns: + dict with keys: folder_info, children_folders, children_files, roots + Returns None if scan_id not found in scans registry + """ + cache = app.extensions['scidk'].setdefault('scan_fs', {}) + if scan_id in cache: + return cache[scan_id] + + scans = app.extensions['scidk'].get('scans', {}) + s = scans.get(scan_id) + if not s: + return None + + checksums = s.get('checksums') or [] + ds_map = app.extensions['scidk']['graph'].datasets # checksum -> dataset + + from ..core.path_utils import parse_remote_path, parent_remote_path + + folder_info = {} + children_files = {} + + def ensure_complete_parent_chain(path_str: str): + """Ensure all parent folders exist in folder_info for any given path""" + if not path_str or path_str in folder_info: + return + + info = parse_remote_path(path_str) + if info.get('is_remote'): + parent = parent_remote_path(path_str) + name = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or path_str) + else: + try: + p = _P(path_str) + parent = str(p.parent) + name = p.name or path_str + except Exception: + parent = '' + name = path_str + + folder_info[path_str] = { + 'path': path_str, + 'name': name, + 'parent': parent, + } + + if parent and parent != path_str: + ensure_complete_parent_chain(parent) + + # Seed scan base path (stable roots even on empty scans) + try: + base_path = s.get('path') or '' + if base_path: + ensure_complete_parent_chain(base_path) + except Exception: + pass + + # Process files and ensure their parent chains exist + for ch in checksums: + d = ds_map.get(ch) + if not d: + continue + file_path = d.get('path') + if not file_path: + continue + + info = parse_remote_path(file_path) + if info.get('is_remote'): + parent = parent_remote_path(file_path) + filename = (info.get('parts')[-1] if info.get('parts') else info.get('remote_name') or file_path) + else: + try: + p = _P(file_path) + parent = str(p.parent) + filename = p.name or file_path + except Exception: + parent = '' + filename = file_path + + file_entry = { + 'id': d.get('id'), + 'path': file_path, + 'filename': d.get('filename') or filename, + 'extension': d.get('extension'), + 'size_bytes': int(d.get('size_bytes') or 0), + 'modified': float(d.get('modified') or 0), + 'mime_type': d.get('mime_type'), + 'checksum': d.get('checksum'), + } + children_files.setdefault(parent, []).append(file_entry) + + if parent: + ensure_complete_parent_chain(parent) + + # Process explicitly recorded folders + for f in (s.get('folders') or []): + path = f.get('path') + if path: + ensure_complete_parent_chain(path) + + # Build children_folders map + children_folders = {} + for fpath, info in folder_info.items(): + par = info.get('parent') + if par and par in folder_info: + children_folders.setdefault(par, []).append(fpath) + + # Find actual roots + roots = sorted([fp for fp, info in folder_info.items() + if not info.get('parent') or info.get('parent') not in folder_info]) + + # Prefer scan base as visible root and drop its ancestors + try: + base_path = s.get('path') or '' + if base_path and base_path in folder_info: + if base_path not in roots: + roots.append(base_path) + def _is_ancestor(candidate: str, child: str) -> bool: + if not candidate or candidate == child: + return False + cinf = parse_remote_path(candidate) + chinf = parse_remote_path(child) + if chinf.get('is_remote') and cinf.get('is_remote'): + return child.startswith(candidate.rstrip('/') + '/') + try: + return str(_P(child)).startswith(str(_P(candidate)) + '/') + except Exception: + return False + roots = [r for r in roots if not _is_ancestor(r, base_path) or r == base_path] + roots = sorted(list(dict.fromkeys(roots))) + except Exception: + pass + + # Sort children deterministically + for k in list(children_folders.keys()): + children_folders[k].sort(key=lambda p: folder_info.get(p, {}).get('name', '').lower()) + for k in list(children_files.keys()): + children_files[k].sort(key=lambda f: (f.get('filename') or '').lower()) + + idx = { + 'folder_info': folder_info, + 'children_folders': children_folders, + 'children_files': children_files, + 'roots': roots, + } + cache[scan_id] = idx + return idx diff --git a/scidk/ui/templates/settings.html b/scidk/ui/templates/settings.html index fa2c17e8..e079ee2c 100644 --- a/scidk/ui/templates/settings.html +++ b/scidk/ui/templates/settings.html @@ -14,7 +14,6 @@

Settings

Channel: {{ info.channel or 'stable' }} Providers: {{ info.providers }} Files viewer: {{ info.files_viewer or '(default)' }} - Rclone mounts: {{ info.rclone_mounts or 'off' }}
@@ -206,10 +205,9 @@

Rclone Interpretation

-{% if rclone_mounts_feature %}

Rclone Mounts

-

Feature flag enabled via SCIDK_RCLONE_MOUNTS=1. Manage mounts under ./data/mounts.

+

Manage rclone mounts under ./data/mounts.

@@ -245,7 +243,6 @@

Rclone Mounts


   
Note: On Windows, cmount/WinFsp may be required; this UI targets Linux/macOS primarily.

 

-{% endif %}
 {% endblock %}
 {% block head %}