From 62f541cfb7182331e555b34d720471f7279a8ca8 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 13:47:42 +0300 Subject: [PATCH 01/29] Base code --- backend/base_classes/base_file.py | 5 ++- backend/collectors/collect_data_files.py | 8 ++-- backend/collectors/collect_manifests.py | 8 ++-- backend/collectors/collector.py | 29 ++++++++++++++ backend/extractors/constants.py | 3 ++ backend/extractors/data_files_extractor.py | 22 +++++----- backend/extractors/extractor.py | 42 +++++++++++++++++++- backend/extractors/manifests_extractor.py | 22 +++++----- backend/graph_normalizer/graph_normalizer.py | 6 +++ frontend/src/graphConstants.js | 6 ++- frontend/src/pages/DocsPage.jsx | 5 +++ frontend/src/pages/TableLayout.jsx | 3 +- 12 files changed, 124 insertions(+), 35 deletions(-) create mode 100644 backend/extractors/constants.py diff --git a/backend/base_classes/base_file.py b/backend/base_classes/base_file.py index 9cf2e22..7cc7e10 100644 --- a/backend/base_classes/base_file.py +++ b/backend/base_classes/base_file.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass, fields -from typing import List +from dataclasses import dataclass, field, fields +from typing import List, Optional from constants import FileType @@ -14,6 +14,7 @@ class BaseFile: type: FileType file_path: str child_files: List[str] + error: Optional[str] = field(default=None, kw_only=True) def to_dict(self): result_dict = {field.name: getattr(self, field.name) for field in fields(self) if not isinstance(getattr(self, field.name), HiddenFile)} diff --git a/backend/collectors/collect_data_files.py b/backend/collectors/collect_data_files.py index 9b41b92..165a2c8 100644 --- a/backend/collectors/collect_data_files.py +++ b/backend/collectors/collect_data_files.py @@ -44,11 +44,11 @@ def __init__( @timed def collect(self) -> FilesCollection: - data_files_extraction_result = DataFilesExtractor(self._table_name, self._manifests).extract_dataframe() - self._errors = data_files_extraction_result.errors + extractor = DataFilesExtractor(self._table_name, self._manifests) + data_files_collection = self._collect_rows_isolating_failures(extractor) + self._errors = data_files_collection.errors - data_files_rows = data_files_extraction_result.dataframe.collect() - self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_rows] + self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_collection.rows] return FilesCollection(files=self._data_files, errors=self._errors) diff --git a/backend/collectors/collect_manifests.py b/backend/collectors/collect_manifests.py index 35d907e..b5a0718 100644 --- a/backend/collectors/collect_manifests.py +++ b/backend/collectors/collect_manifests.py @@ -43,11 +43,11 @@ def __init__( @timed def collect(self) -> FilesCollection: - manifest_extraction_result = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df).extract_dataframe() - self._errors = manifest_extraction_result.errors + extractor = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df) + manifests_collection = self._collect_rows_isolating_failures(extractor) + self._errors = manifests_collection.errors - manifests_rows = manifest_extraction_result.dataframe.collect() - self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_rows] + self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_collection.rows] return FilesCollection(files=self._manifests, errors=self._errors) diff --git a/backend/collectors/collector.py b/backend/collectors/collector.py index 1450467..14899df 100644 --- a/backend/collectors/collector.py +++ b/backend/collectors/collector.py @@ -1,9 +1,15 @@ +import logging from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import Dict, List +import pyspark + from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction +from extractors.extractor import Extractor + +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -13,7 +19,30 @@ class FilesCollection: warnings: Dict[str, str] = field(default_factory=dict) +@dataclass(frozen=True) +class RowsCollection: + rows: List[pyspark.sql.Row] = field(default_factory=list) + errors: Dict[str, str] = field(default_factory=dict) + + class Collector(SparkTableAction, ABC): @abstractmethod def collect(self) -> FilesCollection: pass + + def _collect_rows_isolating_failures(self, extractor: Extractor) -> RowsCollection: + extraction_result = extractor.extract_dataframe() + + try: + return RowsCollection(rows=extraction_result.dataframe.collect(), errors=extraction_result.errors) + except Exception: + logger.warning( + f"[{self._table_name}] Collection failed, retrying without the files that cannot be read", + exc_info=True, + ) + if not extractor.isolate_failing_sources(): + raise + + retry_result = extractor.extract_dataframe() + + return RowsCollection(rows=retry_result.dataframe.collect(), errors=retry_result.errors) diff --git a/backend/extractors/constants.py b/backend/extractors/constants.py new file mode 100644 index 0000000..5cab1f3 --- /dev/null +++ b/backend/extractors/constants.py @@ -0,0 +1,3 @@ +DEFAULT_SOURCE_ERROR_PREFIX = "Failed to read file" +MANIFEST_LIST_SOURCE_ERROR_PREFIX = "Failed to read manifest list" +MANIFEST_SOURCE_ERROR_PREFIX = "Failed to read manifest" diff --git a/backend/extractors/data_files_extractor.py b/backend/extractors/data_files_extractor.py index 7eb2ad5..a23e0eb 100644 --- a/backend/extractors/data_files_extractor.py +++ b/backend/extractors/data_files_extractor.py @@ -6,6 +6,7 @@ from collectors.collect_manifests import ManifestRecord from constants import MAX_DATA_FILES_TO_COLLECT +from extractors.constants import MANIFEST_SOURCE_ERROR_PREFIX from extractors.extractor import ExtractionResult, Extractor max_data_files_to_collect = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", MAX_DATA_FILES_TO_COLLECT)) @@ -39,10 +40,11 @@ class DataFilesExtractor(Extractor): + SOURCE_ERROR_PREFIX = MANIFEST_SOURCE_ERROR_PREFIX + def __init__(self, table_name: str, manifest_entries: List[ManifestRecord]): super().__init__(table_name) self._manifest_entries = manifest_entries - self._errors = {} def extract_dataframe(self) -> ExtractionResult: data_files_df = self._collect_data_files_from_manifests(self._manifest_entries) @@ -140,16 +142,14 @@ def _find_included_data_files(grouped_files_limited_df, snapshot_timestamp_cutof def _collect_data_files_from_manifests(self, manifest_rows): avro_df = None for manifest_entry in manifest_rows: - try: - df = self._collect_data_files_from_manifest(manifest_entry) - - if avro_df is None: - avro_df = df - else: - avro_df = avro_df.unionByName(df, allowMissingColumns=True) - - except Exception as e: - self._errors[manifest_entry.file_path] = f"Avro read error: {e}" + df = self._read_source(manifest_entry.file_path, lambda: self._collect_data_files_from_manifest(manifest_entry)) + if df is None: + continue + + if avro_df is None: + avro_df = df + else: + avro_df = avro_df.unionByName(df, allowMissingColumns=True) if avro_df is None: return self._spark.createDataFrame([], DATA_FILE_RECORD_SCHEMA) diff --git a/backend/extractors/extractor.py b/backend/extractors/extractor.py index 162a837..aff2d7b 100644 --- a/backend/extractors/extractor.py +++ b/backend/extractors/extractor.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Dict +from typing import Callable, Dict, Optional, Set import pyspark from base_classes.spark_table_action import SparkTableAction +from extractors.constants import DEFAULT_SOURCE_ERROR_PREFIX @dataclass(frozen=True) @@ -14,6 +15,45 @@ class ExtractionResult: class Extractor(SparkTableAction, ABC): + SOURCE_ERROR_PREFIX = DEFAULT_SOURCE_ERROR_PREFIX + + def __init__(self, full_table_name: str): + super().__init__(full_table_name) + self._errors: Dict[str, str] = {} + self._source_dataframes: Dict[str, pyspark.sql.DataFrame] = {} + self._failed_source_paths: Set[str] = set() + @abstractmethod def extract_dataframe(self) -> ExtractionResult: pass + + def isolate_failing_sources(self) -> bool: + found_failing_source = False + + for source_path, source_dataframe in list(self._source_dataframes.items()): + try: + source_dataframe.collect() + except Exception as e: + self._record_source_failure(source_path, e) + self._source_dataframes.pop(source_path) + found_failing_source = True + + return found_failing_source + + def _read_source(self, source_path: str, read_source: Callable[[], pyspark.sql.DataFrame]) -> Optional[pyspark.sql.DataFrame]: + if source_path in self._failed_source_paths: + return None + + try: + source_dataframe = read_source() + except Exception as e: + self._record_source_failure(source_path, e) + return None + + self._source_dataframes[source_path] = source_dataframe + + return source_dataframe + + def _record_source_failure(self, source_path: str, error: Exception) -> None: + self._errors[source_path] = f"{self.SOURCE_ERROR_PREFIX}: {error}" + self._failed_source_paths.add(source_path) diff --git a/backend/extractors/manifests_extractor.py b/backend/extractors/manifests_extractor.py index 2af85b2..5d20f09 100644 --- a/backend/extractors/manifests_extractor.py +++ b/backend/extractors/manifests_extractor.py @@ -3,6 +3,7 @@ from pyspark.sql.types import LongType, StringType, StructField, StructType from collectors.collect_snapshots import SnapshotRecord +from extractors.constants import MANIFEST_LIST_SOURCE_ERROR_PREFIX from extractors.extractor import ExtractionResult, Extractor SNAPSHOT_TO_TIMESTAMP_SCHEMA = StructType( @@ -22,6 +23,8 @@ class ManifestsExtractor(Extractor): + SOURCE_ERROR_PREFIX = MANIFEST_LIST_SOURCE_ERROR_PREFIX + def __init__( self, table_name: str, @@ -31,7 +34,6 @@ def __init__( super().__init__(table_name) self._snapshots = snapshots self._manifests_to_ignore_df = manifests_to_ignore_df - self._errors = {} def extract_dataframe(self) -> ExtractionResult: manifests_df = self._union_manifests_for_snapshots() @@ -50,15 +52,15 @@ def _union_manifests_for_snapshots(self) -> pyspark.sql.DataFrame: for snapshot in self._snapshots: snap_id = snapshot.snapshot_id manifest_list_path = snapshot.file_path - try: - df = self._read_manifests_for_snapshot(manifest_list_path, snap_id) - if result is None: - result = df - else: - result = result.unionByName(df, allowMissingColumns=True) - - except Exception as e: - self._errors[manifest_list_path] = f"Failed to read/union manifest list: {e}" + + df = self._read_source(manifest_list_path, lambda: self._read_manifests_for_snapshot(manifest_list_path, snap_id)) + if df is None: + continue + + if result is None: + result = df + else: + result = result.unionByName(df, allowMissingColumns=True) if result is None: result = self._spark.createDataFrame([], MANIFEST_BASE_SCHEMA) diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index 79cab66..b3a143c 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -11,6 +11,8 @@ def __init__(self, table_data: TableInventoryResult): self._current_table_metadata = table_data.current_table_specs def normalize(self): + self._attach_errors_to_files() + nodes = [file.to_dict() for file in self._files] return to_json_safe( @@ -21,3 +23,7 @@ def normalize(self): "warnings": self._warnings, } ) + + def _attach_errors_to_files(self): + for file in self._files: + file.error = self._errors.get(file.file_path) diff --git a/frontend/src/graphConstants.js b/frontend/src/graphConstants.js index 7e28692..a3f4749 100644 --- a/frontend/src/graphConstants.js +++ b/frontend/src/graphConstants.js @@ -18,9 +18,11 @@ export const NODE_STYLE_MAP = { [FileType.SNAPSHOT]: { rgb: [25, 100, 185], level: 0 }, [FileType.MANIFEST]: { rgb: [25, 145, 185], level: 1 }, [FileType.DATA]: { rgb: [25, 150, 115], level: 2 }, - [FileType.POSITION_DELETE]: { rgb: [185, 35, 60], level: 2 }, - [FileType.EQUALITY_DELETE]: { rgb: [185, 35, 60], level: 2 }, + [FileType.POSITION_DELETE]: { rgb: [230, 145, 30], level: 2 }, + [FileType.EQUALITY_DELETE]: { rgb: [230, 145, 30], level: 2 }, }; + +export const ERROR_NODE_RGB = [185, 35, 60]; const FILE_TYPE_LABELS = { [FileType.MAIN_METADATA]: "Main Metadata", [FileType.METADATA]: "Metadata", diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index 975c092..83991f2 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -450,6 +450,11 @@ const SECTIONS = [ Data file — the actual Parquet, ORC, or Avro file containing your rows +
  • + Unreadable file — a file + that could not be read is drawn in red and shows the reason in its + details panel. The rest of the graph still loads +
    • diff --git a/frontend/src/pages/TableLayout.jsx b/frontend/src/pages/TableLayout.jsx index 3c6bc6a..216b881 100644 --- a/frontend/src/pages/TableLayout.jsx +++ b/frontend/src/pages/TableLayout.jsx @@ -19,6 +19,7 @@ import { useTableSpecs } from "../context/TableSpecsContext"; import { BRANCH_CONNECTION_COLOR, DELETED_DATA_FILE_CONNECTION_COLOR, + ERROR_NODE_RGB, FileType, MAIN_BRANCH_NAME, NODE_STYLE_MAP, @@ -249,7 +250,7 @@ export default function TableLayout() { rgb: [100, 100, 100], level: 0, }; - const [r, g, b] = style.rgb; + const [r, g, b] = details.error ? ERROR_NODE_RGB : style.rgb; const colorShift = colorShiftByFilePath.get(details.file_path) ?? 1; return { From 245b7ba45a5d3d4a4b5a283872179997fd70fedb Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 14:16:28 +0300 Subject: [PATCH 02/29] Base code1 --- backend/base_classes/base_file.py | 1 + backend/collectors/collect_data_files.py | 8 ++-- backend/collectors/collect_manifests.py | 10 ++--- backend/collectors/collector.py | 24 +++--------- backend/constants.py | 6 ++- backend/extractors/data_files_extractor.py | 11 +++--- backend/extractors/extractor.py | 41 +++++++++----------- backend/extractors/manifests_extractor.py | 12 ++---- backend/graph_normalizer/graph_normalizer.py | 18 ++++----- backend/table_inventory/table_inventory.py | 8 ++-- frontend/src/pages/DocsPage.jsx | 4 +- 11 files changed, 59 insertions(+), 84 deletions(-) diff --git a/backend/base_classes/base_file.py b/backend/base_classes/base_file.py index 7cc7e10..2f4b303 100644 --- a/backend/base_classes/base_file.py +++ b/backend/base_classes/base_file.py @@ -15,6 +15,7 @@ class BaseFile: file_path: str child_files: List[str] error: Optional[str] = field(default=None, kw_only=True) + warning: Optional[str] = field(default=None, kw_only=True) def to_dict(self): result_dict = {field.name: getattr(self, field.name) for field in fields(self) if not isinstance(getattr(self, field.name), HiddenFile)} diff --git a/backend/collectors/collect_data_files.py b/backend/collectors/collect_data_files.py index 165a2c8..2cd8e9e 100644 --- a/backend/collectors/collect_data_files.py +++ b/backend/collectors/collect_data_files.py @@ -38,19 +38,17 @@ def __init__( ): super().__init__(full_table_name) self._manifests = manifests - self._errors: Dict[str, str] = {} self._data_files: List[DataFileRecord] = [] @timed def collect(self) -> FilesCollection: extractor = DataFilesExtractor(self._table_name, self._manifests) - data_files_collection = self._collect_rows_isolating_failures(extractor) - self._errors = data_files_collection.errors + data_files_rows = self._collect_rows_isolating_failures(extractor) - self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_collection.rows] + self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_rows] - return FilesCollection(files=self._data_files, errors=self._errors) + return FilesCollection(files=self._data_files) def _process_data_file_row(self, data_file_row) -> DataFileRecord: data_file_dict = data_file_row.asDict(recursive=True) diff --git a/backend/collectors/collect_manifests.py b/backend/collectors/collect_manifests.py index b5a0718..d29ed91 100644 --- a/backend/collectors/collect_manifests.py +++ b/backend/collectors/collect_manifests.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, List, Optional +from typing import List, Optional import pyspark @@ -37,19 +37,17 @@ def __init__( super().__init__(full_table_name) self._snapshots = snapshots self._manifests_to_ignore_df = manifests_to_ignore_df - self._errors: Dict[str, str] = {} self._manifests: List[ManifestRecord] = [] @timed def collect(self) -> FilesCollection: extractor = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df) - manifests_collection = self._collect_rows_isolating_failures(extractor) - self._errors = manifests_collection.errors + manifests_rows = self._collect_rows_isolating_failures(extractor) - self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_collection.rows] + self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_rows] - return FilesCollection(files=self._manifests, errors=self._errors) + return FilesCollection(files=self._manifests) @staticmethod def _process_manifest_row(manifest_row) -> ManifestRecord: diff --git a/backend/collectors/collector.py b/backend/collectors/collector.py index 14899df..b2f8313 100644 --- a/backend/collectors/collector.py +++ b/backend/collectors/collector.py @@ -1,4 +1,3 @@ -import logging from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import Dict, List @@ -8,21 +7,13 @@ from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction from extractors.extractor import Extractor - -logger = logging.getLogger(__name__) +from icegraph_logger import logger @dataclass(frozen=True) class FilesCollection: files: List[BaseFile] = field(default_factory=list) errors: Dict[str, str] = field(default_factory=dict) - warnings: Dict[str, str] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RowsCollection: - rows: List[pyspark.sql.Row] = field(default_factory=list) - errors: Dict[str, str] = field(default_factory=dict) class Collector(SparkTableAction, ABC): @@ -30,19 +21,14 @@ class Collector(SparkTableAction, ABC): def collect(self) -> FilesCollection: pass - def _collect_rows_isolating_failures(self, extractor: Extractor) -> RowsCollection: - extraction_result = extractor.extract_dataframe() - + def _collect_rows_isolating_failures(self, extractor: Extractor) -> List[pyspark.sql.Row]: try: - return RowsCollection(rows=extraction_result.dataframe.collect(), errors=extraction_result.errors) + return extractor.extract_dataframe().collect() except Exception: logger.warning( f"[{self._table_name}] Collection failed, retrying without the files that cannot be read", exc_info=True, ) - if not extractor.isolate_failing_sources(): - raise - - retry_result = extractor.extract_dataframe() + extractor.isolate_failing_sources() - return RowsCollection(rows=retry_result.dataframe.collect(), errors=retry_result.errors) + return extractor.extract_dataframe().collect() diff --git a/backend/constants.py b/backend/constants.py index d308b24..7ce68bd 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -9,7 +9,7 @@ COMPUTE_CLEANUP_TIME_SECONDS = 12 -MAX_DATA_FILES_TO_COLLECT = 5_000 +MAX_DATA_FILES_TO_COLLECT = 5 TABLE_LIST_CACHE_TTL_SECONDS = 60 @@ -53,3 +53,7 @@ class FileType(Enum): unless a newer visible snapshot also references them, in which case they are included. Every data file you see is referenced by at least one snapshot that is newer than the cut-off snapshot. """) + +DATA_FILES_CUTOFF_MANIFEST_WARNING = inspect.cleandoc(""" +The data files of this manifest were not loaded because the limit of {max_data_files_to_collect} data files was reached. +""") diff --git a/backend/extractors/data_files_extractor.py b/backend/extractors/data_files_extractor.py index a23e0eb..1f0e4b5 100644 --- a/backend/extractors/data_files_extractor.py +++ b/backend/extractors/data_files_extractor.py @@ -1,13 +1,14 @@ import os from typing import List +import pyspark from pyspark.sql import Window, functions as F from pyspark.sql.types import LongType, StringType, StructField, StructType from collectors.collect_manifests import ManifestRecord from constants import MAX_DATA_FILES_TO_COLLECT from extractors.constants import MANIFEST_SOURCE_ERROR_PREFIX -from extractors.extractor import ExtractionResult, Extractor +from extractors.extractor import Extractor max_data_files_to_collect = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", MAX_DATA_FILES_TO_COLLECT)) @@ -46,7 +47,7 @@ def __init__(self, table_name: str, manifest_entries: List[ManifestRecord]): super().__init__(table_name) self._manifest_entries = manifest_entries - def extract_dataframe(self) -> ExtractionResult: + def extract_dataframe(self) -> pyspark.sql.DataFrame: data_files_df = self._collect_data_files_from_manifests(self._manifest_entries) data_files_with_latest_ts_df = self._match_data_file_to_latest_snapshot(data_files_df) @@ -57,9 +58,7 @@ def extract_dataframe(self) -> ExtractionResult: snapshot_timestamp_cutoff_df = self._find_cutoff_snapshot_timestamp(data_files_limited_df) - included_data_files_df = self._find_included_data_files(data_files_limited_df, snapshot_timestamp_cutoff_df) - - return ExtractionResult(included_data_files_df, self._errors) + return self._find_included_data_files(data_files_limited_df, snapshot_timestamp_cutoff_df) @staticmethod def _group_data_files_by_manifests(avro_df): @@ -142,7 +141,7 @@ def _find_included_data_files(grouped_files_limited_df, snapshot_timestamp_cutof def _collect_data_files_from_manifests(self, manifest_rows): avro_df = None for manifest_entry in manifest_rows: - df = self._read_source(manifest_entry.file_path, lambda: self._collect_data_files_from_manifest(manifest_entry)) + df = self._read_source(manifest_entry, lambda: self._collect_data_files_from_manifest(manifest_entry)) if df is None: continue diff --git a/backend/extractors/extractor.py b/backend/extractors/extractor.py index aff2d7b..6bd4fbd 100644 --- a/backend/extractors/extractor.py +++ b/backend/extractors/extractor.py @@ -1,17 +1,18 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Callable, Dict, Optional, Set import pyspark +from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction from extractors.constants import DEFAULT_SOURCE_ERROR_PREFIX @dataclass(frozen=True) -class ExtractionResult: +class SourceRead: + file: BaseFile dataframe: pyspark.sql.DataFrame - errors: Dict[str, str] = field(default_factory=dict) class Extractor(SparkTableAction, ABC): @@ -19,41 +20,35 @@ class Extractor(SparkTableAction, ABC): def __init__(self, full_table_name: str): super().__init__(full_table_name) - self._errors: Dict[str, str] = {} - self._source_dataframes: Dict[str, pyspark.sql.DataFrame] = {} + self._sources: Dict[str, SourceRead] = {} self._failed_source_paths: Set[str] = set() @abstractmethod - def extract_dataframe(self) -> ExtractionResult: + def extract_dataframe(self) -> pyspark.sql.DataFrame: pass - def isolate_failing_sources(self) -> bool: - found_failing_source = False - - for source_path, source_dataframe in list(self._source_dataframes.items()): + def isolate_failing_sources(self) -> None: + for source_path, source in list(self._sources.items()): try: - source_dataframe.collect() + source.dataframe.collect() except Exception as e: - self._record_source_failure(source_path, e) - self._source_dataframes.pop(source_path) - found_failing_source = True - - return found_failing_source + self._record_source_failure(source.file, e) + self._sources.pop(source_path) - def _read_source(self, source_path: str, read_source: Callable[[], pyspark.sql.DataFrame]) -> Optional[pyspark.sql.DataFrame]: - if source_path in self._failed_source_paths: + def _read_source(self, source_file: BaseFile, read_source: Callable[[], pyspark.sql.DataFrame]) -> Optional[pyspark.sql.DataFrame]: + if source_file.file_path in self._failed_source_paths: return None try: source_dataframe = read_source() except Exception as e: - self._record_source_failure(source_path, e) + self._record_source_failure(source_file, e) return None - self._source_dataframes[source_path] = source_dataframe + self._sources[source_file.file_path] = SourceRead(file=source_file, dataframe=source_dataframe) return source_dataframe - def _record_source_failure(self, source_path: str, error: Exception) -> None: - self._errors[source_path] = f"{self.SOURCE_ERROR_PREFIX}: {error}" - self._failed_source_paths.add(source_path) + def _record_source_failure(self, source_file: BaseFile, error: Exception) -> None: + source_file.error = f"{self.SOURCE_ERROR_PREFIX}: {error}" + self._failed_source_paths.add(source_file.file_path) diff --git a/backend/extractors/manifests_extractor.py b/backend/extractors/manifests_extractor.py index 5d20f09..0411e47 100644 --- a/backend/extractors/manifests_extractor.py +++ b/backend/extractors/manifests_extractor.py @@ -4,7 +4,7 @@ from collectors.collect_snapshots import SnapshotRecord from extractors.constants import MANIFEST_LIST_SOURCE_ERROR_PREFIX -from extractors.extractor import ExtractionResult, Extractor +from extractors.extractor import Extractor SNAPSHOT_TO_TIMESTAMP_SCHEMA = StructType( [ @@ -35,17 +35,13 @@ def __init__( self._snapshots = snapshots self._manifests_to_ignore_df = manifests_to_ignore_df - def extract_dataframe(self) -> ExtractionResult: + def extract_dataframe(self) -> pyspark.sql.DataFrame: manifests_df = self._union_manifests_for_snapshots() manifests_with_timestamps_df = self._enrich_manifests_with_timestamps(manifests_df) valid_manifests_df = self._filter_ignored_manifests(manifests_with_timestamps_df) - manifests_df = self._aggregate_snapshots_by_manifests_sorted(valid_manifests_df) - return ExtractionResult( - manifests_df, - self._errors, - ) + return self._aggregate_snapshots_by_manifests_sorted(valid_manifests_df) def _union_manifests_for_snapshots(self) -> pyspark.sql.DataFrame: result = None @@ -53,7 +49,7 @@ def _union_manifests_for_snapshots(self) -> pyspark.sql.DataFrame: snap_id = snapshot.snapshot_id manifest_list_path = snapshot.file_path - df = self._read_source(manifest_list_path, lambda: self._read_manifests_for_snapshot(manifest_list_path, snap_id)) + df = self._read_source(snapshot, lambda: self._read_manifests_for_snapshot(manifest_list_path, snap_id)) if df is None: continue diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index b3a143c..16cfc6f 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -1,3 +1,6 @@ +from typing import Callable, Dict, Optional + +from base_classes.base_file import BaseFile from table_inventory.table_inventory import TableInventoryResult from graph_normalizer.utils import to_json_safe @@ -6,24 +9,21 @@ class GraphNormalizer: def __init__(self, table_data: TableInventoryResult): self._files = table_data.metadata_files + table_data.snapshots + table_data.manifests + table_data.data_files - self._errors = table_data.errors - self._warnings = table_data.warnings + self._table_errors = table_data.errors + self._table_warnings = table_data.warnings self._current_table_metadata = table_data.current_table_specs def normalize(self): - self._attach_errors_to_files() - nodes = [file.to_dict() for file in self._files] return to_json_safe( { "nodes": nodes, "metadata": self._current_table_metadata, - "errors": self._errors, - "warnings": self._warnings, + "errors": self._file_issues(lambda file: file.error) | self._table_errors, + "warnings": self._file_issues(lambda file: file.warning) | self._table_warnings, } ) - def _attach_errors_to_files(self): - for file in self._files: - file.error = self._errors.get(file.file_path) + def _file_issues(self, get_issue: Callable[[BaseFile], Optional[str]]) -> Dict[str, str]: + return {file.file_path: get_issue(file) for file in self._files if get_issue(file)} diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index fd6f7eb..e9b4091 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -9,7 +9,7 @@ from collectors.collect_manifests import CollectManifests, ManifestRecord from collectors.collect_metadata import CollectMetadata, MetadataFileRecord from collectors.collect_snapshots import CollectSnapshots, SnapshotRecord -from constants import DATA_FILES_CUTOFF_WARNING, FileType, MAX_DATA_FILES_TO_COLLECT +from constants import DATA_FILES_CUTOFF_MANIFEST_WARNING, DATA_FILES_CUTOFF_WARNING, FileType, MAX_DATA_FILES_TO_COLLECT from icegraph_logger import logger from search_cutoff.find_search_cutoff import SearchCutoff, find_search_cutoff from table_inventory.utils import format_schemas_to_full_dict, get_json_metadata_from_path, parse_json_string_fields @@ -92,7 +92,6 @@ def _collect_and_set_snapshots(self): ).collect() self._errors.update(snapshot_collection.errors) - self._warnings.update(snapshot_collection.warnings) self._snapshots = snapshot_collection.files @@ -105,7 +104,6 @@ def _collect_metadata_manifests_and_data_files(self): metadata_collection = metadata_future.result() self._errors.update(metadata_collection.errors) - self._warnings.update(metadata_collection.warnings) self._metadata_files = metadata_collection.files @@ -118,8 +116,6 @@ def _collect_metadata_manifests_and_data_files(self): self._errors.update(manifests_collection.errors) self._errors.update(data_files_collection.errors) - self._warnings.update(manifests_collection.warnings) - self._warnings.update(data_files_collection.warnings) self._manifests = manifests_collection.files self._data_files = data_files_collection.files @@ -205,6 +201,8 @@ def _warn_if_data_cutoff_happened(self): for manifest in self._manifests: if len(manifest.child_files) == 0: + manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=max_data_files_to_collect) + if max_manifest_added_snapshot_timestamp is None or max_manifest_added_snapshot_timestamp < manifest.added_snapshot_timestamp: max_manifest_added_snapshot_timestamp = manifest.added_snapshot_timestamp max_manifest_added_snapshot_id = manifest.added_snapshot_id diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index 83991f2..23c5bb7 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -452,8 +452,8 @@ const SECTIONS = [
  • Unreadable file — a file - that could not be read is drawn in red and shows the reason in its - details panel. The rest of the graph still loads + whose metadata could not be obtained is drawn in red and shows the + reason in its details panel. The rest of the graph still loads
    • From 8aff196a42f56ec80be095828f9936b12541d0c7 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 14:24:38 +0300 Subject: [PATCH 03/29] Better reading --- backend/collectors/collect_data_files.py | 3 +- backend/collectors/collect_manifests.py | 3 +- backend/collectors/collector.py | 16 ---------- backend/extractors/extractor.py | 40 +++++------------------- 4 files changed, 10 insertions(+), 52 deletions(-) diff --git a/backend/collectors/collect_data_files.py b/backend/collectors/collect_data_files.py index 2cd8e9e..724c771 100644 --- a/backend/collectors/collect_data_files.py +++ b/backend/collectors/collect_data_files.py @@ -43,8 +43,7 @@ def __init__( @timed def collect(self) -> FilesCollection: - extractor = DataFilesExtractor(self._table_name, self._manifests) - data_files_rows = self._collect_rows_isolating_failures(extractor) + data_files_rows = DataFilesExtractor(self._table_name, self._manifests).extract_dataframe().collect() self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_rows] diff --git a/backend/collectors/collect_manifests.py b/backend/collectors/collect_manifests.py index d29ed91..b104e1b 100644 --- a/backend/collectors/collect_manifests.py +++ b/backend/collectors/collect_manifests.py @@ -42,8 +42,7 @@ def __init__( @timed def collect(self) -> FilesCollection: - extractor = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df) - manifests_rows = self._collect_rows_isolating_failures(extractor) + manifests_rows = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df).extract_dataframe().collect() self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_rows] diff --git a/backend/collectors/collector.py b/backend/collectors/collector.py index b2f8313..70f5abe 100644 --- a/backend/collectors/collector.py +++ b/backend/collectors/collector.py @@ -2,12 +2,8 @@ from dataclasses import dataclass, field from typing import Dict, List -import pyspark - from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction -from extractors.extractor import Extractor -from icegraph_logger import logger @dataclass(frozen=True) @@ -20,15 +16,3 @@ class Collector(SparkTableAction, ABC): @abstractmethod def collect(self) -> FilesCollection: pass - - def _collect_rows_isolating_failures(self, extractor: Extractor) -> List[pyspark.sql.Row]: - try: - return extractor.extract_dataframe().collect() - except Exception: - logger.warning( - f"[{self._table_name}] Collection failed, retrying without the files that cannot be read", - exc_info=True, - ) - extractor.isolate_failing_sources() - - return extractor.extract_dataframe().collect() diff --git a/backend/extractors/extractor.py b/backend/extractors/extractor.py index 6bd4fbd..757b6aa 100644 --- a/backend/extractors/extractor.py +++ b/backend/extractors/extractor.py @@ -1,54 +1,30 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Callable, Dict, Optional, Set +from typing import Callable, Optional import pyspark from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction from extractors.constants import DEFAULT_SOURCE_ERROR_PREFIX - - -@dataclass(frozen=True) -class SourceRead: - file: BaseFile - dataframe: pyspark.sql.DataFrame +from icegraph_logger import logger class Extractor(SparkTableAction, ABC): SOURCE_ERROR_PREFIX = DEFAULT_SOURCE_ERROR_PREFIX - def __init__(self, full_table_name: str): - super().__init__(full_table_name) - self._sources: Dict[str, SourceRead] = {} - self._failed_source_paths: Set[str] = set() - @abstractmethod def extract_dataframe(self) -> pyspark.sql.DataFrame: pass - def isolate_failing_sources(self) -> None: - for source_path, source in list(self._sources.items()): - try: - source.dataframe.collect() - except Exception as e: - self._record_source_failure(source.file, e) - self._sources.pop(source_path) - def _read_source(self, source_file: BaseFile, read_source: Callable[[], pyspark.sql.DataFrame]) -> Optional[pyspark.sql.DataFrame]: - if source_file.file_path in self._failed_source_paths: - return None - try: source_dataframe = read_source() - except Exception as e: - self._record_source_failure(source_file, e) - return None + source_dataframe.count() - self._sources[source_file.file_path] = SourceRead(file=source_file, dataframe=source_dataframe) + return source_dataframe - return source_dataframe + except Exception as e: + logger.warning(f"[{self._table_name}] Skipping unreadable file {source_file.file_path}", exc_info=True) + source_file.error = f"{self.SOURCE_ERROR_PREFIX}: {e}" - def _record_source_failure(self, source_file: BaseFile, error: Exception) -> None: - source_file.error = f"{self.SOURCE_ERROR_PREFIX}: {error}" - self._failed_source_paths.add(source_file.file_path) + return None From a0dacbc13456b4f848ebd8f141a7e3be10dbec4d Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 16:11:29 +0300 Subject: [PATCH 04/29] Fix None snapshot values --- backend/constants.py | 2 +- backend/table_inventory/table_inventory.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/constants.py b/backend/constants.py index 7ce68bd..5d392a9 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -9,7 +9,7 @@ COMPUTE_CLEANUP_TIME_SECONDS = 12 -MAX_DATA_FILES_TO_COLLECT = 5 +MAX_DATA_FILES_TO_COLLECT = 5_000 TABLE_LIST_CACHE_TTL_SECONDS = 60 diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index e9b4091..6f0f101 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -9,7 +9,12 @@ from collectors.collect_manifests import CollectManifests, ManifestRecord from collectors.collect_metadata import CollectMetadata, MetadataFileRecord from collectors.collect_snapshots import CollectSnapshots, SnapshotRecord -from constants import DATA_FILES_CUTOFF_MANIFEST_WARNING, DATA_FILES_CUTOFF_WARNING, FileType, MAX_DATA_FILES_TO_COLLECT +from constants import ( + DATA_FILES_CUTOFF_MANIFEST_WARNING, + DATA_FILES_CUTOFF_WARNING, + FileType, + MAX_DATA_FILES_TO_COLLECT, +) from icegraph_logger import logger from search_cutoff.find_search_cutoff import SearchCutoff, find_search_cutoff from table_inventory.utils import format_schemas_to_full_dict, get_json_metadata_from_path, parse_json_string_fields @@ -203,6 +208,9 @@ def _warn_if_data_cutoff_happened(self): if len(manifest.child_files) == 0: manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=max_data_files_to_collect) + if manifest.added_snapshot_timestamp is None: + continue + if max_manifest_added_snapshot_timestamp is None or max_manifest_added_snapshot_timestamp < manifest.added_snapshot_timestamp: max_manifest_added_snapshot_timestamp = manifest.added_snapshot_timestamp max_manifest_added_snapshot_id = manifest.added_snapshot_id From 1e1f3e2c3c6a7a6aef8c46dce27e39dd4201737f Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 17:56:30 +0300 Subject: [PATCH 05/29] Showing errors better --- backend/graph_normalizer/graph_normalizer.py | 4 +-- frontend/src/components/PanelIssueNotice.jsx | 35 ++++++++++++++++++++ frontend/src/pages/GraphPage.jsx | 12 ++++++- 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/PanelIssueNotice.jsx diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index 16cfc6f..aee6e38 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -20,8 +20,8 @@ def normalize(self): { "nodes": nodes, "metadata": self._current_table_metadata, - "errors": self._file_issues(lambda file: file.error) | self._table_errors, - "warnings": self._file_issues(lambda file: file.warning) | self._table_warnings, + "errors": self._table_errors, + "warnings": self._table_warnings, } ) diff --git a/frontend/src/components/PanelIssueNotice.jsx b/frontend/src/components/PanelIssueNotice.jsx new file mode 100644 index 0000000..db0e68b --- /dev/null +++ b/frontend/src/components/PanelIssueNotice.jsx @@ -0,0 +1,35 @@ +const ISSUE_STYLES = { + error: { + container: "border-red-500/60 bg-red-950/55", + label: "text-red-300", + message: "text-red-100", + }, + warning: { + container: "border-yellow-500/60 bg-yellow-950/45", + label: "text-yellow-300", + message: "text-yellow-100", + }, +}; + +export default function PanelIssueNotice({ type, children }) { + const styles = ISSUE_STYLES[type]; + if (!styles || children == null || children === "") return null; + + return ( +
    +
    + {type} +
    +
    + {String(children)} +
    +
    + ); +} diff --git a/frontend/src/pages/GraphPage.jsx b/frontend/src/pages/GraphPage.jsx index f094f6d..b6e47a7 100644 --- a/frontend/src/pages/GraphPage.jsx +++ b/frontend/src/pages/GraphPage.jsx @@ -7,6 +7,7 @@ import { PanelHeader, PANEL_STATUS_BADGE_CLASS, } from "../components/PanelContent"; +import PanelIssueNotice from "../components/PanelIssueNotice"; import { UI_DIALOG_SECTION_TITLE_CLASS, UI_POPUP_HINT_CLASS, @@ -683,7 +684,10 @@ export default function GraphPage() { const sticky = stickyNode ? { rows: Object.entries(stickyNode.details) - .filter(([label]) => label.toLowerCase() !== "type") + .filter( + ([label]) => + !["type", "error", "warning"].includes(label.toLowerCase()), + ) .map(([label, value]) => ({ label, value, @@ -832,6 +836,12 @@ export default function GraphPage() { {isInspectMode && ( 🔒 Locked View )} + + {stickyNode.details.error} + + + {stickyNode.details.warning} + {sticky.rows .filter((r) => !isEmptyValue(r.value)) .map((r, i) => ( From eee4d9f77e76884ede2e6de8d7b05c5ea607a36c Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 18:06:07 +0300 Subject: [PATCH 06/29] Cleaner code --- backend/graph_normalizer/graph_normalizer.py | 3 --- backend/table_inventory/table_inventory.py | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index aee6e38..be904d6 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -24,6 +24,3 @@ def normalize(self): "warnings": self._table_warnings, } ) - - def _file_issues(self, get_issue: Callable[[BaseFile], Optional[str]]) -> Dict[str, str]: - return {file.file_path: get_issue(file) for file in self._files if get_issue(file)} diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index 6f0f101..a8b2b79 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -70,6 +70,7 @@ def build(self): self._warn_if_data_cutoff_happened() self._set_current_table_specs() + self._collect_file_errors() return TableInventoryResult( errors=self._errors, @@ -222,6 +223,11 @@ def _warn_if_data_cutoff_happened(self): added_snapshot_timestamp=max_manifest_added_snapshot_timestamp, ) + def _collect_file_errors(self): + file_groups = (self._metadata_files, self._snapshots, self._manifests, self._data_files) + for files in file_groups: + self._errors.update({file.file_path: file.error for file in files if file.error}) + def _set_current_table_specs(self): self._current_table_specs = {"table-name": self._table_name} From 48a82f79a570076b0c456a70eea9e460120a9b66 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 18:07:53 +0300 Subject: [PATCH 07/29] Better msg --- backend/constants.py | 2 +- backend/graph_normalizer/graph_normalizer.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/constants.py b/backend/constants.py index 5d392a9..f91d898 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -55,5 +55,5 @@ class FileType(Enum): """) DATA_FILES_CUTOFF_MANIFEST_WARNING = inspect.cleandoc(""" -The data files of this manifest were not loaded because the limit of {max_data_files_to_collect} data files was reached. +The data files of the manifest were not loaded/attached because the limit of {max_data_files_to_collect} data files was reached. """) diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index be904d6..a79c4ef 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -1,6 +1,3 @@ -from typing import Callable, Dict, Optional - -from base_classes.base_file import BaseFile from table_inventory.table_inventory import TableInventoryResult from graph_normalizer.utils import to_json_safe From 1daab2d5262ef494f2f73ce0e576b07fe4b797e1 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 18:12:37 +0300 Subject: [PATCH 08/29] Remove not needed const --- backend/extractors/constants.py | 3 --- backend/extractors/data_files_extractor.py | 3 --- backend/extractors/extractor.py | 5 +---- backend/extractors/manifests_extractor.py | 3 --- 4 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 backend/extractors/constants.py diff --git a/backend/extractors/constants.py b/backend/extractors/constants.py deleted file mode 100644 index 5cab1f3..0000000 --- a/backend/extractors/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_SOURCE_ERROR_PREFIX = "Failed to read file" -MANIFEST_LIST_SOURCE_ERROR_PREFIX = "Failed to read manifest list" -MANIFEST_SOURCE_ERROR_PREFIX = "Failed to read manifest" diff --git a/backend/extractors/data_files_extractor.py b/backend/extractors/data_files_extractor.py index 1f0e4b5..1c271a8 100644 --- a/backend/extractors/data_files_extractor.py +++ b/backend/extractors/data_files_extractor.py @@ -7,7 +7,6 @@ from collectors.collect_manifests import ManifestRecord from constants import MAX_DATA_FILES_TO_COLLECT -from extractors.constants import MANIFEST_SOURCE_ERROR_PREFIX from extractors.extractor import Extractor max_data_files_to_collect = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", MAX_DATA_FILES_TO_COLLECT)) @@ -41,8 +40,6 @@ class DataFilesExtractor(Extractor): - SOURCE_ERROR_PREFIX = MANIFEST_SOURCE_ERROR_PREFIX - def __init__(self, table_name: str, manifest_entries: List[ManifestRecord]): super().__init__(table_name) self._manifest_entries = manifest_entries diff --git a/backend/extractors/extractor.py b/backend/extractors/extractor.py index 757b6aa..c71b796 100644 --- a/backend/extractors/extractor.py +++ b/backend/extractors/extractor.py @@ -5,13 +5,10 @@ from base_classes.base_file import BaseFile from base_classes.spark_table_action import SparkTableAction -from extractors.constants import DEFAULT_SOURCE_ERROR_PREFIX from icegraph_logger import logger class Extractor(SparkTableAction, ABC): - SOURCE_ERROR_PREFIX = DEFAULT_SOURCE_ERROR_PREFIX - @abstractmethod def extract_dataframe(self) -> pyspark.sql.DataFrame: pass @@ -25,6 +22,6 @@ def _read_source(self, source_file: BaseFile, read_source: Callable[[], pyspark. except Exception as e: logger.warning(f"[{self._table_name}] Skipping unreadable file {source_file.file_path}", exc_info=True) - source_file.error = f"{self.SOURCE_ERROR_PREFIX}: {e}" + source_file.error = str(e) return None diff --git a/backend/extractors/manifests_extractor.py b/backend/extractors/manifests_extractor.py index 0411e47..ba6d25c 100644 --- a/backend/extractors/manifests_extractor.py +++ b/backend/extractors/manifests_extractor.py @@ -3,7 +3,6 @@ from pyspark.sql.types import LongType, StringType, StructField, StructType from collectors.collect_snapshots import SnapshotRecord -from extractors.constants import MANIFEST_LIST_SOURCE_ERROR_PREFIX from extractors.extractor import Extractor SNAPSHOT_TO_TIMESTAMP_SCHEMA = StructType( @@ -23,8 +22,6 @@ class ManifestsExtractor(Extractor): - SOURCE_ERROR_PREFIX = MANIFEST_LIST_SOURCE_ERROR_PREFIX - def __init__( self, table_name: str, From 97f27fb964db18e0ec4c1ba05ccd758b2933a5a0 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 18:44:21 +0300 Subject: [PATCH 09/29] Simpler extract --- backend/extractors/extractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/extractors/extractor.py b/backend/extractors/extractor.py index c71b796..fcfbd75 100644 --- a/backend/extractors/extractor.py +++ b/backend/extractors/extractor.py @@ -15,13 +15,13 @@ def extract_dataframe(self) -> pyspark.sql.DataFrame: def _read_source(self, source_file: BaseFile, read_source: Callable[[], pyspark.sql.DataFrame]) -> Optional[pyspark.sql.DataFrame]: try: - source_dataframe = read_source() - source_dataframe.count() + data = read_source() + data.schema # Trigger the file metadata read - return source_dataframe + return data except Exception as e: - logger.warning(f"[{self._table_name}] Skipping unreadable file {source_file.file_path}", exc_info=True) + logger.error(f"[{self._table_name}] Failed to read file {source_file.file_path}", exc_info=True) source_file.error = str(e) return None From e2b0232a05c3e8b0ad3365c03369c08d0421e69f Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Thu, 13 Aug 2026 19:11:10 +0300 Subject: [PATCH 10/29] Showing bad metadata files --- backend/collectors/collect_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 93369f5..ea0c5ec 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -58,9 +58,9 @@ def collect(self) -> FilesCollection: snap_id_to_path = self._get_snap_id_to_path() rows = metadata_files_df.orderBy(F.desc("metadata_timestamp")).collect() - self._metadata_files = [ + self._metadata_files.extend( self._parse_metadata_row(index, row.asDict(recursive=True), rows, snap_id_to_path) for index, row in enumerate(rows) - ] + ) except Exception as e: logger.error(f"[{self._table_name}] metadata collection failed", exc_info=True) @@ -91,7 +91,7 @@ def _build_metadata_files_df(self, metadata_files: dict) -> Optional[pyspark.sql f"[{self._table_name}] Metadata file read error for {file}", exc_info=True, ) - self._errors[file] = f"Metadata file read error: {e}" + self._metadata_files.append(BaseFile(type=FileType.METADATA, file_path=file, child_files=[], error=str(e))) return metadata_files_df From 578886c09cfb729f8a07b86a6fd1f8e3dd814395 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 00:34:00 +0300 Subject: [PATCH 11/29] Better desc --- frontend/src/pages/DocsPage.jsx | 4 +++- frontend/src/pages/TableLayout.jsx | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index 23c5bb7..d72134e 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -448,7 +448,9 @@ const SECTIONS = [
  • Data file — the actual - Parquet, ORC, or Avro file containing your rows + Parquet, ORC, or Avro file containing your rows. IceGraph is not + reading the data file: all of the data shown comes from the manifest + entries that point at it
  • Unreadable file — a file diff --git a/frontend/src/pages/TableLayout.jsx b/frontend/src/pages/TableLayout.jsx index 216b881..6449fd5 100644 --- a/frontend/src/pages/TableLayout.jsx +++ b/frontend/src/pages/TableLayout.jsx @@ -251,7 +251,9 @@ export default function TableLayout() { level: 0, }; const [r, g, b] = details.error ? ERROR_NODE_RGB : style.rgb; - const colorShift = colorShiftByFilePath.get(details.file_path) ?? 1; + const colorShift = details.error + ? 1 + : (colorShiftByFilePath.get(details.file_path) ?? 1); return { id: details.file_path, From 94db58d60c6054028c041ca38602a96f493664bd Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 00:47:44 +0300 Subject: [PATCH 12/29] Better desc --- frontend/src/pages/DocsPage.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index d72134e..c061b73 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -453,7 +453,7 @@ const SECTIONS = [ entries that point at it
  • - Unreadable file — a file + Unreadable file — a file whose metadata could not be obtained is drawn in red and shows the reason in its details panel. The rest of the graph still loads
    • From 3011605d33ce22d68bddf2d96c5f7d2ee51ac837 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 02:31:29 +0300 Subject: [PATCH 13/29] Metadata location fix --- backend/collectors/collect_metadata.py | 79 ++++++++++++++++++++------ frontend/src/pages/DocsPage.jsx | 4 +- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index ea0c5ec..209c187 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -1,3 +1,4 @@ +from google.protobuf.internal import message_listener from base_classes.utils import column_to_string_utc import json from dataclasses import dataclass @@ -18,17 +19,17 @@ @dataclass class MetadataFileRecord(BaseFile): - timestamp: str + timestamp: Optional[str] snapshot_id: Optional[int] previous_file: Optional[str] last_sequence_number: Optional[int] - partition_spec_id: int - current_schema_id: int - sort_order_id: int + partition_spec_id: Optional[int] + current_schema_id: Optional[int] + sort_order_id: Optional[int] refs: Dict[str, Any] properties: Dict[str, str] pointed_snapshots_files: Optional[List[Dict[str, str]]] - pointed_metadata_log_count: int + pointed_metadata_log_count: Optional[int] class CollectMetadata(Collector): @@ -45,22 +46,29 @@ def __init__( self._start_metadata_cutoff = start_metadata_cutoff self._end_metadata_cutoff = end_metadata_cutoff + self._ordered_metadata_to_timestamp: Dict[str, str] = {} + self._metadata_files: List[MetadataFileRecord] = [] + self._bad_metadata_files: List[MetadataFileRecord] = [] self._errors: Dict[str, str] = {} @timed def collect(self) -> FilesCollection: try: - metadata_files = self._query_metadata_files() - metadata_files_df = self._build_metadata_files_df(metadata_files) + self._ordered_metadata_to_timestamp = self._query_metadata_files() + metadata_files_df = self._build_metadata_files_df() if metadata_files_df is not None: snap_id_to_path = self._get_snap_id_to_path() rows = metadata_files_df.orderBy(F.desc("metadata_timestamp")).collect() - self._metadata_files.extend( - self._parse_metadata_row(index, row.asDict(recursive=True), rows, snap_id_to_path) for index, row in enumerate(rows) - ) + for index, row in enumerate(rows): + row_dict = row.asDict(recursive=True) + metadata_file_type = FileType.MAIN_METADATA if index == 0 else FileType.METADATA + + self._metadata_files.append(self._parse_metadata_row(metadata_file_type, row_dict, snap_id_to_path)) + + self._add_bad_metadata_files() except Exception as e: logger.error(f"[{self._table_name}] metadata collection failed", exc_info=True) @@ -68,6 +76,10 @@ def collect(self) -> FilesCollection: return FilesCollection(files=self._metadata_files, errors=self._errors) + @property + def _ordered_metadata_paths(self) -> list[str]: + return list(self._ordered_metadata_to_timestamp.keys()) + def _query_metadata_files(self) -> dict: metadata_df = ( self._spark.sql(f"SELECT * FROM {self._table_name}.metadata_log_entries") @@ -79,9 +91,9 @@ def _query_metadata_files(self) -> dict: ) return {row.file: row.metadata_timestamp for row in metadata_df.collect()} - def _build_metadata_files_df(self, metadata_files: dict) -> Optional[pyspark.sql.DataFrame]: + def _build_metadata_files_df(self) -> Optional[pyspark.sql.DataFrame]: metadata_files_df = None - for file, timestamp in metadata_files.items(): + for file, timestamp in self._ordered_metadata_to_timestamp.items(): try: df = get_metadata_row_slim_df_from_path(file).withColumn("metadata_timestamp", F.lit(timestamp)).withColumn("file", F.lit(file)) metadata_files_df = df if metadata_files_df is None else metadata_files_df.unionByName(df, allowMissingColumns=True) @@ -91,13 +103,47 @@ def _build_metadata_files_df(self, metadata_files: dict) -> Optional[pyspark.sql f"[{self._table_name}] Metadata file read error for {file}", exc_info=True, ) - self._metadata_files.append(BaseFile(type=FileType.METADATA, file_path=file, child_files=[], error=str(e))) + self._bad_metadata_files.append( + MetadataFileRecord( + type=FileType.METADATA, + file_path=file, + child_files=[], + error=str(e), + timestamp=timestamp, + snapshot_id=None, + previous_file=None, + last_sequence_number=None, + partition_spec_id=None, + current_schema_id=None, + sort_order_id=None, + refs={}, + properties={}, + pointed_snapshots_files=None, + pointed_metadata_log_count=None, + ) + ) return metadata_files_df + def _add_bad_metadata_files(self) -> None: + for bad_file in self._bad_metadata_files: + index_to_insert = len(self._metadata_files) + + for index, metadata_file in enumerate(self._metadata_files): + if metadata_file.previous_file == bad_file.file_path: + index_to_insert = index + 1 + break + + self._metadata_files.insert(index_to_insert, bad_file) + def _get_snap_id_to_path(self) -> Dict[int, str]: return {s.snapshot_id: s.file_path for s in (self._snapshots or [])} + def _get_previous_metadata_file(self, file_path: str) -> Optional[str]: + file_index = self._ordered_metadata_paths.index(file_path) + + return self._ordered_metadata_paths[file_index - 1] if file_index - 1 >= 0 else None + @staticmethod def _parse_refs(row: dict) -> dict: return json.loads(row["refs"]) if row.get("refs") else {} @@ -115,10 +161,7 @@ def _build_branches_child_files(refs: dict, snap_id_to_path: dict) -> List[str]: return branches_child_files - def _parse_metadata_row(self, index: int, row: dict, rows: list, snap_id_to_path: dict) -> MetadataFileRecord: - number_of_rows = len(rows) - file_type = FileType.MAIN_METADATA if index == 0 else FileType.METADATA - + def _parse_metadata_row(self, file_type: FileType, row: dict, snap_id_to_path: dict) -> MetadataFileRecord: refs = self._parse_refs(row) branches_child_files = self._build_branches_child_files(refs, snap_id_to_path) @@ -130,7 +173,7 @@ def _parse_metadata_row(self, index: int, row: dict, rows: list, snap_id_to_path file_path=row["file"], timestamp=str(row["metadata_timestamp"]), snapshot_id=row["current-snapshot-id"], - previous_file=(rows[index + 1]["file"] if index + 1 < number_of_rows else None), + previous_file=self._get_previous_metadata_file(row["file"]), last_sequence_number=(row["last-sequence-number"] if "last-sequence-number" in row else None), partition_spec_id=row["default-spec-id"], current_schema_id=row["current-schema-id"], diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index c061b73..a609cf9 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -449,8 +449,8 @@ const SECTIONS = [
  • Data file — the actual Parquet, ORC, or Avro file containing your rows. IceGraph is not - reading the data file: all of the data shown comes from the manifest - entries that point at it + reading the data file: all of the data shown comes from the + manifest entries that point at it
  • Unreadable file — a file From 6156c353bcabae2f6ae40777659a37594a9c9d96 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 02:37:33 +0300 Subject: [PATCH 14/29] Better caching --- backend/collectors/collect_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 209c187..2215003 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -2,6 +2,7 @@ from base_classes.utils import column_to_string_utc import json from dataclasses import dataclass +from functools import cached_property from typing import Any, Dict, List, Optional import pyspark.sql @@ -76,7 +77,7 @@ def collect(self) -> FilesCollection: return FilesCollection(files=self._metadata_files, errors=self._errors) - @property + @cached_property def _ordered_metadata_paths(self) -> list[str]: return list(self._ordered_metadata_to_timestamp.keys()) From 33dddbb7df8aa5137679bcd2af1ebaeaff8b8276 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 02:45:11 +0300 Subject: [PATCH 15/29] rm import --- backend/collectors/collect_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 2215003..5c8d41d 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -1,4 +1,3 @@ -from google.protobuf.internal import message_listener from base_classes.utils import column_to_string_utc import json from dataclasses import dataclass From 01f7973471047679553722f1addd5b1e4366d6c7 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 03:01:11 +0300 Subject: [PATCH 16/29] Not display warnning for nothing in manifests --- backend/table_inventory/table_inventory.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index a8b2b79..462480f 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -206,15 +206,14 @@ def _warn_if_data_cutoff_happened(self): max_manifest_added_snapshot_id = None for manifest in self._manifests: - if len(manifest.child_files) == 0: - manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=max_data_files_to_collect) + if manifest.child_files or manifest.error or manifest.added_snapshot_timestamp is None: + continue - if manifest.added_snapshot_timestamp is None: - continue + manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=max_data_files_to_collect) - if max_manifest_added_snapshot_timestamp is None or max_manifest_added_snapshot_timestamp < manifest.added_snapshot_timestamp: - max_manifest_added_snapshot_timestamp = manifest.added_snapshot_timestamp - max_manifest_added_snapshot_id = manifest.added_snapshot_id + if max_manifest_added_snapshot_timestamp is None or max_manifest_added_snapshot_timestamp < manifest.added_snapshot_timestamp: + max_manifest_added_snapshot_timestamp = manifest.added_snapshot_timestamp + max_manifest_added_snapshot_id = manifest.added_snapshot_id if max_manifest_added_snapshot_timestamp is not None: self._warnings["data_files_cutoff"] = DATA_FILES_CUTOFF_WARNING.format( From ffcd65f34bc62f0fa9ab2cfd271a1d2b5c8d1f19 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 11:20:27 +0300 Subject: [PATCH 17/29] Added ruff --- .github/workflows/ci.yml | 17 ++++++++++ README.md | 27 ++++++++++++++- backend/collectors/collect_data_files.py | 2 +- backend/collectors/collect_snapshots.py | 2 +- backend/graph_normalizer/graph_normalizer.py | 1 - backend/pyproject.toml | 10 ++++-- backend/search_cutoff/find_search_cutoff.py | 32 +++++++++++++----- backend/table_inventory/table_inventory.py | 1 - .../table_list_catalog/table_list_catalog.py | 7 +++- backend/uv.lock | 33 +++++++++++++++++++ icegraph-client/icegraph_client/cli.py | 4 ++- icegraph-client/pyproject.toml | 7 +++- icegraph-client/uv.lock | 33 +++++++++++++++++++ 13 files changed, 158 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70f504e..5e245f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,23 @@ on: branches: [master] jobs: + python-format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check backend formatting + uses: astral-sh/ruff-action@v4.1.0 + with: + version: "0.16.0" + args: format --check --diff + src: backend + - name: Check icegraph-client formatting + uses: astral-sh/ruff-action@v4.1.0 + with: + version: "0.16.0" + args: format --check --diff + src: icegraph-client + frontend: runs-on: ubuntu-latest defaults: diff --git a/README.md b/README.md index 1462f78..83da667 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,11 @@ cd backend uv sync ``` +```bash +cd icegraph-client +uv sync +``` + ```bash cd frontend npm i @@ -136,9 +141,29 @@ Go to `http://localhost:3000` and explore your tables. ### 4. Before Every Commit -CI fails the build on any violation, so run all three from the `frontend` directory before you commit: +CI checks Python formatting and the frontend toolchain. Format each Python project from its own directory. + +Backend: + +```bash +cd backend + +uv run ruff format . +``` + +Python client: + +```bash +cd icegraph-client + +uv run ruff format . +``` + +Run all three frontend checks from the `frontend` directory: ```bash +cd frontend + npm run format npm run lint npm run typecheck diff --git a/backend/collectors/collect_data_files.py b/backend/collectors/collect_data_files.py index 724c771..2dead6e 100644 --- a/backend/collectors/collect_data_files.py +++ b/backend/collectors/collect_data_files.py @@ -56,7 +56,7 @@ def _process_data_file_row(self, data_file_row) -> DataFileRecord: type=self._detect_file_type(data_file_dict["content"]), file_path=data_file_dict["file_path"], format=data_file_dict["file_format"], - size_gb=f"{(data_file_dict['file_size_in_bytes'] / 1024 ** 3):.10f}", + size_gb=f"{(data_file_dict['file_size_in_bytes'] / 1024**3):.10f}", row_count=data_file_dict["record_count"], partition=format_partition(data_file_dict["partition"]), earliest_appearing_snapshot_id=data_file_dict["earliest_snapshot_id"], diff --git a/backend/collectors/collect_snapshots.py b/backend/collectors/collect_snapshots.py index 461a5f2..a81a11b 100644 --- a/backend/collectors/collect_snapshots.py +++ b/backend/collectors/collect_snapshots.py @@ -68,7 +68,7 @@ def _validate_snapshot_count(snapshots_df: pyspark.sql.DataFrame) -> None: @staticmethod def _format_summary(summary: dict) -> dict: - return {k: (f"{(int(v) / (1024 ** 3)):.5f} GB" if k.endswith("files-size") else v) for k, v in summary.items()} + return {k: (f"{(int(v) / (1024**3)):.5f} GB" if k.endswith("files-size") else v) for k, v in summary.items()} def _parse_snapshot_row(self, snapshot) -> SnapshotRecord: return SnapshotRecord( diff --git a/backend/graph_normalizer/graph_normalizer.py b/backend/graph_normalizer/graph_normalizer.py index a79c4ef..68fc815 100644 --- a/backend/graph_normalizer/graph_normalizer.py +++ b/backend/graph_normalizer/graph_normalizer.py @@ -3,7 +3,6 @@ class GraphNormalizer: - def __init__(self, table_data: TableInventoryResult): self._files = table_data.metadata_files + table_data.snapshots + table_data.manifests + table_data.data_files self._table_errors = table_data.errors diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c8d718a..4b7b9db 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -16,5 +16,11 @@ dependencies = [ "setuptools>=75.0.0", "waitress==3.0.2", ] -[tool.black] -line-length = 150 \ No newline at end of file + +[dependency-groups] +dev = [ + "ruff==0.16.0", +] + +[tool.ruff] +line-length = 150 diff --git a/backend/search_cutoff/find_search_cutoff.py b/backend/search_cutoff/find_search_cutoff.py index a247e4b..26ec510 100644 --- a/backend/search_cutoff/find_search_cutoff.py +++ b/backend/search_cutoff/find_search_cutoff.py @@ -65,11 +65,15 @@ def _get_start_cutoffs( table_name: str, start_snapshot_id: int, ) -> StartCutoffs: - row = spark.sql(f""" + row = ( + spark.sql(f""" SELECT committed_at, parent_id FROM {table_name}.snapshots WHERE snapshot_id = {start_snapshot_id} - """).withColumn("committed_at", column_to_string_utc("committed_at")).first() + """) + .withColumn("committed_at", column_to_string_utc("committed_at")) + .first() + ) if not row: return StartCutoffs( @@ -80,11 +84,15 @@ def _get_start_cutoffs( snapshot_cutoff = to_arrow_utc(row.committed_at) - meta_row = spark.sql(f""" + meta_row = ( + spark.sql(f""" SELECT MIN(timestamp) AS ts FROM {table_name}.metadata_log_entries WHERE latest_snapshot_id = {start_snapshot_id} - """).withColumn("ts", column_to_string_utc("ts")).first() + """) + .withColumn("ts", column_to_string_utc("ts")) + .first() + ) metadata_cutoff = to_arrow_utc(meta_row.ts) if meta_row and meta_row.ts else snapshot_cutoff @@ -102,11 +110,15 @@ def _get_end_cutoffs( table_name: str, end_snapshot_id: int, ) -> EndCutoffs: - row = spark.sql(f""" + row = ( + spark.sql(f""" SELECT committed_at FROM {table_name}.snapshots WHERE snapshot_id = {end_snapshot_id} - """).withColumn("committed_at", column_to_string_utc("committed_at")).first() + """) + .withColumn("committed_at", column_to_string_utc("committed_at")) + .first() + ) if not row: return EndCutoffs( @@ -116,11 +128,15 @@ def _get_end_cutoffs( snapshot_cutoff = to_arrow_utc(row.committed_at) - meta_row = spark.sql(f""" + meta_row = ( + spark.sql(f""" SELECT MAX(timestamp) AS ts FROM {table_name}.metadata_log_entries WHERE latest_snapshot_id = {end_snapshot_id} - """).withColumn("ts", column_to_string_utc("ts")).first() + """) + .withColumn("ts", column_to_string_utc("ts")) + .first() + ) metadata_cutoff = to_arrow_utc(meta_row.ts) if meta_row and meta_row.ts else snapshot_cutoff diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index 462480f..4806421 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -163,7 +163,6 @@ def _attach_snapshot_files_to_manifest_files(self): for manifest in self._manifests: for snapshot_id in manifest.hidden_manifest_data.pointing_snapshots: - snapshot = snapshot_id_to_snapshot_file_map.get(snapshot_id) if not snapshot: self._errors[f"Linking {snapshot_id} -> {manifest.file_path}"] = "Snapshot not found" diff --git a/backend/table_list_catalog/table_list_catalog.py b/backend/table_list_catalog/table_list_catalog.py index 3865a31..ab96289 100644 --- a/backend/table_list_catalog/table_list_catalog.py +++ b/backend/table_list_catalog/table_list_catalog.py @@ -8,7 +8,12 @@ from base_classes.utils import timed from constants import TABLE_LIST_CACHE_TTL_SECONDS from spark_connect import open_spark_connect_session -from table_list_catalog.utils import collect_catalogs_tables_names, collect_databases_in_catalogs, list_catalog_names, filter_catalogs_to_include +from table_list_catalog.utils import ( + collect_catalogs_tables_names, + collect_databases_in_catalogs, + list_catalog_names, + filter_catalogs_to_include, +) table_list_cache_ttl_seconds = int(os.getenv("TABLE_LIST_CACHE_TTL_SECONDS", TABLE_LIST_CACHE_TTL_SECONDS)) diff --git a/backend/uv.lock b/backend/uv.lock index 5bb38f8..6cced5d 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -146,6 +146,11 @@ dependencies = [ { name = "waitress" }, ] +[package.dev-dependencies] +dev = [ + { name = "ruff" }, +] + [package.metadata] requires-dist = [ { name = "arrow", specifier = "==1.4.0" }, @@ -160,6 +165,9 @@ requires-dist = [ { name = "waitress", specifier = "==3.0.2" }, ] +[package.metadata.requires-dev] +dev = [{ name = "ruff", specifier = "==0.16.0" }] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -423,6 +431,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, ] +[[package]] +name = "ruff" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, + { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, + { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, +] + [[package]] name = "setuptools" version = "82.0.1" diff --git a/icegraph-client/icegraph_client/cli.py b/icegraph-client/icegraph_client/cli.py index 1591a59..3a026ee 100644 --- a/icegraph-client/icegraph_client/cli.py +++ b/icegraph-client/icegraph_client/cli.py @@ -33,7 +33,9 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="icegraph", description="CLI for the IceGraph server API") parser.add_argument("--base-url", default=None, help=f"IceGraph server URL. Falls back to the {BASE_URL_ENV_VAR} environment variable") parser.add_argument( - "--token", default=None, help=f"Bearer token, for servers that require authentication. Falls back to the {TOKEN_ENV_VAR} environment variable" + "--token", + default=None, + help=f"Bearer token, for servers that require authentication. Falls back to the {TOKEN_ENV_VAR} environment variable", ) parser.add_argument( "--cookie", diff --git a/icegraph-client/pyproject.toml b/icegraph-client/pyproject.toml index 6d2f185..869ae53 100644 --- a/icegraph-client/pyproject.toml +++ b/icegraph-client/pyproject.toml @@ -24,5 +24,10 @@ root = ".." tag_regex = "^v(?P.*)$" git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--match", "v*"] -[tool.black] +[dependency-groups] +dev = [ + "ruff==0.16.0", +] + +[tool.ruff] line-length = 150 diff --git a/icegraph-client/uv.lock b/icegraph-client/uv.lock index b9f0f2a..6e27c6a 100644 --- a/icegraph-client/uv.lock +++ b/icegraph-client/uv.lock @@ -137,12 +137,20 @@ dependencies = [ { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +[package.dev-dependencies] +dev = [ + { name = "ruff" }, +] + [package.metadata] requires-dist = [ { name = "arrow", specifier = ">=1.0" }, { name = "requests", specifier = ">=2.25" }, ] +[package.metadata.requires-dev] +dev = [{ name = "ruff", specifier = "==0.16.0" }] + [[package]] name = "idna" version = "3.18" @@ -200,6 +208,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "ruff" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, + { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, + { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, +] + [[package]] name = "six" version = "1.17.0" From f975a765fac5d78d1eada6867f24a8f5f97b835d Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 11:31:40 +0300 Subject: [PATCH 18/29] File tree visual error --- frontend/src/pages/DocsPage.jsx | 4 +++ frontend/src/pages/FileTreePage.jsx | 41 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index a609cf9..c8d99f5 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -400,6 +400,10 @@ const SECTIONS = [
    • Expand directories to see individual files
    • Choose a branch, and within it, a snapshot to explore
      • +
    • + If the selected snapshot or one of its included files could not be + read, an error notice identifies the file and explains why +
    • Many small files in one partition path often indicates a small-file problem diff --git a/frontend/src/pages/FileTreePage.jsx b/frontend/src/pages/FileTreePage.jsx index bbe0088..a0ebc84 100644 --- a/frontend/src/pages/FileTreePage.jsx +++ b/frontend/src/pages/FileTreePage.jsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { useOutletContext } from "react-router-dom"; -import { FileType } from "../graphConstants"; +import PanelIssueNotice from "../components/PanelIssueNotice"; +import { FileType, fileTypeLabel } from "../graphConstants"; import { useViewInGraph } from "../hooks/useViewInGraph"; import { UI_BODY_MUTED_ITALIC_CLASS, @@ -115,6 +116,33 @@ function buildTree(partitions) { return root; } +function getSnapshotFileErrors(snapshot, adjacency, nodeById) { + if (!snapshot) return []; + + const errors = []; + const visited = new Set(); + const queue = [snapshot.id]; + + while (queue.length > 0) { + const current = queue.shift(); + if (visited.has(current)) continue; + visited.add(current); + + const node = nodeById[current]; + if (!node) continue; + + if (node.details?.error) { + errors.push( + `${fileTypeLabel(node.type)} (${node.label || node.id}): ${node.details.error}`, + ); + } + + for (const { to } of adjacency[current] || []) queue.push(to); + } + + return errors; +} + function FileRow({ filePath, checkedFiles, @@ -612,6 +640,11 @@ export default function FileTreePage() { } const currentSnapshot = displayedSnapshots[effectiveIdx]; + const snapshotFileErrors = getSnapshotFileErrors( + currentSnapshot, + adjacency, + nodeById, + ); return (
      @@ -950,6 +983,12 @@ export default function FileTreePage() {
      + {snapshotFileErrors.length > 0 && ( + + {snapshotFileErrors.join("\n")} + + )} + {totalPartitions === 0 && (

      {search From 85e69a077bb912e97db5b1b8381609eff10c2304 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 12:01:28 +0300 Subject: [PATCH 19/29] Timeline unknown event --- frontend/src/pages/DocsPage.jsx | 6 ++++++ frontend/src/pages/TimelinePage.jsx | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index c8d99f5..c9c46ff 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -325,6 +325,12 @@ const SECTIONS = [ Use the Timeline to pinpoint when a large write happened, spot unexpected deletes, or verify that a compaction job ran as expected.

      +

      + A red Unknown Events marker + appears when metadata or snapshot data is missing. It indicates that + one or more events occurred in that part of the timeline, even when + the exact events cannot be determined. +

      Zoom & pan

      diff --git a/frontend/src/pages/TimelinePage.jsx b/frontend/src/pages/TimelinePage.jsx index 5963d60..1f41de6 100644 --- a/frontend/src/pages/TimelinePage.jsx +++ b/frontend/src/pages/TimelinePage.jsx @@ -30,6 +30,7 @@ const COLOR_A = "#1964B9"; const COLOR_B = "#6437D2"; const COLOR_C = "#0EA5E9"; const COLOR_INIT = "#D97706"; +const COLOR_ERROR = "#B9233C"; function formatTs(tsStr) { if (!tsStr) return null; @@ -124,6 +125,7 @@ function colorFor(type) { if (type === "A") return COLOR_A; if (type === "B") return COLOR_B; if (type === "C") return COLOR_C; + if (type === "error") return COLOR_ERROR; return COLOR_INIT; } @@ -131,6 +133,7 @@ function labelFor(type) { if (type === "A") return "Write"; if (type === "B") return "Metadata Op"; if (type === "C") return "Branch Write"; + if (type === "error") return "Unknown Events"; return "Init"; } @@ -594,6 +597,17 @@ export default function TimelinePage() { type = "C"; } + const snapshotId = type === "C" ? branchSnapId : details.snapshot_id; + const referencedSnapshot = snapMap[snapshotId]; + const isSnapshotMissing = + snapshotId != null && + String(snapshotId) !== "-1" && + (!referencedSnapshot || referencedSnapshot.error); + + if (details.error || isSnapshotMissing) { + type = "error"; + } + const diff = (type === "B" || type === "C") && prev ? Object.keys(details) @@ -605,7 +619,7 @@ export default function TimelinePage() { details, type, diff, - snapshotId: type === "C" ? branchSnapId : details.snapshot_id, + snapshotId, branchName, metadataNodeId, }; @@ -776,6 +790,7 @@ export default function TimelinePage() { ["A", "Write"], ["B", "Metadata Op"], ["C", "Branch Write"], + ["error", "Unknown Events"], ].map(([type, lbl]) => (

      Date: Fri, 14 Aug 2026 12:19:30 +0300 Subject: [PATCH 20/29] Timeline after unknown event --- frontend/src/pages/DocsPage.jsx | 5 ++- frontend/src/pages/TimelinePage.jsx | 54 +++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index c9c46ff..f366fa7 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -329,7 +329,10 @@ const SECTIONS = [ A red Unknown Events marker appears when metadata or snapshot data is missing. It indicates that one or more events occurred in that part of the timeline, even when - the exact events cannot be determined. + the exact events cannot be determined. The next readable event is + compared with the previous readable metadata. Its details show the + metadata changes, and when the snapshot changed they also show that + snapshot's operation.

      Zoom & pan

      diff --git a/frontend/src/pages/TimelinePage.jsx b/frontend/src/pages/TimelinePage.jsx index 1f41de6..cfd1e3c 100644 --- a/frontend/src/pages/TimelinePage.jsx +++ b/frontend/src/pages/TimelinePage.jsx @@ -565,26 +565,27 @@ export default function TimelinePage() { } }); - const timeline = metaNodes.map(({ details, id: metadataNodeId }, i) => { - const prev = i > 0 ? metaNodes[i - 1].details : null; - let type = !prev + let previousValidDetails = null; + const timeline = metaNodes.map(({ details, id: metadataNodeId }) => { + const previousDetails = previousValidDetails; + let type = !previousDetails ? "init" - : details.snapshot_id !== prev.snapshot_id + : details.snapshot_id !== previousDetails.snapshot_id ? "A" : "B"; let branchSnapId = null; let branchName = null; - if (prev && details.refs && prev.refs) { + if (previousDetails && details.refs && previousDetails.refs) { const currentRefs = details.refs; - const prevRefs = prev.refs; + const previousRefs = previousDetails.refs; - for (const key of Object.keys(prevRefs)) { - if (currentRefs[key] && prevRefs[key]) { + for (const key of Object.keys(previousRefs)) { + if (currentRefs[key] && previousRefs[key]) { const currentSnapId = currentRefs[key]["snapshot-id"]; - const prevSnapId = prevRefs[key]["snapshot-id"]; - if (currentSnapId !== prevSnapId) { + const previousSnapId = previousRefs[key]["snapshot-id"]; + if (currentSnapId !== previousSnapId) { branchSnapId = currentSnapId; branchName = key; break; @@ -609,13 +610,24 @@ export default function TimelinePage() { } const diff = - (type === "B" || type === "C") && prev - ? Object.keys(details) - .filter((k) => hasFieldChanged(details[k], prev[k])) - .map((k) => ({ key: k, before: prev[k], after: details[k] })) + type !== "error" && previousDetails + ? Array.from( + new Set([ + ...Object.keys(previousDetails), + ...Object.keys(details), + ]), + ) + .filter((key) => + hasFieldChanged(previousDetails[key], details[key]), + ) + .map((key) => ({ + key, + before: previousDetails[key], + after: details[key], + })) : []; - return { + const event = { details, type, diff, @@ -623,6 +635,12 @@ export default function TimelinePage() { branchName, metadataNodeId, }; + + if (type !== "error") { + previousValidDetails = details; + } + + return event; }); return { @@ -985,6 +1003,12 @@ export default function TimelinePage() { )} +
      + + Metadata Changes + + +
      )} From d09ec183cf7b51f64f51471bd7ebd3643710d78c Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 16:15:06 +0300 Subject: [PATCH 21/29] Add centralized Env configuration --- ARCHITECTURE_PHILOSOPHY.md | 2 +- CLAUDE.md | 4 +- README.md | 13 +----- backend/collectors/collect_snapshots.py | 13 +++--- backend/constants.py | 20 --------- backend/env.py | 39 ++++++++++++++++ backend/extractors/data_files_extractor.py | 12 +++-- backend/main.py | 45 ++++++------------- backend/table_inventory/table_inventory.py | 15 ++----- .../table_list_catalog/table_list_catalog.py | 14 +++--- backend/table_list_catalog/utils.py | 11 ++--- 11 files changed, 80 insertions(+), 108 deletions(-) create mode 100644 backend/env.py diff --git a/ARCHITECTURE_PHILOSOPHY.md b/ARCHITECTURE_PHILOSOPHY.md index 8920a28..b265ff0 100644 --- a/ARCHITECTURE_PHILOSOPHY.md +++ b/ARCHITECTURE_PHILOSOPHY.md @@ -12,7 +12,7 @@ Introduce no service or infrastructure component unless the feature is impossibl ## 3. Simple deployment -`SPARK_REMOTE` is the only mandatory configuration. All other tunables default in `backend/constants.py`, overridable via environment variable. One Docker image serves both frontend and backend; no second service to provision or wire up. +`SPARK_REMOTE` is the only mandatory configuration. All other tunables are defined in `backend/env.py` with defaults and can be overridden via environment variables. One Docker image serves both frontend and backend; no second service to provision or wire up. ## 4. The process is disposable diff --git a/CLAUDE.md b/CLAUDE.md index 4d188e2..8301659 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - Python files must not exceed 400 lines. - Never run git commands — only the user does. - When `icegraph-client`'s public API or CLI commands change, update the CLI section in `frontend/src/pages/DocsPage.jsx` and the Python Client & CLI bullet in `README.md` to match. -- Whenever creating a constant that is settable via an environment variable, define its default value in `backend/constants.py` and add its description to `README.md`. +- Whenever creating a setting that is configurable via an environment variable, define it in `backend/env.py`. - Frontend work has its own rules, philosophy, and styling conventions in [frontend/CLAUDE.md](frontend/CLAUDE.md) and [frontend/PHILOSOPHY.md](frontend/PHILOSOPHY.md). ## Project Overview @@ -116,7 +116,7 @@ Backend environment variables (set in `backend/.env`): | `MAX_SNAPSHOTS_TO_COMPUTE` | 50 | Max snapshots processed per job | | `MAX_DATA_FILES_TO_COLLECT` | 5000 | Data file iteration limit | | `MAX_NUMBER_OF_GRAPHS_TO_COMPUTE` | 15 | Concurrent job limit | -| `MAX_SNAPSHOTS_TO_SHOW` | 2000 | Snapshot selection UI limit | +| `MAX_SNAPSHOTS_TO_SHOW` | 20 | Snapshot selection UI limit | | `INCLUDE_NONE_ICEBERG_CATALOGS` | `true` | Include non-Iceberg catalogs (e.g. Spark session catalog) in `/api/v1/tables` | | `TABLE_LIST_CACHE_TTL_SECONDS` | 60 | Cache TTL for table list endpoint | diff --git a/README.md b/README.md index 83da667..3de742b 100644 --- a/README.md +++ b/README.md @@ -111,18 +111,7 @@ We will create an `.env` file in the root of the backend directory: SPARK_REMOTE=sc://localhost:15002 # Our local testing spark, If you use docker, change it to your ip. ``` -If you want to change the default values of the application, you can set the following environment variables: - -- `MAX_NUMBER_OF_GRAPHS_TO_COMPUTE`: The maximum number of graphs to compute in parallel. Default is 15. -- `MAX_SNAPSHOTS_TO_SHOW`: The maximum number of snapshots to show in the snapshot selection page. Default is 2000. -- `COMPUTE_CLEANUP_TIME_SECONDS`: The time to wait before cleaning up the computed graphs. Default is 12. -- `MAX_DATA_FILES_TO_COLLECT`: The maximum number of data files to collect. Default is 5000. -- `MAX_SNAPSHOTS_TO_COMPUTE`: The maximum number of snapshots to compute. Default is 50. -- `MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS`: The time to wait before forcing an exit of the application when shutting it down. Default is 10. -- `INCLUDE_NONE_ICEBERG_CATALOGS`: Whether to include catalogs that are not exclusively Iceberg (such as the Spark session catalog) in the table selection autocompletion. Default is true. -- `TABLE_LIST_CACHE_TTL_SECONDS`: The cache time-to-live for table selection autocompletion endpoint before refresh. Default is 60. -- `PRODUCTION_MODE`: Whether to serve the app with the Waitress production WSGI server instead of Flask's development server. Always single-process (multi-threaded) so the in-memory job-polling state stays consistent. Default is false; the Docker image sets this to true. -- `WSGI_THREADS`: The number of threads Waitress uses to handle concurrent requests when `PRODUCTION_MODE` is enabled. Default is 20. +The supported environment variables, their defaults, and their descriptions are defined in the [`Env` class](backend/env.py). ### 3. Run diff --git a/backend/collectors/collect_snapshots.py b/backend/collectors/collect_snapshots.py index a81a11b..21eb402 100644 --- a/backend/collectors/collect_snapshots.py +++ b/backend/collectors/collect_snapshots.py @@ -1,5 +1,3 @@ -from base_classes.utils import column_to_string_utc -import os from dataclasses import dataclass from typing import Dict, List, Optional @@ -8,12 +6,11 @@ from pyspark.sql import functions as F from base_classes.base_file import BaseFile +from base_classes.utils import column_to_string_utc, timed from collectors.collector import Collector, FilesCollection -from constants import FileType, MAX_SNAPSHOTS_TO_COMPUTE +from constants import FileType +from env import Env from icegraph_logger import logger -from base_classes.utils import timed - -max_snapshots_to_compute = int(os.getenv("MAX_SNAPSHOTS_TO_COMPUTE", MAX_SNAPSHOTS_TO_COMPUTE)) @dataclass @@ -63,8 +60,8 @@ def _query_snapshots_df(self) -> pyspark.sql.DataFrame: @staticmethod def _validate_snapshot_count(snapshots_df: pyspark.sql.DataFrame) -> None: - if snapshots_df.count() > max_snapshots_to_compute: - raise ValueError(f"Too many snapshots to compute. Maximum is {max_snapshots_to_compute}.") + if snapshots_df.count() > Env.MAX_SNAPSHOTS_TO_COMPUTE: + raise ValueError(f"Too many snapshots to compute. Maximum is {Env.MAX_SNAPSHOTS_TO_COMPUTE}.") @staticmethod def _format_summary(summary: dict) -> dict: diff --git a/backend/constants.py b/backend/constants.py index f91d898..ddd5bc6 100644 --- a/backend/constants.py +++ b/backend/constants.py @@ -1,28 +1,8 @@ import inspect from enum import Enum -MAX_NUMBER_OF_GRAPHS_TO_COMPUTE = 15 - -MAX_SNAPSHOTS_TO_SHOW = 20 - -MAX_SNAPSHOTS_TO_COMPUTE = 50 - -COMPUTE_CLEANUP_TIME_SECONDS = 12 - -MAX_DATA_FILES_TO_COLLECT = 5_000 - -TABLE_LIST_CACHE_TTL_SECONDS = 60 - -INCLUDE_NONE_ICEBERG_CATALOGS = "true" - -MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS = 10 - APPLICATION_PORT = 5_050 -PRODUCTION_MODE = "false" - -WSGI_THREADS = 20 - MAIN_BRANCH_ICEBERG_TABLE_NAME = "main" JOB_TOKEN_FIELD = "X-IceGraph-Job-Token" diff --git a/backend/env.py b/backend/env.py new file mode 100644 index 0000000..13d6590 --- /dev/null +++ b/backend/env.py @@ -0,0 +1,39 @@ +import os + +from dotenv import load_dotenv + +load_dotenv() + + +class Env: + """Application settings loaded from environment variables.""" + + # Maximum number of graphs to compute in parallel. + MAX_NUMBER_OF_GRAPHS_TO_COMPUTE = int(os.getenv("MAX_NUMBER_OF_GRAPHS_TO_COMPUTE", "15")) + + # Maximum number of snapshots shown on the snapshot selection page. + MAX_SNAPSHOTS_TO_SHOW = int(os.getenv("MAX_SNAPSHOTS_TO_SHOW", "20")) + + # Maximum number of snapshots processed per graph. + MAX_SNAPSHOTS_TO_COMPUTE = int(os.getenv("MAX_SNAPSHOTS_TO_COMPUTE", "50")) + + # Delay before a completed graph is removed from memory. + COMPUTE_CLEANUP_TIME_SECONDS = int(os.getenv("COMPUTE_CLEANUP_TIME_SECONDS", "12")) + + # Maximum number of data files collected per graph. + MAX_DATA_FILES_TO_COLLECT = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", "5000")) + + # Cache lifetime for the table selection endpoint. + TABLE_LIST_CACHE_TTL_SECONDS = int(os.getenv("TABLE_LIST_CACHE_TTL_SECONDS", "60")) + + # Whether non-Iceberg catalogs are included in table selection. + INCLUDE_NONE_ICEBERG_CATALOGS = os.getenv("INCLUDE_NONE_ICEBERG_CATALOGS", "true").lower() == "true" + + # Maximum time allowed for graceful application shutdown. + MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS = int(os.getenv("MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS", "10")) + + # Whether to serve the application with Waitress. + PRODUCTION_MODE = os.getenv("PRODUCTION_MODE", "false").lower() == "true" + + # Number of request threads used by Waitress. + WSGI_THREADS = int(os.getenv("WSGI_THREADS", "20")) diff --git a/backend/extractors/data_files_extractor.py b/backend/extractors/data_files_extractor.py index 1c271a8..398ecb1 100644 --- a/backend/extractors/data_files_extractor.py +++ b/backend/extractors/data_files_extractor.py @@ -1,16 +1,14 @@ -import os from typing import List import pyspark -from pyspark.sql import Window, functions as F +from pyspark.sql import Window +from pyspark.sql import functions as F from pyspark.sql.types import LongType, StringType, StructField, StructType from collectors.collect_manifests import ManifestRecord -from constants import MAX_DATA_FILES_TO_COLLECT +from env import Env from extractors.extractor import Extractor -max_data_files_to_collect = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", MAX_DATA_FILES_TO_COLLECT)) - DATA_FILE_RECORD_SCHEMA = StructType( [ StructField("status", StringType(), False), @@ -112,7 +110,7 @@ def _join_data_file_with_manifest_entries(latest_df, manifest_entries_df): @staticmethod def _limit_and_rank_files_by_snapshot_timestamp(df): - df = df.orderBy(F.desc("latest_snapshot_timestamp")).limit(max_data_files_to_collect + 1) + df = df.orderBy(F.desc("latest_snapshot_timestamp")).limit(Env.MAX_DATA_FILES_TO_COLLECT + 1) row_num_window = Window.orderBy(F.desc("latest_snapshot_timestamp")) df = df.withColumn("row_num", F.row_number().over(row_num_window)) @@ -122,7 +120,7 @@ def _limit_and_rank_files_by_snapshot_timestamp(df): @staticmethod def _find_cutoff_snapshot_timestamp(df): return ( - df.filter(F.col("row_num") == max_data_files_to_collect + 1) + df.filter(F.col("row_num") == Env.MAX_DATA_FILES_TO_COLLECT + 1) .agg(F.coalesce(F.first("latest_snapshot_timestamp"), F.lit(0).cast("timestamp")).alias("snapshot_timestamp_cutoff")) .select("snapshot_timestamp_cutoff") ) diff --git a/backend/main.py b/backend/main.py index b233cbc..de78e1c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,46 +6,27 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from dotenv import load_dotenv from flask import Flask, jsonify, request, send_from_directory from pyspark.errors import AnalysisException -from constants import ( - APPLICATION_PORT, - COMPUTE_CLEANUP_TIME_SECONDS, - JOB_TOKEN_FIELD, - MAX_NUMBER_OF_GRAPHS_TO_COMPUTE, - MAX_SNAPSHOTS_TO_SHOW, - MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS, - INCLUDE_NONE_ICEBERG_CATALOGS, - PRODUCTION_MODE, - WSGI_THREADS, -) -from spark_connect import close_spark_connect_session +from base_classes.utils import verify_iceberg_table +from constants import APPLICATION_PORT, JOB_TOKEN_FIELD +from env import Env from graph_normalizer.graph_normalizer import GraphNormalizer from icegraph_logger import logger from snapshot_analyzer.snapshot_analyzer import SnapshotAnalyzer from snapshot_map.snapshot_mapping import collect_snapshot_map -from table_list_catalog.table_list_catalog import TableListCatalog +from spark_connect import close_spark_connect_session from table_inventory.table_inventory import TableInventory -from base_classes.utils import verify_iceberg_table +from table_list_catalog.table_list_catalog import TableListCatalog -load_dotenv() app = Flask(__name__, static_url_path="/static") app.json.sort_keys = False job_lock = threading.Lock() jobs: dict[str, dict] = {} -max_number_of_graphs_to_compute = int(os.getenv("MAX_NUMBER_OF_GRAPHS_TO_COMPUTE", MAX_NUMBER_OF_GRAPHS_TO_COMPUTE)) -compute_cleanup_time_seconds = int(os.getenv("COMPUTE_CLEANUP_TIME_SECONDS", COMPUTE_CLEANUP_TIME_SECONDS)) -max_snapshots_to_show = int(os.getenv("MAX_SNAPSHOTS_TO_SHOW", MAX_SNAPSHOTS_TO_SHOW)) -max_graceful_shutdown_time_seconds = int(os.getenv("MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS", MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS)) -include_none_iceberg_catalogs = str(os.getenv("INCLUDE_NONE_ICEBERG_CATALOGS", INCLUDE_NONE_ICEBERG_CATALOGS)).lower() == "true" -production_mode = str(os.getenv("PRODUCTION_MODE", PRODUCTION_MODE)).lower() == "true" -wsgi_threads = int(os.getenv("WSGI_THREADS", WSGI_THREADS)) - -executor_pool = ThreadPoolExecutor(max_workers=max_number_of_graphs_to_compute) +executor_pool = ThreadPoolExecutor(max_workers=Env.MAX_NUMBER_OF_GRAPHS_TO_COMPUTE) def _safe_update_job(job_id, **fields): @@ -62,7 +43,7 @@ def _cleanup_job(job_id): def _schedule_cleanup(job_id, is_in_lock_block=False): timer = threading.Timer( - compute_cleanup_time_seconds, + Env.COMPUTE_CLEANUP_TIME_SECONDS, lambda job_id=job_id: _cleanup_job(job_id), ) timer.daemon = True @@ -118,7 +99,7 @@ def list_tables(): return jsonify( { "tables": tables, - "include_none_iceberg_catalogs": include_none_iceberg_catalogs, + "include_none_iceberg_catalogs": Env.INCLUDE_NONE_ICEBERG_CATALOGS, } ) @@ -136,7 +117,7 @@ def snapshot_map(table_name): try: verify_iceberg_table(table_name) - result = collect_snapshot_map(table_name, max_snapshots_to_show) + result = collect_snapshot_map(table_name, Env.MAX_SNAPSHOTS_TO_SHOW) return jsonify(result) @@ -228,21 +209,21 @@ def get_job_status(job_id): def _force_exit(): - logger.error(f"Graceful shutdown timed out after {max_graceful_shutdown_time_seconds} seconds - forcing exit.") + logger.error(f"Graceful shutdown timed out after {Env.MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS} seconds - forcing exit.") os._exit(1) if __name__ == "__main__": try: - if production_mode: + if Env.PRODUCTION_MODE: from waitress import serve - serve(app, host="0.0.0.0", port=APPLICATION_PORT, threads=wsgi_threads) + serve(app, host="0.0.0.0", port=APPLICATION_PORT, threads=Env.WSGI_THREADS) else: app.run(host="0.0.0.0", port=APPLICATION_PORT, debug=True) finally: - watchdog = threading.Timer(max_graceful_shutdown_time_seconds, _force_exit) + watchdog = threading.Timer(Env.MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS, _force_exit) watchdog.daemon = True watchdog.start() diff --git a/backend/table_inventory/table_inventory.py b/backend/table_inventory/table_inventory.py index 4806421..8a27621 100644 --- a/backend/table_inventory/table_inventory.py +++ b/backend/table_inventory/table_inventory.py @@ -1,4 +1,3 @@ -import os from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Any, Dict, List, Optional @@ -9,18 +8,12 @@ from collectors.collect_manifests import CollectManifests, ManifestRecord from collectors.collect_metadata import CollectMetadata, MetadataFileRecord from collectors.collect_snapshots import CollectSnapshots, SnapshotRecord -from constants import ( - DATA_FILES_CUTOFF_MANIFEST_WARNING, - DATA_FILES_CUTOFF_WARNING, - FileType, - MAX_DATA_FILES_TO_COLLECT, -) +from constants import DATA_FILES_CUTOFF_MANIFEST_WARNING, DATA_FILES_CUTOFF_WARNING, FileType +from env import Env from icegraph_logger import logger from search_cutoff.find_search_cutoff import SearchCutoff, find_search_cutoff from table_inventory.utils import format_schemas_to_full_dict, get_json_metadata_from_path, parse_json_string_fields -max_data_files_to_collect = int(os.getenv("MAX_DATA_FILES_TO_COLLECT", MAX_DATA_FILES_TO_COLLECT)) - @dataclass class TableInventoryResult: @@ -208,7 +201,7 @@ def _warn_if_data_cutoff_happened(self): if manifest.child_files or manifest.error or manifest.added_snapshot_timestamp is None: continue - manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=max_data_files_to_collect) + manifest.warning = DATA_FILES_CUTOFF_MANIFEST_WARNING.format(max_data_files_to_collect=Env.MAX_DATA_FILES_TO_COLLECT) if max_manifest_added_snapshot_timestamp is None or max_manifest_added_snapshot_timestamp < manifest.added_snapshot_timestamp: max_manifest_added_snapshot_timestamp = manifest.added_snapshot_timestamp @@ -216,7 +209,7 @@ def _warn_if_data_cutoff_happened(self): if max_manifest_added_snapshot_timestamp is not None: self._warnings["data_files_cutoff"] = DATA_FILES_CUTOFF_WARNING.format( - max_data_files_to_collect=max_data_files_to_collect, + max_data_files_to_collect=Env.MAX_DATA_FILES_TO_COLLECT, added_snapshot_id=max_manifest_added_snapshot_id, added_snapshot_timestamp=max_manifest_added_snapshot_timestamp, ) diff --git a/backend/table_list_catalog/table_list_catalog.py b/backend/table_list_catalog/table_list_catalog.py index ab96289..57ad4b5 100644 --- a/backend/table_list_catalog/table_list_catalog.py +++ b/backend/table_list_catalog/table_list_catalog.py @@ -1,22 +1,20 @@ -from table_list_catalog.utils import get_spark_default_catalog -import os import threading -import arrow from dataclasses import dataclass from typing import Optional +import arrow + from base_classes.utils import timed -from constants import TABLE_LIST_CACHE_TTL_SECONDS +from env import Env from spark_connect import open_spark_connect_session from table_list_catalog.utils import ( collect_catalogs_tables_names, collect_databases_in_catalogs, - list_catalog_names, filter_catalogs_to_include, + get_spark_default_catalog, + list_catalog_names, ) -table_list_cache_ttl_seconds = int(os.getenv("TABLE_LIST_CACHE_TTL_SECONDS", TABLE_LIST_CACHE_TTL_SECONDS)) - @dataclass class CacheEntry: @@ -51,7 +49,7 @@ def collect(self) -> list[str]: @staticmethod def _fresh_cached_tables() -> Optional[list[str]]: cache = TableListCatalog._cache - if cache and (arrow.utcnow() - cache.timestamp).total_seconds() < table_list_cache_ttl_seconds: + if cache and (arrow.utcnow() - cache.timestamp).total_seconds() < Env.TABLE_LIST_CACHE_TTL_SECONDS: return cache.tables return None diff --git a/backend/table_list_catalog/utils.py b/backend/table_list_catalog/utils.py index 7828009..801b6e1 100644 --- a/backend/table_list_catalog/utils.py +++ b/backend/table_list_catalog/utils.py @@ -1,13 +1,10 @@ -from typing import List -from pyspark.sql import functions as F -import os from contextlib import suppress -from typing import Optional +from typing import List, Optional from pyspark.sql import SparkSession -from constants import INCLUDE_NONE_ICEBERG_CATALOGS +from pyspark.sql import functions as F -include_none_iceberg_catalogs = str(os.getenv("INCLUDE_NONE_ICEBERG_CATALOGS", INCLUDE_NONE_ICEBERG_CATALOGS)).lower() == "true" +from env import Env def get_spark_default_catalog(spark: SparkSession) -> str: @@ -32,7 +29,7 @@ def filter_catalogs_to_include(spark: SparkSession, catalogs: list[str]) -> list for catalog in catalogs: catalog_config_value = get_spark_catalog_config_value(spark, catalog) - if include_none_iceberg_catalogs or is_iceberg_spark_catalog(catalog_config_value): + if Env.INCLUDE_NONE_ICEBERG_CATALOGS or is_iceberg_spark_catalog(catalog_config_value): catalogs_to_include.append(catalog) return catalogs_to_include From 6dd6e39ba79cfc320748f8665b3cd18654316f52 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 20:27:18 +0300 Subject: [PATCH 22/29] Better location for sorting --- backend/collectors/collect_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 5c8d41d..5c349e8 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -61,7 +61,7 @@ def collect(self) -> FilesCollection: if metadata_files_df is not None: snap_id_to_path = self._get_snap_id_to_path() - rows = metadata_files_df.orderBy(F.desc("metadata_timestamp")).collect() + rows = metadata_files_df.collect() for index, row in enumerate(rows): row_dict = row.asDict(recursive=True) metadata_file_type = FileType.MAIN_METADATA if index == 0 else FileType.METADATA @@ -87,6 +87,7 @@ def _query_metadata_files(self) -> dict: .select("file", "metadata_timestamp") .filter(F.col("metadata_timestamp") >= F.lit(str(self._start_metadata_cutoff))) .filter(F.col("metadata_timestamp") <= F.lit(str(self._end_metadata_cutoff))) + .orderBy(F.desc("metadata_timestamp")) .withColumn("metadata_timestamp", column_to_string_utc("metadata_timestamp")) ) return {row.file: row.metadata_timestamp for row in metadata_df.collect()} From 70ff5f36e3b76a1902ebadae6157ace90cf5014c Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Fri, 14 Aug 2026 20:37:30 +0300 Subject: [PATCH 23/29] Order location --- backend/collectors/collect_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 5c349e8..2e2c68c 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -141,9 +141,9 @@ def _get_snap_id_to_path(self) -> Dict[int, str]: return {s.snapshot_id: s.file_path for s in (self._snapshots or [])} def _get_previous_metadata_file(self, file_path: str) -> Optional[str]: - file_index = self._ordered_metadata_paths.index(file_path) + older_index = self._ordered_metadata_paths.index(file_path) + 1 - return self._ordered_metadata_paths[file_index - 1] if file_index - 1 >= 0 else None + return self._ordered_metadata_paths[older_index] if older_index < len(self._ordered_metadata_paths) else None @staticmethod def _parse_refs(row: dict) -> dict: From 89d7bd74213d37af46b46199934bdd39135f7206 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 00:14:43 +0300 Subject: [PATCH 24/29] Simpler metadata files order --- backend/collectors/collect_metadata.py | 30 ++++++++++---------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 2e2c68c..4f5bbfe 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -61,14 +61,11 @@ def collect(self) -> FilesCollection: if metadata_files_df is not None: snap_id_to_path = self._get_snap_id_to_path() - rows = metadata_files_df.collect() - for index, row in enumerate(rows): - row_dict = row.asDict(recursive=True) - metadata_file_type = FileType.MAIN_METADATA if index == 0 else FileType.METADATA + for row in metadata_files_df.collect(): + self._metadata_files.append(self._parse_metadata_row(row.asDict(recursive=True), snap_id_to_path)) - self._metadata_files.append(self._parse_metadata_row(metadata_file_type, row_dict, snap_id_to_path)) - - self._add_bad_metadata_files() + self._metadata_files.extend(self._bad_metadata_files) + self._apply_metadata_order() except Exception as e: logger.error(f"[{self._table_name}] metadata collection failed", exc_info=True) @@ -112,7 +109,7 @@ def _build_metadata_files_df(self) -> Optional[pyspark.sql.DataFrame]: error=str(e), timestamp=timestamp, snapshot_id=None, - previous_file=None, + previous_file=self._get_previous_metadata_file(file), last_sequence_number=None, partition_spec_id=None, current_schema_id=None, @@ -126,16 +123,11 @@ def _build_metadata_files_df(self) -> Optional[pyspark.sql.DataFrame]: return metadata_files_df - def _add_bad_metadata_files(self) -> None: - for bad_file in self._bad_metadata_files: - index_to_insert = len(self._metadata_files) - - for index, metadata_file in enumerate(self._metadata_files): - if metadata_file.previous_file == bad_file.file_path: - index_to_insert = index + 1 - break + def _apply_metadata_order(self) -> None: + metadata_file_by_path = {metadata_file.file_path: metadata_file for metadata_file in self._metadata_files} - self._metadata_files.insert(index_to_insert, bad_file) + self._metadata_files = [metadata_file_by_path[file_path] for file_path in self._ordered_metadata_paths] + self._metadata_files[0].type = FileType.MAIN_METADATA def _get_snap_id_to_path(self) -> Dict[int, str]: return {s.snapshot_id: s.file_path for s in (self._snapshots or [])} @@ -162,7 +154,7 @@ def _build_branches_child_files(refs: dict, snap_id_to_path: dict) -> List[str]: return branches_child_files - def _parse_metadata_row(self, file_type: FileType, row: dict, snap_id_to_path: dict) -> MetadataFileRecord: + def _parse_metadata_row(self, row: dict, snap_id_to_path: dict) -> MetadataFileRecord: refs = self._parse_refs(row) branches_child_files = self._build_branches_child_files(refs, snap_id_to_path) @@ -170,7 +162,7 @@ def _parse_metadata_row(self, file_type: FileType, row: dict, snap_id_to_path: d child_files = ([current_snap_path] if current_snap_path else []) + branches_child_files return MetadataFileRecord( - type=file_type, + type=FileType.METADATA, file_path=row["file"], timestamp=str(row["metadata_timestamp"]), snapshot_id=row["current-snapshot-id"], From 8168c72992729dd6bb7ad0b1d64ef86aeea5a5c5 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 00:30:36 +0300 Subject: [PATCH 25/29] Case of all the files are errors --- backend/collectors/collect_metadata.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/collectors/collect_metadata.py b/backend/collectors/collect_metadata.py index 4f5bbfe..5b397d1 100644 --- a/backend/collectors/collect_metadata.py +++ b/backend/collectors/collect_metadata.py @@ -64,8 +64,8 @@ def collect(self) -> FilesCollection: for row in metadata_files_df.collect(): self._metadata_files.append(self._parse_metadata_row(row.asDict(recursive=True), snap_id_to_path)) - self._metadata_files.extend(self._bad_metadata_files) - self._apply_metadata_order() + self._metadata_files.extend(self._bad_metadata_files) + self._apply_metadata_order() except Exception as e: logger.error(f"[{self._table_name}] metadata collection failed", exc_info=True) @@ -124,6 +124,9 @@ def _build_metadata_files_df(self) -> Optional[pyspark.sql.DataFrame]: return metadata_files_df def _apply_metadata_order(self) -> None: + if not self._metadata_files: + return + metadata_file_by_path = {metadata_file.file_path: metadata_file for metadata_file in self._metadata_files} self._metadata_files = [metadata_file_by_path[file_path] for file_path in self._ordered_metadata_paths] From ca5daa8ac7e542d97ccf531c1cc7be6c7f1bcce0 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 01:28:19 +0300 Subject: [PATCH 26/29] Fix ui error timeline false positive --- frontend/src/pages/DocsPage.jsx | 13 +++++++------ frontend/src/pages/TimelinePage.jsx | 6 +----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/frontend/src/pages/DocsPage.jsx b/frontend/src/pages/DocsPage.jsx index f366fa7..f4924d4 100644 --- a/frontend/src/pages/DocsPage.jsx +++ b/frontend/src/pages/DocsPage.jsx @@ -327,12 +327,13 @@ const SECTIONS = [

      A red Unknown Events marker - appears when metadata or snapshot data is missing. It indicates that - one or more events occurred in that part of the timeline, even when - the exact events cannot be determined. The next readable event is - compared with the previous readable metadata. Its details show the - metadata changes, and when the snapshot changed they also show that - snapshot's operation. + appears when metadata or snapshot data could not be read. Snapshots + that simply fall outside the selected range are not flagged. It + indicates that one or more events occurred in that part of the + timeline, even when the exact events cannot be determined. The next + readable event is compared with the previous readable metadata. Its + details show the metadata changes, and when the snapshot changed they + also show that snapshot's operation.

      Zoom & pan

      diff --git a/frontend/src/pages/TimelinePage.jsx b/frontend/src/pages/TimelinePage.jsx index cfd1e3c..13bd660 100644 --- a/frontend/src/pages/TimelinePage.jsx +++ b/frontend/src/pages/TimelinePage.jsx @@ -600,12 +600,8 @@ export default function TimelinePage() { const snapshotId = type === "C" ? branchSnapId : details.snapshot_id; const referencedSnapshot = snapMap[snapshotId]; - const isSnapshotMissing = - snapshotId != null && - String(snapshotId) !== "-1" && - (!referencedSnapshot || referencedSnapshot.error); - if (details.error || isSnapshotMissing) { + if (details.error || referencedSnapshot?.error) { type = "error"; } From 12cbe20ae4f8b228009c2300268f10692169bdcb Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 13:18:20 +0300 Subject: [PATCH 27/29] Better relese notes --- .github/workflows/deploy.yml | 4 +- .github/workflows/docker-publish.yml | 4 +- .github/workflows/publish-icegraph-client.yml | 4 +- .github/workflows/release.yml | 70 +++++++++++++++++++ 4 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 63be792..1a4c31e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,9 +1,7 @@ name: Deploy to GitHub Pages on: - push: - tags: - - "v*" + workflow_call: permissions: contents: write diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ccac01e..89e5d38 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,9 +1,7 @@ name: Publish Docker image on: - push: - tags: - - 'v*' + workflow_call: jobs: push_to_registry: diff --git a/.github/workflows/publish-icegraph-client.yml b/.github/workflows/publish-icegraph-client.yml index 7e09f43..06a7387 100644 --- a/.github/workflows/publish-icegraph-client.yml +++ b/.github/workflows/publish-icegraph-client.yml @@ -1,9 +1,7 @@ name: Publish icegraph-client to PyPI on: - push: - tags: - - "v*" + workflow_call: jobs: publish: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b78dad2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + docker: + name: Docker image + uses: ./.github/workflows/docker-publish.yml + secrets: inherit + + pypi: + name: Python client + uses: ./.github/workflows/publish-icegraph-client.yml + secrets: inherit + + pages: + name: Frontend demo + uses: ./.github/workflows/deploy.yml + secrets: inherit + + release-notes: + name: Write release notes + runs-on: ubuntu-latest + needs: [docker, pypi, pages] + steps: + - name: Build notes and publish release + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + DOCKER_IMAGE: yanivzalach/icegraph + DEMO_URL: https://yanivzalach.github.io/IceGraph/ + run: | + set -euo pipefail + + VERSION="${TAG#v}" + NOTES_FILE="$RUNNER_TEMP/release-notes.md" + + GENERATED="$(gh api "repos/$REPO/releases/generate-notes" -f tag_name="$TAG" --jq .body)" + CONTRIBUTORS="$(printf '%s\n' "$GENERATED" | grep -oE '@[A-Za-z0-9-]+' | sort -u | paste -sd , - | sed 's/,/, /g' || true)" + + cat > "$NOTES_FILE" <[Docker Hub](https://hub.docker.com/r/$DOCKER_IMAGE/tags?name=$TAG) | + | 🐍 Python client | \`pip install icegraph-client==$VERSION\`
      [PyPI](https://pypi.org/project/icegraph-client/$VERSION/) | + | 🌐 Live demo | [$DEMO_URL]($DEMO_URL) | + | 📖 Docs | [${DEMO_URL}docs](${DEMO_URL}docs) | + + EOF + + printf '%s\n' "$GENERATED" >> "$NOTES_FILE" + + if [ -n "$CONTRIBUTORS" ]; then + printf '\n## Contributors\n\nThanks to %s 🧊\n' "$CONTRIBUTORS" >> "$NOTES_FILE" + fi + + if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then + gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE" --latest + else + gh release create "$TAG" --repo "$REPO" --title "IceGraph $TAG" --notes-file "$NOTES_FILE" --latest + fi From 52b448626346a6caa6700a2a8120b57a65feb4a5 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 13:24:33 +0300 Subject: [PATCH 28/29] Better sec --- .github/dependabot.yml | 11 +++++++++++ .github/workflows/docker-publish.yml | 5 +++++ .github/workflows/publish-icegraph-client.yml | 5 +++++ .github/workflows/release.yml | 19 ++++++++++++++----- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..17b65ec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 + +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 89e5d38..98c4e07 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,6 +2,11 @@ name: Publish Docker image on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true jobs: push_to_registry: diff --git a/.github/workflows/publish-icegraph-client.yml b/.github/workflows/publish-icegraph-client.yml index 06a7387..03ffc37 100644 --- a/.github/workflows/publish-icegraph-client.yml +++ b/.github/workflows/publish-icegraph-client.yml @@ -2,11 +2,16 @@ name: Publish icegraph-client to PyPI on: workflow_call: + secrets: + PYPI_API_TOKEN: + required: true jobs: publish: name: Build and publish to PyPI runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Check out the repo uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b78dad2..4145cc5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,29 +5,38 @@ on: tags: - "v*" -permissions: - contents: write +permissions: {} jobs: docker: name: Docker image uses: ./.github/workflows/docker-publish.yml - secrets: inherit + permissions: + contents: read + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} pypi: name: Python client uses: ./.github/workflows/publish-icegraph-client.yml - secrets: inherit + permissions: + contents: read + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} pages: name: Frontend demo uses: ./.github/workflows/deploy.yml - secrets: inherit + permissions: + contents: write release-notes: name: Write release notes runs-on: ubuntu-latest needs: [docker, pypi, pages] + permissions: + contents: write steps: - name: Build notes and publish release env: From 4482d7220e6e81c30101120c1df5c1822d702393 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Aug 2026 10:31:19 +0000 Subject: [PATCH 29/29] Bump the actions group with 7 updates Bumps the actions group with 7 updates: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `4` | `7` | | [actions/setup-node](https://github.com/actions/setup-node) | `4` | `7` | | [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) | `3` | `4` | | [docker/login-action](https://github.com/docker/login-action) | `3` | `4` | | [docker/metadata-action](https://github.com/docker/metadata-action) | `5` | `6` | | [docker/build-push-action](https://github.com/docker/build-push-action) | `6` | `7` | | [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) | `3` | `7` | Updates `actions/checkout` from 4 to 7 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v7) Updates `actions/setup-node` from 4 to 7 - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v7) Updates `docker/setup-buildx-action` from 3 to 4 - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3...v4) Updates `docker/login-action` from 3 to 4 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3...v4) Updates `docker/metadata-action` from 5 to 6 - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v5...v6) Updates `docker/build-push-action` from 6 to 7 - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6...v7) Updates `astral-sh/setup-uv` from 3 to 7 - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v3...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/setup-node dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: docker/setup-buildx-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: docker/login-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: docker/metadata-action dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: docker/build-push-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: astral-sh/setup-uv dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- .github/workflows/deploy.yml | 4 ++-- .github/workflows/docker-publish.yml | 10 +++++----- .github/workflows/publish-icegraph-client.yml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e245f3..2a1ba5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: python-format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Check backend formatting uses: astral-sh/ruff-action@v4.1.0 with: @@ -30,8 +30,8 @@ jobs: run: working-directory: frontend steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 with: node-version: 22 cache: npm diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1a4c31e..aa43c97 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "24" cache: "npm" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 98c4e07..29b8ae3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -16,20 +16,20 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ secrets.DOCKERHUB_USERNAME }}/icegraph tags: | @@ -37,7 +37,7 @@ jobs: type=raw,value=latest - name: Build and push Docker image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . push: true diff --git a/.github/workflows/publish-icegraph-client.yml b/.github/workflows/publish-icegraph-client.yml index 03ffc37..1e4da82 100644 --- a/.github/workflows/publish-icegraph-client.yml +++ b/.github/workflows/publish-icegraph-client.yml @@ -14,12 +14,12 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v7 - name: Build working-directory: icegraph-client