From 57e7f00fd932f3f4f36039bded1edb3a200bf84a Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 17:51:14 +0300 Subject: [PATCH 1/4] Exclude database views from table catalog Refactor table filtering to exclude views alongside temporary tables --- backend/table_list_catalog/utils.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/backend/table_list_catalog/utils.py b/backend/table_list_catalog/utils.py index 801b6e1..4f7d8ba 100644 --- a/backend/table_list_catalog/utils.py +++ b/backend/table_list_catalog/utils.py @@ -1,10 +1,11 @@ from contextlib import suppress from typing import List, Optional -from pyspark.sql import SparkSession +from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as F from env import Env +from icegraph_logger import logger def get_spark_default_catalog(spark: SparkSession) -> str: @@ -57,14 +58,28 @@ def collect_databases_in_catalogs(spark: SparkSession, catalogs: List[str]) -> L return [row.database for row in databases_df.collect()] +def get_database_views_df(spark: SparkSession, database: str) -> Optional[DataFrame]: + with suppress(Exception): + return spark.sql(f"show views in {database}").withColumn("table", F.concat(F.lit(f"{database}."), F.col("viewName"))).select("table") + + logger.warning(f"Could not list views in {database}") + return None + + def collect_catalogs_tables_names(spark: SparkSession, databases: List[str]) -> List[str]: tables_df = None for database in databases: df = ( spark.sql(f"show tables in {database}") + .filter(~F.col("isTemporary")) .withColumn("table", F.concat(F.lit(f"{database}."), F.col("tableName"))) - .select("table", "isTemporary") + .select("table") ) + + views_df = get_database_views_df(spark, database) + if views_df is not None: + df = df.subtract(views_df) + if tables_df is None: tables_df = df else: @@ -73,4 +88,4 @@ def collect_catalogs_tables_names(spark: SparkSession, databases: List[str]) -> if tables_df is None: return [] - return [row.table for row in tables_df.filter(F.col("isTemporary") == False).select("table").collect()] + return [row.table for row in tables_df.collect()] From dd4adee9a5d3437b4f877f39a41742dc760e67ee Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 17:57:40 +0300 Subject: [PATCH 2/4] Narrow views listing failure handling Catch PySparkException instead of every exception and log the error reason. Co-Authored-By: Claude Opus 5 (1M context) --- backend/table_list_catalog/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/table_list_catalog/utils.py b/backend/table_list_catalog/utils.py index 4f7d8ba..f2e70f2 100644 --- a/backend/table_list_catalog/utils.py +++ b/backend/table_list_catalog/utils.py @@ -1,6 +1,7 @@ from contextlib import suppress from typing import List, Optional +from pyspark.errors import PySparkException from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as F @@ -59,11 +60,11 @@ def collect_databases_in_catalogs(spark: SparkSession, catalogs: List[str]) -> L def get_database_views_df(spark: SparkSession, database: str) -> Optional[DataFrame]: - with suppress(Exception): + try: return spark.sql(f"show views in {database}").withColumn("table", F.concat(F.lit(f"{database}."), F.col("viewName"))).select("table") - - logger.warning(f"Could not list views in {database}") - return None + except PySparkException as error: + logger.warning(f"Could not list views in {database}: {error}") + return None def collect_catalogs_tables_names(spark: SparkSession, databases: List[str]) -> List[str]: From 8fa5e4c2ba6699dfd3ee1a0c5f9de5f1d6311bfd Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 18:02:00 +0300 Subject: [PATCH 3/4] Add blank line before views listing except clause Co-Authored-By: Claude Opus 5 (1M context) --- backend/table_list_catalog/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/table_list_catalog/utils.py b/backend/table_list_catalog/utils.py index f2e70f2..12bd237 100644 --- a/backend/table_list_catalog/utils.py +++ b/backend/table_list_catalog/utils.py @@ -62,6 +62,7 @@ def collect_databases_in_catalogs(spark: SparkSession, catalogs: List[str]) -> L def get_database_views_df(spark: SparkSession, database: str) -> Optional[DataFrame]: try: return spark.sql(f"show views in {database}").withColumn("table", F.concat(F.lit(f"{database}."), F.col("viewName"))).select("table") + except PySparkException as error: logger.warning(f"Could not list views in {database}: {error}") return None From 16b7321d9cf9ba1df6ec2f3d10efa0e235e3d851 Mon Sep 17 00:00:00 2001 From: Yaniv Zalach Date: Sat, 15 Aug 2026 18:06:02 +0300 Subject: [PATCH 4/4] Subtract only persistent views from tables list Temporary views share the tables namespace, so a temp view named after a permanent table would drop that table from the catalog listing. Co-Authored-By: Claude Opus 5 (1M context) --- backend/table_list_catalog/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/table_list_catalog/utils.py b/backend/table_list_catalog/utils.py index 12bd237..d863167 100644 --- a/backend/table_list_catalog/utils.py +++ b/backend/table_list_catalog/utils.py @@ -61,7 +61,12 @@ def collect_databases_in_catalogs(spark: SparkSession, catalogs: List[str]) -> L def get_database_views_df(spark: SparkSession, database: str) -> Optional[DataFrame]: try: - return spark.sql(f"show views in {database}").withColumn("table", F.concat(F.lit(f"{database}."), F.col("viewName"))).select("table") + return ( + spark.sql(f"show views in {database}") + .filter(~F.col("isTemporary")) + .withColumn("table", F.concat(F.lit(f"{database}."), F.col("viewName"))) + .select("table") + ) except PySparkException as error: logger.warning(f"Could not list views in {database}: {error}")