[pull] main from ERPGulf:main

changai/changai/api/v2/auto_gen_api.py

@@ -201,11 +201,11 @@ def _extract_existing_keys(data: List[Any]) -> Set[tuple]:
     return keys
 
 
-def _build_master_data_row(entity_type: str, entity_id:str,title_field:str) -> Dict[str, Any]:
+def _build_master_data_row(entity_type: str, entity_id:str,title_field:str,filter: Optional[Dict[str, Any]]) -> Dict[str, Any]:
     return {
         "entity_type": entity_type,
-        "entity_id": title_field,
-        "filters": {"field": title_field if title_field else "name", "value": entity_id},
+        "entity_id": entity_id,
+        "filters": filter or {"field": title_field if title_field else "name", "value": entity_id},
     }
 
 
@@ -271,25 +271,17 @@ def sync_master_data_smart() -> Dict[str, Any]:
         for rec in live_records:
             if mod == "Item":
                 item_code = rec.get("name")
-                if item_code:
-                    rebuilt_rows.append(
-                _build_master_data_row(
-                    entity_type,
-                    item_code,
-                    "name"
-                ))
                 item_name = rec.get(title_field)
-                if item_name:
+                if item_code:
+                    filters = [{"field": "item_code", "value": item_code}]
+                    if item_name and item_name != item_code:
+                        filters.append({"field": "item_name", "value": item_name})
                     rebuilt_rows.append(
-                _build_master_data_row(
-                    entity_type,
-                    item_name,
-                    title_field
-                )
-            )
-            else:                    
+                        _build_master_data_row(entity_type, item_code, title_field, filters)
+                    )
+            else:
                 entity_id = rec.get(title_field) if title_field in rec else rec.get("name")
-                rebuilt_rows.append(_build_master_data_row(entity_type, entity_id,title_field))
+                rebuilt_rows.append(_build_master_data_row(entity_type, entity_id, title_field, None))   
     final_data = rebuilt_rows
     meta["last_sync"] = str(now_datetime())
     settings = frappe.get_single("ChangAI Settings")
@@ -463,6 +455,7 @@ def _update_or_create_table_block(
         "table": table,
         "description": "",
         "fields": fields,
+        "grain":"",
         "desc_done": False,
     }
 def _build_field_entry(
@@ -559,6 +552,12 @@ def _has_pending_descriptions(fields: List[Dict[str, Any]]) -> bool:
         if isinstance(field, dict) and field.get("name")
     )
 
+def _infer_grain_label(meta_dt: Any, table: str) -> str:
+
+    if meta_dt.istable:
+        return f"GRAIN: 1 row per parent document + idx (child table of {table}). Comparison-ready across parent records."
+    return f"GRAIN: 1 row per {_strip_tab(table)} document (master/transaction). Use only for single-entity lookups, not cross-record comparison unless fields are per-relationship (e.g. supplier, price_list)."
+
 
 def _process_schema_table(table: str, by_table: Dict[str, Dict[str, Any]]) -> bool:
     dt = _strip_tab(table)
@@ -569,6 +568,7 @@ def _process_schema_table(table: str, by_table: Dict[str, Dict[str, Any]]) -> bo
     meta_dt = frappe.get_meta(dt)
     block = by_table.setdefault(table, {})
     block["is_table"] = bool(meta_dt.istable)
+    block["grain"] = _infer_grain_label(meta_dt, table)
     existing_fields = _get_existing_fields_for_table(by_table, table)
     fields = _build_fields_from_meta(meta_dt, existing_fields)
     _update_or_create_table_block(by_table, table, fields)
@@ -655,7 +655,6 @@ def fill_missing_field_descriptions(
 def sync_tables_and_schema_smart() -> Dict[str, Any]:
     payload = _read_filedoctype(SCHEMA_YAML, RAG_FOLDER)
     meta, tables_blocks = _normalize_schema_payload(payload)
-
     by_table = _build_table_map(tables_blocks)
     last_sync_raw = meta.get("last_sync")
     changed_doctypes = _get_changed_doctypes(last_sync_raw)

changai/changai/api/v2/build_cards_faiss_index_v2.py

@@ -162,29 +162,12 @@ def _is_valid_schema_table(table_block: Any) -> bool:
     return isinstance(table_block, dict) and "table" in table_block
 
 
-def _build_field_metadata(table_name: str, field_name: str, module: str, join_hint: str, options: Any) -> Dict[str, Any]:
-    metadata = {
-        "type": "field",
-        "table": table_name,
-        "field": field_name,
-        "module": module,
-    }
-    if options:
-        metadata["options"] = options
-    if join_hint:
-        metadata["join_hint"] = join_hint
-    return metadata
-
-
-def _build_field_page_content(table_name: str, field_name: str, field_desc: str, join_hint: str, options: Any) -> str:
-    page_content = f"[FIELD] {field_name} | [TABLE] {table_name}\n{field_desc}"
-    if join_hint:
-        page_content += f"\n{join_hint}"
-    if options:
-        page_content += f"\n{options}"
-    return page_content
-
-def _build_field_document(table_name: str, module: str, field_row: Dict[str, Any]) -> Optional[Document]:
+def _build_field_document(
+    table_name: str,
+    module: str,
+    field_row: Dict[str, Any],
+    grain: str = ""                                     # ✅ Add this param
+) -> Optional[Document]:
     if not isinstance(field_row, dict):
         return None
 
@@ -194,37 +177,41 @@ def _build_field_document(table_name: str, module: str, field_row: Dict[str, Any
 
     field_desc = field_row.get("description", "") or ""
     join_hint = field_row.get("join_hint") or ""
-    child_hint = field_row.get("child_hint") or ""      # ✅ Add this
+    child_hint = field_row.get("child_hint") or ""
     options = field_row.get("options") or ""
-    fieldtype = field_row.get("fieldtype") or ""        # ✅ Add this
-    is_table_field = fieldtype in ("Table", "Table MultiSelect")  # ✅ Add this
+    fieldtype = field_row.get("fieldtype") or ""
+    is_table_field = fieldtype in ("Table", "Table MultiSelect")
 
     # Build page content
     page_content = f"[FIELD] {field_name} | [TABLE] {table_name}\n{field_desc}"
     if join_hint:
         page_content += f"\n{join_hint}"
-    if child_hint and is_table_field:                   # ✅ Add child hint to content
+    if child_hint and is_table_field:
         if isinstance(child_hint, dict):
             page_content += f"\nChild Table: {child_hint.get('child_table', '')}"
         else:
             page_content += f"\n{child_hint}"
     if options:
         page_content += f"\n{options}"
+    if grain:                                            # ✅ Add this
+        page_content += f"\nGRAIN: {grain}"               # ✅ Add this
 
     # Build metadata
     metadata = {
         "type": "field",
         "table": table_name,
         "field": field_name,
-        "fieldtype": fieldtype,                         # ✅ Store fieldtype
+        "fieldtype": fieldtype,
         "module": module,
     }
     if options:
         metadata["options"] = options
     if join_hint:
         metadata["join_hint"] = join_hint
-    if child_hint and is_table_field:                   # ✅ Store child_hint
+    if child_hint and is_table_field:
         metadata["child_hint"] = child_hint
+    if grain:                                            # ✅ Add this
+        metadata["grain"] = grain                         # ✅ Add this — this is what retrieve.py reads
 
     return Document(
         page_content=page_content,
@@ -267,6 +254,7 @@ def build_schema_docs(schema: Dict[str, Any]) -> List[Document]:
 
         table_name = table_block.get("table")
         module = table_block.get("module", "")
+        grain = table_block.get("grain", "")  
         fields = table_block.get("fields") or []
 
         if not isinstance(fields, list):
@@ -277,7 +265,7 @@ def build_schema_docs(schema: Dict[str, Any]) -> List[Document]:
             if field_name in GENERIC_FIELDS:
                 continue
 
-            doc = _build_field_document(table_name, module, field_row)
+            doc = _build_field_document(table_name, module, field_row,grain)
             if doc:
                 docs.append(doc)
 
@@ -307,13 +295,24 @@ def _build_entity_text(md: Dict[str, Any]) -> str:
 
 
 def _build_entity_metadata(md: Dict[str, Any]) -> Dict[str, Any]:
+    filters = md.get("filters")
+    if isinstance(filters, dict):
+        filters = [filters]
+    elif not isinstance(filters, list):
+        filters = []
+
+    labels = [
+        f"{f.get('field')}:{f.get('value')}"
+        for f in filters
+        if isinstance(f, dict) and f.get("field")
+    ]
+    entity_label = "; ".join(labels) if labels else ""
+
     return {
         "type": "entity",
         "entity_type": md.get("entity_type"),
         "entity_id": md.get('entity_id'),
-        "entity_label": f"{md.get('filters', {}).get('field')}:{md.get('filters', {}).get('value')}"
-        # "canonical_name": md.get("canonical_name"),
-        # "aliases": md.get("aliases", []),
+        "entity_label": entity_label,
     }
 
 
@@ -458,7 +457,7 @@ def build_schema_fvs_job():
         # schema = clean_schema(schema,schema_path)
         schema_docs = build_schema_docs(schema)
         app_base, _, _, schema_path, _,schema_emb_dir, _ = _get_fvs_paths()
-        _build_and_save_faiss(schema_docs, schema_path, "ERPNext Schema FVS", app_base)
+        # _build_and_save_faiss(schema_docs, schema_path, "ERPNext Schema FVS", app_base)
         save_field_matrix(schema_docs, schema_emb_dir)
         frappe.logger().info(f"ERPNext Schema FVS built: {len(schema_docs)} docs")
     except Exception :

changai/changai/api/v2/retrieve.py

@@ -518,6 +518,7 @@ def call_fvs_field_search_global_k(
         is_table = meta.get("is_table")
         table = meta.get("table")
         field = meta.get("field") or meta.get("name")
+        grain = meta.get("grain", "")
 
         if not table or not field:
             continue
@@ -562,6 +563,7 @@ def call_fvs_field_search_global_k(
 
         grouped.setdefault(table, {
             "is_table": is_table,
+            "grain":grain,
             "fields": []
         })
         grouped[table]["fields"].append(name)

changai/changai/api/v2/schema_utils.py

@@ -30,6 +30,8 @@
 JSON_EXT = ".json"
 YAML_EXT = ".yaml"
 
+
+
 def get_report_filter_fields(report_name: str):
     try:
         script = get_script(report_name).get("script") or ""
@@ -61,12 +63,24 @@ def phonetic_bucket():
     master_items = master_data_content["data"]
     for item in master_items:
         table = item["entity_type"]
-        field = item["filters"]["field"]
-        value = item["filters"]["value"]
-        _VALUE_TO_FIELD[value] = f"{table}.{field}:{value}"
-        first_word = value.split()[0]
-        key = jellyfish.metaphone(first_word)
-        _PHONETIC_BUCKETS[key].append(value)
+        filters = item.get("filters")
+        if isinstance(filters, dict):
+            filters = [filters]
+        elif not isinstance(filters, list):
+            continue
+
+        for f in filters:
+            if not isinstance(f, dict):
+                continue
+            field = f.get("field")
+            value = f.get("value")
+            if not field or not value:
+                continue
+
+            _VALUE_TO_FIELD[value] = f"{table}.{field}:{value}"
+            first_word = value.split()[0]
+            key = jellyfish.metaphone(first_word)
+            _PHONETIC_BUCKETS[key].append(value)
 
 
 @frappe.whitelist(allow_guest=False)
@@ -210,18 +224,31 @@ def is_doctype_schema_changed(doc, last_sync):
     latest = max(candidates, default=None)
     return bool(latest and last_sync and latest > get_datetime(last_sync))
 
-def is_master_data_changed(last_sync, stored_data: list):
+def is_master_data_changed(last_sync: str, stored_data: list):
     for doc in MASTER_DOCTYPES:
         meta = frappe.get_meta(doc)
         title_field = meta.title_field or "name"
         entity_type = f"tab{doc}"
-
-        # ✅ Only compare rows matching title_field
         allowed_fields = [f.fieldname for f in meta.fields] + ["name"]
         if title_field not in allowed_fields:
             frappe.log_error(f"Invalid title_field: {title_field}", "is_master_data_changed")
             continue
-
+        stored_titles = set()
+        for row in stored_data:
+            if (row.get("entity_type") != entity_type):
+                continue
+            filters = row.get("filters")
+            if isinstance(filters, dict):
+                filters = [filters]
+            elif not isinstance(filters, list):
+                filters = []
+            for f in filters:
+                if (
+                    isinstance(f, dict)
+                    and f.get("field") == title_field
+                    and f.get("value")
+                ):
+                    stored_titles.add(f.get("value"))
         live_records = frappe.get_all(
             doc,
             fields=[title_field],
@@ -231,13 +258,13 @@ def is_master_data_changed(last_sync, stored_data: list):
         for rec in live_records:
             if rec.get(title_field):
                 live_titles.add(rec.get(title_field))
-
+        if stored_titles == live_titles:
+            return False
         if stored_titles != live_titles:
             return True
-
     return False
 
-@frappe.whitelist(allow_guest=False)
+
 def check_file_updates(file_name: str):
     RAG_FOLDER = "Home/RAG Sources"
     from changai.changai.api.v2.build_cards_faiss_index_v2 import _read_file_doc
@@ -279,7 +306,6 @@ def check_file_updates(file_name: str):
             stored_data = parsed.get("data", []) if isinstance(parsed, dict) else []
         else:
             stored_data = []
-
         if is_master_data_changed(last_sync, stored_data):
             changed = True
 
@@ -421,6 +447,7 @@ def format_schema_context(grouped: dict) -> str:
         if isinstance(table_data, dict):
             raw_fields = table_data.get("fields", [])
             is_table_value = table_data.get("is_table")
+            grain = table_data.get("grain", "") 
 
             if is_table_value is None:
                 child = is_child_table(table)
@@ -429,11 +456,14 @@ def format_schema_context(grouped: dict) -> str:
         else:
             raw_fields = table_data
             child = is_child_table(table)
+            grain = ""
 
         fields = enrich_fields_for_sql_context(table, raw_fields)
 
         parts.append(f"TABLE: {table}")
         parts.append(f"TYPE: {'Child Table' if child else 'Main Table'}")
+        if grain:
+            parts.append(f"GRAIN: {grain}")
 
         if child:
             parts.append("JOIN RULES:")