Making URL as key in json

.gitignore

@@ -4,4 +4,5 @@ demo_repo
 UiPathProject/
 output
 __pycache__
-notes.txt
+notes.txt
+*.pyc

main.py

@@ -2,7 +2,7 @@
 from extractor.element_extractor import ElementExtractor
 from repository.json_storage import JsonStorage
 from screenshot.screenshot_service import ScreenshotService
-from utils.name_utils import get_base_name, get_screen_name
+from utils.name_utils import get_base_name, get_screen_name, get_page_key
 from screenshot.draw_annotator import DrawAnnotator
 from playwright.sync_api import Error
 
@@ -40,12 +40,13 @@ def main():
     def handle_ui_change():
 
         try:
+            print("Waiting for load state...")
             page.wait_for_load_state("networkidle")
             page.wait_for_timeout(700)
         except:
             pass
 
-        name = get_screen_name(page)
+        name = get_page_key(page)
 
         # prevent duplicate extraction
         if name == last_screen["name"]:
@@ -85,7 +86,10 @@ def on_nav(frame):
             return
 
         try:
+            print("Navigated to:", frame.url)
+            print("Waiting for load state...")
             page.wait_for_load_state("load")
+            print("Waiting for network idle...")
             page.wait_for_load_state("networkidle")
             page.wait_for_timeout(700)
         except:

utils/name_utils.py

@@ -26,5 +26,33 @@ def get_screen_name(page):
 
         return title.strip()[:120]
 
+    except:
+        return "unknown_page"
+
+
+def get_page_key(page):
+
+    try:
+        url = page.url
+
+        parsed = urlparse(url)
+
+        path = parsed.path.strip("/").replace("/", "_")
+
+        if not path:
+            path = "home"
+
+        query = parsed.query
+
+        if query:
+            query = query.replace("=", "_").replace("&", "_")
+            path = f"{path}_{query}"
+
+        name = f"{parsed.netloc}_{path}"
+
+        name = re.sub(r'[\\/*?:"<>|]', "_", name)
+
+        return name[:200]
+
     except:
         return "unknown_page"