Add local website Markdown extraction

.env.example

@@ -11,6 +11,11 @@ OPENROUTER_MODEL=openai/gpt-4o-mini
 # Optional local model override. If Ollama is running, this can point to a local model.
 LOCI_LLM_MODEL=
 
+# Optional curl.md website-to-Markdown extraction. URLs are sent to curl.md only when enabled.
+LOCI_CURLMD_ENABLED=0
+CURLMD_API_KEY=
+# LOCI_CURLMD_BASE_URL=https://curl.md
+
 # Optional extraction/conversion helpers.
 LOCI_PYTHON=
 LOCI_EXTRACT_SCRIPT=

.github/workflows/release.yml

@@ -77,12 +77,22 @@ jobs:
           [[ "${ARCHS}" == *" arm64 "* ]]
           [[ "${ARCHS}" == *" x86_64 "* ]]
           codesign --verify --deep --strict --verbose=2 "${RUNNER_TEMP}/loci-zip/Loci.app"
+          test -f "${RUNNER_TEMP}/loci-zip/Loci.app/Contents/Resources/SwiftPM/Loci_Loci.bundle/AppIcon.png"
+          test -f "${RUNNER_TEMP}/loci-zip/Loci.app/Contents/Resources/SwiftPM/GRDB_GRDB.bundle/PrivacyInfo.xcprivacy"
           xcrun stapler validate "${RUNNER_TEMP}/loci-zip/Loci.app"
           spctl -a -vv -t exec "${RUNNER_TEMP}/loci-zip/Loci.app"
           hdiutil verify "${DMG}"
           codesign --verify --verbose=2 "${DMG}"
           xcrun stapler validate "${DMG}"
           spctl -a -vv -t open --context context:primary-signature "${DMG}"
+          MOUNT_DIR="${RUNNER_TEMP}/loci-dmg"
+          mkdir -p "${MOUNT_DIR}"
+          hdiutil attach "${DMG}" -nobrowse -readonly -mountpoint "${MOUNT_DIR}"
+          test -d "${MOUNT_DIR}/Loci.app"
+          test -L "${MOUNT_DIR}/Applications"
+          test "$(readlink "${MOUNT_DIR}/Applications")" = "/Applications"
+          codesign --verify --deep --strict --verbose=2 "${MOUNT_DIR}/Loci.app"
+          hdiutil detach "${MOUNT_DIR}"
           unzip -t "${ZIP}"
           (cd dist && shasum -a 256 -c SHA256SUMS.txt)
 

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to Loci will be documented here.
 
 ## Unreleased
 
+- Added local rendered website-to-Markdown extraction with deterministic noise removal, quality scoring, diagnostics, and an optional curl.md fallback for weak results.
+- Made extraction/compilation automation settings authoritative and store optional curl.md API keys in macOS Keychain.
+- Fixed macOS distribution packaging to embed SwiftPM resources, build universal binaries without requiring full Xcode locally, and verify the app and Applications shortcut from inside the DMG.
 - Added open-source README, license, contribution guide, security policy, issue templates, environment example, project structure docs, and integrations docs.
 - Documented privacy boundaries for telemetry and LLM workflows.
 - Documented release packaging, X OAuth setup, local API, browser extension, and portable library locations.

README.md

@@ -30,6 +30,7 @@ Loci keeps references visible and connected instead of scattering them across br
 - Search by text, dominant color, and visual similarity.
 - Preview images, websites, PDFs, Office documents, and text files.
 - Extract text with Vision OCR and optional document-processing helpers.
+- Turn rendered websites into clean local Markdown, with optional curl.md fallback for weak extractions.
 - Build and export an Obsidian-compatible Markdown vault.
 - Use local search without an LLM, or optionally connect OpenRouter or Ollama.
 - Keep telemetry off by default; when enabled, only allowlisted aggregate events are recorded.
@@ -79,6 +80,7 @@ Loci is local-first:
 - The local API binds to loopback by default and protects sensitive routes with a bearer token.
 - Telemetry is disabled by default and excludes file contents, URLs, bookmark text, prompts, model responses, tokens, and local paths.
 - LLM features are optional. Source text is sent only when you choose a configured external provider.
+- Local website extraction is enabled by default and can be disabled; the optional curl.md fallback sends an eligible website URL only after you enable it.
 
 Read the complete [Telemetry and Privacy](docs/TELEMETRY_AND_PRIVACY.md) and [Security](SECURITY.md) policies.
 

Sources/Loci/AutoRulesEngine.swift

@@ -101,7 +101,13 @@ enum AutoRulesEngine {
         let rules = allRules().filter { $0.isEnabled }
         for rule in rules {
             guard shouldTrigger(rule: rule, fileExtension: fileExtension, sourceURL: sourceURL) else { continue }
-            executeAction(rule: rule, itemID: itemID)
+            executeAction(
+                rule: rule,
+                itemID: itemID,
+                source: nil,
+                payload: nil,
+                importPipelineAlreadyQueued: false
+            )
             incrementRunCount(ruleID: rule.id)
         }
     }
@@ -122,7 +128,13 @@ enum AutoRulesEngine {
                 matchesTrigger = true
             }
             guard matchesTrigger else { continue }
-            executeAction(rule: rule, itemID: itemID)
+            executeAction(
+                rule: rule,
+                itemID: itemID,
+                source: source,
+                payload: payload,
+                importPipelineAlreadyQueued: true
+            )
             incrementRunCount(ruleID: rule.id)
         }
     }
@@ -140,24 +152,78 @@ enum AutoRulesEngine {
         }
     }
 
-    private static func executeAction(rule: AutoRule, itemID: ReferenceItem.ID) {
+    private static func executeAction(
+        rule: AutoRule,
+        itemID: ReferenceItem.ID,
+        source: ImportSourceKind?,
+        payload: String?,
+        importPipelineAlreadyQueued: Bool
+    ) {
         switch rule.action {
         case .autoTag:
             let tagName = rule.name.replacingOccurrences(of: " ", with: "-").lowercased()
             TagHierarchy.addTag(tagName, to: itemID)
         case .autoCollection:
             break
         case .autoExtract:
-            Task { @MainActor in
-                await ImportCoordinator.shared.enqueueProcess()
+            // Avoid duplicating work already queued by the import pipeline; direct rule
+            // evaluation still supplies an extraction job when no pipeline job exists.
+            if !importPipelineAlreadyQueued || !ImportAutomationSettings.shouldRunExtractionOnImport {
+                enqueuePipeline(itemID: itemID, source: source, payload: payload, compile: false)
             }
         case .autoCompile:
-            Task { @MainActor in
-                await ImportCoordinator.shared.enqueueProcess()
+            if !importPipelineAlreadyQueued || !ImportAutomationSettings.autoCompileEnabled {
+                enqueuePipeline(
+                    itemID: itemID,
+                    source: source,
+                    payload: payload,
+                    compile: true,
+                    extractionAlreadyQueued: importPipelineAlreadyQueued
+                        && ImportAutomationSettings.autoExtractEnabled
+                )
             }
         }
     }
 
+    private static func enqueuePipeline(
+        itemID: ReferenceItem.ID,
+        source: ImportSourceKind?,
+        payload: String?,
+        compile: Bool,
+        extractionAlreadyQueued: Bool = false
+    ) {
+        guard let persistence = LociPersistentStore.shared,
+              let item = persistence.loadReference(id: itemID) else { return }
+        let resolvedPayload: String
+        if let payload {
+            resolvedPayload = payload
+        } else if source == .file || item.websiteURL == nil {
+            resolvedPayload = persistence.originalsURL.appendingPathComponent(item.fileName).path
+        } else {
+            resolvedPayload = item.websiteURL?.absoluteString ?? item.subtitle
+        }
+
+        if !extractionAlreadyQueued,
+           !persistence.hasPendingImportJob(source: .extract, referenceID: itemID) {
+            persistence.enqueueImportJob(
+                source: .extract,
+                payload: resolvedPayload,
+                status: .queued,
+                referenceID: itemID
+            )
+        }
+        if compile,
+           !persistence.hasPendingImportJob(source: .wikiCompile, referenceID: itemID) {
+            persistence.enqueueImportJob(
+                source: .wikiCompile,
+                payload: resolvedPayload,
+                status: .queued,
+                referenceID: itemID
+            )
+        }
+        Task { await ImportCoordinator.shared.enqueueProcess() }
+    }
+
     private static func incrementRunCount(ruleID: UUID) {
         guard let queue = LociPersistentStore.shared?.grdbQueue else { return }
         let now = ISO8601DateFormatter().string(from: Date())