diff --git a/iii-directory/README.md b/iii-directory/README.md index 7ace3c410..240f1587a 100644 --- a/iii-directory/README.md +++ b/iii-directory/README.md @@ -101,6 +101,7 @@ registry_url: https://api.workers.iii.dev # workers registry base URL download_timeout_ms: 60000 # per git-clone / HTTP request timeout (ms) registry_cache_ttl_ms: 60000 # in-process TTL for registry::workers::* responses filter_unregistered: true # hide skills whose namespace isn't an installed worker + # (explicitly repo-downloaded namespaces stay visible) ``` The `skills_folder` is created on first download if it doesn't exist. diff --git a/iii-directory/src/functions/download.rs b/iii-directory/src/functions/download.rs index 57d3770bf..29158c9c7 100644 --- a/iii-directory/src/functions/download.rs +++ b/iii-directory/src/functions/download.rs @@ -313,12 +313,12 @@ pub(crate) async fn run_download( std::fs::create_dir_all(&folder) .map_err(|e| format!("create_dir_all {}: {e}", folder.display()))?; - match classified { + let result = match classified { ClassifiedInput::Repo { repo, skill, branch, - } => sources::git::download(repo, skill, branch, &folder, cfg.download_timeout_ms).await, + } => sources::git::download(repo, skill, branch, &folder, cfg.download_timeout_ms).await?, ClassifiedInput::Registry { worker, spec } => { sources::registry::download( cfg.registry_base(), @@ -327,9 +327,23 @@ pub(crate) async fn run_download( &folder, cfg.download_timeout_ms, ) - .await + .await? + } + }; + + // Record provenance for every function-path download, mirroring the + // auto-download path: the reconcile keys on the marker, and the + // visibility filter keeps repo-sourced namespaces because of it. + match classified { + ClassifiedInput::Repo { branch, .. } => { + write_marker(&folder, &result.namespace, "repo", branch)?; + } + ClassifiedInput::Registry { worker, spec } => { + write_completion_marker(&folder, worker, spec)?; } } + + Ok(result) } fn build_output(classified: &ClassifiedInput, result: DownloadResult) -> DownloadOutput { @@ -393,10 +407,12 @@ async fn fan_out( // ────────────────── completion marker ────────────────────────────────── // -// After a successful registry download, a `.iii-skill-complete` JSON -// marker is written inside the namespace directory. The reconcile path -// treats a namespace as "present" only if the marker exists — this -// prevents half-downloaded namespaces from hiding a needed re-download. +// After a successful download, a `.iii-skill-complete` JSON marker is +// written inside the namespace directory. The reconcile path treats a +// namespace as "present" only if the marker exists — this prevents +// half-downloaded namespaces from hiding a needed re-download — and the +// visibility filter keeps repo-sourced namespaces because their marker +// records the explicit pull. /// Marker filename written inside a namespace after a complete download. const COMPLETION_MARKER: &str = ".iii-skill-complete"; @@ -410,31 +426,76 @@ struct CompletionMarker { schema: u32, } -/// Write the completion marker under `//`. -fn write_completion_marker( +/// Write a completion marker under `//`. +/// `source` is `"registry"` or `"repo"`; `tag_or_version` records the +/// requested version/tag (registry) or branch (repo). +fn write_marker( skills_folder: &std::path::Path, - worker: &str, - spec: &VersionSpec, + namespace: &str, + source: &str, + tag_or_version: &str, ) -> Result<(), String> { let marker = CompletionMarker { - worker: worker.to_string(), - source: "registry".to_string(), - tag_or_version: match spec { - VersionSpec::Version(v) => v.clone(), - VersionSpec::Tag(t) => t.clone(), - }, + worker: namespace.to_string(), + source: source.to_string(), + tag_or_version: tag_or_version.to_string(), schema: 1, }; let json = serde_json::to_string_pretty(&marker).map_err(|e| format!("encode marker: {e}"))?; - let dest = skills_folder.join(worker).join(COMPLETION_MARKER); + let dest = skills_folder.join(namespace).join(COMPLETION_MARKER); sources::write_file_atomic(&dest, json.as_bytes()) } +/// Write the registry completion marker under `//`. +fn write_completion_marker( + skills_folder: &std::path::Path, + worker: &str, + spec: &VersionSpec, +) -> Result<(), String> { + let tag_or_version = match spec { + VersionSpec::Version(v) => v.as_str(), + VersionSpec::Tag(t) => t.as_str(), + }; + write_marker(skills_folder, worker, "registry", tag_or_version) +} + /// Check if a completion marker exists for `worker` under `skills_folder`. pub fn has_completion_marker(skills_folder: &std::path::Path, worker: &str) -> bool { skills_folder.join(worker).join(COMPLETION_MARKER).exists() } +/// Namespaces under `skills_folder` whose completion marker records an +/// explicit repo download. These represent knowledge the operator pulled +/// on purpose (e.g. the README quickstart's `repo=` examples), so the +/// visibility filter keeps them even though no worker by that name is +/// installed. An unreadable or unparseable marker is simply not counted. +pub fn repo_pinned_namespaces( + skills_folder: &std::path::Path, +) -> std::collections::HashSet { + let mut pinned = std::collections::HashSet::new(); + let Ok(entries) = std::fs::read_dir(skills_folder) else { + return pinned; + }; + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Ok(raw) = std::fs::read_to_string(path.join(COMPLETION_MARKER)) else { + continue; + }; + let Ok(marker) = serde_json::from_str::(&raw) else { + continue; + }; + if marker.source == "repo" { + if let Some(name) = entry.file_name().to_str() { + pinned.insert(name.to_string()); + } + } + } + pinned +} + // ────────────────── auto-download helper ────────────────────────────── // // auto-download flow (ASCII): diff --git a/iii-directory/src/functions/skills.rs b/iii-directory/src/functions/skills.rs index 3f7e8bcff..0d971e376 100644 --- a/iii-directory/src/functions/skills.rs +++ b/iii-directory/src/functions/skills.rs @@ -402,8 +402,10 @@ pub async fn resolve_visible_skills( cache.get_or_fetch(iii).await }; + let pinned = super::download::repo_pinned_namespaces(&cfg.resolved_skills_folder()); + match registered { - Some(registered) => filter_to_registered(merged, ®istered), + Some(registered) => filter_to_registered(merged, ®istered, &pinned), None => { tracing::info!( "no cached registered workers and daemon unreachable; \ @@ -433,11 +435,16 @@ pub const ENGINE_NAMESPACE: &str = "iii"; /// `registered` set, but its skill is always visible. /// 4. Its top namespace segment is in the `registered` set (i.e. it /// belongs to an installed worker). +/// 5. Its top namespace segment is in the `pinned` set — namespaces +/// whose completion marker records an explicit repo download; the +/// operator pulled them on purpose, so no matching worker is +/// required. /// /// Everything else (skills from uninstalled workers) is dropped. pub(crate) fn filter_to_registered( merged: Vec, registered: &HashSet, + pinned: &HashSet, ) -> Vec { merged .into_iter() @@ -451,6 +458,8 @@ pub(crate) fn filter_to_registered( || top_seg == ENGINE_NAMESPACE // Belongs to a registered (installed) worker. || registered.contains(top_seg) + // Explicitly repo-downloaded knowledge. + || pinned.contains(top_seg) }) .collect() } @@ -2875,7 +2884,7 @@ First paragraph. fn filter_keeps_root_doc_without_namespace() { let registered = HashSet::from(["resend".to_string()]); let merged = vec![fs_skill("index")]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); assert_eq!(result.len(), 1); assert_eq!(result[0].id, "index"); } @@ -2884,11 +2893,24 @@ First paragraph. fn filter_keeps_directory_namespace_docs() { let registered = HashSet::new(); // nothing registered let merged = vec![fs_skill("directory/engine/functions/info")]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); assert_eq!(result.len(), 1); assert_eq!(result[0].id, "directory/engine/functions/info"); } + #[test] + fn filter_keeps_repo_pinned_namespace() { + let registered = HashSet::new(); // no worker matches either namespace + let pinned = HashSet::from(["repo-pulled".to_string()]); + let merged = vec![ + fs_skill("repo-pulled/index"), + fs_skill("leftover-worker/index"), + ]; + let result = filter_to_registered(merged, ®istered, &pinned); + assert_eq!(result.len(), 1); + assert_eq!(result[0].id, "repo-pulled/index"); + } + // ── engine_fallback_worker ───────────────────────────────────────── #[test] @@ -2941,7 +2963,7 @@ First paragraph. fn filter_keeps_engine_namespace_docs() { let registered = HashSet::new(); // nothing registered; `iii` is not a worker let merged = vec![fs_skill("iii/index"), fs_skill("iii/SKILL")]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); let ids: Vec<&str> = result.iter().map(|s| s.id.as_str()).collect(); assert!(ids.contains(&"iii/index")); assert!(ids.contains(&"iii/SKILL")); @@ -3207,7 +3229,7 @@ First paragraph. fn filter_keeps_registered_worker_skills() { let registered = HashSet::from(["resend".to_string()]); let merged = vec![fs_skill("resend/index"), fs_skill("resend/emails/send")]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); assert_eq!(result.len(), 2); } @@ -3220,7 +3242,7 @@ First paragraph. fs_skill("index"), fs_skill("directory/skills/list"), ]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); let ids: Vec<&str> = result.iter().map(|s| s.id.as_str()).collect(); assert!(ids.contains(&"resend/index")); assert!(ids.contains(&"index")); @@ -3232,7 +3254,7 @@ First paragraph. fn filter_drops_resend_when_not_registered() { let registered = HashSet::from(["agent-memory".to_string()]); let merged = vec![fs_skill("resend/index"), fs_skill("agent-memory/index")]; - let result = filter_to_registered(merged, ®istered); + let result = filter_to_registered(merged, ®istered, &HashSet::new()); assert_eq!(result.len(), 1); assert_eq!(result[0].id, "agent-memory/index"); }