From f70a83abcf03738c877946c65aa62dfba7fc19bb Mon Sep 17 00:00:00 2001 From: Anna Chi Date: Tue, 11 Aug 2026 17:17:38 +0100 Subject: [PATCH] fix(cli, api): round trip the overall extraction instruction in ixp packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An IXP project's overall extraction instruction is fed to the model as part of its prompt, so a project uploaded from a package was running a different prompt from the one the package was downloaded from, and scored differently in Measure as a result — with no indication anything had been dropped. The instruction lives on the `default` label group rather than on the dataset, and neither `LabelGroup` nor `UpdateDataset` modelled it, so it was lost when the package was written and never sent when it was read back. The label defs the group is created with do not cover it. Add it to both, and send it from `re package upload` alongside the model config it already restores. Only send it when the packaged project has one, so uploading an older package leaves the new project's own default in place rather than blanking it. Field group, field and field type instructions were already round tripped; this was the only level that was not. --- CHANGELOG.md | 6 +++ api/src/resources/dataset.rs | 59 ++++++++++++++++++++++++++++++ api/src/resources/label_group.rs | 8 ++++ cli/src/commands/package/upload.rs | 19 ++++++++++ cli/src/commands/update/dataset.rs | 1 + cli/tests/test_datasets.rs | 1 + 6 files changed, 94 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a15894..bd9fd035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Unreleased +- Fix `re package download`/`re package upload` silently dropping an IXP project's overall + extraction instruction, so an uploaded project ran a different prompt from the one it was + downloaded from and scored differently in Measure. `LabelGroup` gains an `instructions` field and + `UpdateDataset` a `default_label_group_instructions` field +- Breaking (`reinfer-client` API): `LabelGroup` and `UpdateDataset` each gain a field, so struct + literals of them need an extra initializer - Update `--help` text, the READMEs and the crate descriptions to refer to UiPath IXP rather than Re:infer, and print `re` rather than `reinfer-cli` in `re --help` and `re --version`. The config file location (`~/.config/reinfer`), the `REINFER_CLI_NUM_THREADS` environment variable, the diff --git a/api/src/resources/dataset.rs b/api/src/resources/dataset.rs index ba43fbfe..be715bcc 100644 --- a/api/src/resources/dataset.rs +++ b/api/src/resources/dataset.rs @@ -570,6 +570,16 @@ pub struct UpdateDataset<'request> { #[serde(rename = "_model_config", skip_serializing_if = "Option::is_none")] pub model_config: Option, + /// The instructions for the dataset's `default` label group, surfaced in + /// IXP projects as the "overall extraction instruction". Applied to the + /// group that already exists on the dataset, so this does not need to be + /// sent after the group's label defs. + #[serde( + rename = "_default_label_group_instructions", + skip_serializing_if = "Option::is_none" + )] + pub default_label_group_instructions: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] pub entity_defs: Vec, } @@ -898,4 +908,53 @@ mod tests { r#"{"kind":"gpt_ixp","model_version":"a_model_from_the_future","flags":["a_flag_from_the_future"],"attribution_method":"table_formatted_word_ids"}"# ); } + + /// The "overall extraction instruction" is fed to the model as part of its + /// prompt, so `re package download` has to capture it for + /// `re package upload` to be able to reproduce a project's scores. + #[test] + fn test_deserialize_default_label_group_instructions() { + let dataset: Dataset = serde_json::from_str( + r#"{"id":"aaaaaaaaaaaaaaaa","name":"ixp-one","owner":"proj","title":"IXP", + "description":"","created":"2026-01-01T00:00:00Z", + "last_modified":"2026-01-01T00:00:00Z","model_family":"english", + "source_ids":[],"has_sentiment":false,"entity_defs":[],"general_fields":[], + "label_defs":[], + "label_groups":[{"name":"default","instructions":"Extract from invoices only.", + "label_defs":[]}], + "_dataset_flags":["ixp"],"_model_config":{"kind":"gpt_ixp","flags":[]}}"#, + ) + .expect("an ixp dataset must parse"); + + assert_eq!( + dataset.label_groups[0].instructions, + "Extract from invoices only." + ); + } + + /// Groups other than `default` have their instructions round-tripped by + /// the label defs themselves, so only the default group's are sent here, + /// and only when set — an absent field must leave the new dataset's own + /// default in place rather than blanking it. + #[test] + fn test_serialize_update_dataset_default_label_group_instructions() { + let with_instructions = UpdateDataset { + source_ids: None, + title: None, + description: None, + model_config: None, + default_label_group_instructions: Some("Extract from invoices only.".to_owned()), + entity_defs: Vec::new(), + }; + assert_eq!( + serde_json::to_string(&with_instructions).unwrap(), + r#"{"_default_label_group_instructions":"Extract from invoices only."}"# + ); + + let without_instructions = UpdateDataset { + default_label_group_instructions: None, + ..with_instructions + }; + assert_eq!(serde_json::to_string(&without_instructions).unwrap(), "{}"); + } } diff --git a/api/src/resources/label_group.rs b/api/src/resources/label_group.rs index 8189cc42..d3f29add 100644 --- a/api/src/resources/label_group.rs +++ b/api/src/resources/label_group.rs @@ -13,6 +13,14 @@ pub static DEFAULT_LABEL_GROUP_NAME: Lazy = Lazy::new(|| Name("default".to pub struct LabelGroup { pub name: Name, + /// For the `default` group of an IXP project this is the "overall + /// extraction instruction", which is fed to the model as part of its + /// prompt. Set it with [`UpdateDataset::default_label_group_instructions`]. + /// + /// [`UpdateDataset::default_label_group_instructions`]: crate::UpdateDataset::default_label_group_instructions + #[serde(default)] + pub instructions: String, + #[serde(default)] pub label_defs: Vec, } diff --git a/cli/src/commands/package/upload.rs b/cli/src/commands/package/upload.rs index 13b430c7..8ae6d404 100644 --- a/cli/src/commands/package/upload.rs +++ b/cli/src/commands/package/upload.rs @@ -128,6 +128,20 @@ fn wait_until( Err(anyhow!("Timeout waiting for {what}")) } +/// The packaged dataset's "overall extraction instruction", or `None` when it +/// is unset so that we leave the new dataset's own default in place. +/// +/// The instruction lives on the `default` label group rather than on the +/// dataset, so it is not covered by the label defs the group is created with. +fn default_label_group_instructions(dataset: &Dataset) -> Option { + dataset + .label_groups + .iter() + .find(|label_group| label_group.name == *DEFAULT_LABEL_GROUP_NAME) + .map(|label_group| label_group.instructions.clone()) + .filter(|instructions| !instructions.is_empty()) +} + fn create_ixp_dataset( name: DatasetName, label_defs: Vec, @@ -135,6 +149,7 @@ fn create_ixp_dataset( timeout_s: u64, model_config: ModelConfig, entity_defs: Vec, + default_label_group_instructions: Option, ) -> Result { let mut new_label_defs = Vec::new(); @@ -154,6 +169,7 @@ fn create_ixp_dataset( source_ids: None, title: None, description: None, + default_label_group_instructions, entity_defs, }, )?; @@ -841,6 +857,8 @@ fn unpack_ixp( let packaged_sources = get_ixp_source(&dataset, SourceProvider::Packaged(package)) .context("Could not get ixp source from package")?; + let default_label_group_instructions = default_label_group_instructions(&dataset); + // We use title here as the name will already have a hex appended, the api will normalize // the title into an api name let new_dataset = create_ixp_dataset( @@ -876,6 +894,7 @@ fn unpack_ixp( instructions: def.instructions, }) .collect(), + default_label_group_instructions, ) .context("Could not create dataset")?; diff --git a/cli/src/commands/update/dataset.rs b/cli/src/commands/update/dataset.rs index 0d4bd199..39f81228 100644 --- a/cli/src/commands/update/dataset.rs +++ b/cli/src/commands/update/dataset.rs @@ -59,6 +59,7 @@ pub fn update(client: &Client, args: &UpdateDatasetArgs, printer: &Printer) -> R title: title.as_deref(), description: description.as_deref(), model_config: None, + default_label_group_instructions: None, entity_defs: Vec::new(), }, ) diff --git a/cli/tests/test_datasets.rs b/cli/tests/test_datasets.rs index f1f1704d..1cb53418 100644 --- a/cli/tests/test_datasets.rs +++ b/cli/tests/test_datasets.rs @@ -289,6 +289,7 @@ fn test_create_update_dataset_custom() { ], label_groups: vec![LabelGroup { name: LabelGroupName("default".to_owned()), + instructions: String::new(), label_defs: vec![ LabelDef { name: LabelName("bar".to_owned()),