diff --git a/CHANGELOG.md b/CHANGELOG.md index b5a15894..bd9fd035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Unreleased +- Fix `re package download`/`re package upload` silently dropping an IXP project's overall + extraction instruction, so an uploaded project ran a different prompt from the one it was + downloaded from and scored differently in Measure. `LabelGroup` gains an `instructions` field and + `UpdateDataset` a `default_label_group_instructions` field +- Breaking (`reinfer-client` API): `LabelGroup` and `UpdateDataset` each gain a field, so struct + literals of them need an extra initializer - Update `--help` text, the READMEs and the crate descriptions to refer to UiPath IXP rather than Re:infer, and print `re` rather than `reinfer-cli` in `re --help` and `re --version`. The config file location (`~/.config/reinfer`), the `REINFER_CLI_NUM_THREADS` environment variable, the diff --git a/api/src/resources/dataset.rs b/api/src/resources/dataset.rs index ba43fbfe..be715bcc 100644 --- a/api/src/resources/dataset.rs +++ b/api/src/resources/dataset.rs @@ -570,6 +570,16 @@ pub struct UpdateDataset<'request> { #[serde(rename = "_model_config", skip_serializing_if = "Option::is_none")] pub model_config: Option, + /// The instructions for the dataset's `default` label group, surfaced in + /// IXP projects as the "overall extraction instruction". Applied to the + /// group that already exists on the dataset, so this does not need to be + /// sent after the group's label defs. + #[serde( + rename = "_default_label_group_instructions", + skip_serializing_if = "Option::is_none" + )] + pub default_label_group_instructions: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] pub entity_defs: Vec, } @@ -898,4 +908,53 @@ mod tests { r#"{"kind":"gpt_ixp","model_version":"a_model_from_the_future","flags":["a_flag_from_the_future"],"attribution_method":"table_formatted_word_ids"}"# ); } + + /// The "overall extraction instruction" is fed to the model as part of its + /// prompt, so `re package download` has to capture it for + /// `re package upload` to be able to reproduce a project's scores. + #[test] + fn test_deserialize_default_label_group_instructions() { + let dataset: Dataset = serde_json::from_str( + r#"{"id":"aaaaaaaaaaaaaaaa","name":"ixp-one","owner":"proj","title":"IXP", + "description":"","created":"2026-01-01T00:00:00Z", + "last_modified":"2026-01-01T00:00:00Z","model_family":"english", + "source_ids":[],"has_sentiment":false,"entity_defs":[],"general_fields":[], + "label_defs":[], + "label_groups":[{"name":"default","instructions":"Extract from invoices only.", + "label_defs":[]}], + "_dataset_flags":["ixp"],"_model_config":{"kind":"gpt_ixp","flags":[]}}"#, + ) + .expect("an ixp dataset must parse"); + + assert_eq!( + dataset.label_groups[0].instructions, + "Extract from invoices only." + ); + } + + /// Groups other than `default` have their instructions round-tripped by + /// the label defs themselves, so only the default group's are sent here, + /// and only when set — an absent field must leave the new dataset's own + /// default in place rather than blanking it. + #[test] + fn test_serialize_update_dataset_default_label_group_instructions() { + let with_instructions = UpdateDataset { + source_ids: None, + title: None, + description: None, + model_config: None, + default_label_group_instructions: Some("Extract from invoices only.".to_owned()), + entity_defs: Vec::new(), + }; + assert_eq!( + serde_json::to_string(&with_instructions).unwrap(), + r#"{"_default_label_group_instructions":"Extract from invoices only."}"# + ); + + let without_instructions = UpdateDataset { + default_label_group_instructions: None, + ..with_instructions + }; + assert_eq!(serde_json::to_string(&without_instructions).unwrap(), "{}"); + } } diff --git a/api/src/resources/label_group.rs b/api/src/resources/label_group.rs index 8189cc42..d3f29add 100644 --- a/api/src/resources/label_group.rs +++ b/api/src/resources/label_group.rs @@ -13,6 +13,14 @@ pub static DEFAULT_LABEL_GROUP_NAME: Lazy = Lazy::new(|| Name("default".to pub struct LabelGroup { pub name: Name, + /// For the `default` group of an IXP project this is the "overall + /// extraction instruction", which is fed to the model as part of its + /// prompt. Set it with [`UpdateDataset::default_label_group_instructions`]. + /// + /// [`UpdateDataset::default_label_group_instructions`]: crate::UpdateDataset::default_label_group_instructions + #[serde(default)] + pub instructions: String, + #[serde(default)] pub label_defs: Vec, } diff --git a/cli/src/commands/package/upload.rs b/cli/src/commands/package/upload.rs index 13b430c7..8ae6d404 100644 --- a/cli/src/commands/package/upload.rs +++ b/cli/src/commands/package/upload.rs @@ -128,6 +128,20 @@ fn wait_until( Err(anyhow!("Timeout waiting for {what}")) } +/// The packaged dataset's "overall extraction instruction", or `None` when it +/// is unset so that we leave the new dataset's own default in place. +/// +/// The instruction lives on the `default` label group rather than on the +/// dataset, so it is not covered by the label defs the group is created with. +fn default_label_group_instructions(dataset: &Dataset) -> Option { + dataset + .label_groups + .iter() + .find(|label_group| label_group.name == *DEFAULT_LABEL_GROUP_NAME) + .map(|label_group| label_group.instructions.clone()) + .filter(|instructions| !instructions.is_empty()) +} + fn create_ixp_dataset( name: DatasetName, label_defs: Vec, @@ -135,6 +149,7 @@ fn create_ixp_dataset( timeout_s: u64, model_config: ModelConfig, entity_defs: Vec, + default_label_group_instructions: Option, ) -> Result { let mut new_label_defs = Vec::new(); @@ -154,6 +169,7 @@ fn create_ixp_dataset( source_ids: None, title: None, description: None, + default_label_group_instructions, entity_defs, }, )?; @@ -841,6 +857,8 @@ fn unpack_ixp( let packaged_sources = get_ixp_source(&dataset, SourceProvider::Packaged(package)) .context("Could not get ixp source from package")?; + let default_label_group_instructions = default_label_group_instructions(&dataset); + // We use title here as the name will already have a hex appended, the api will normalize // the title into an api name let new_dataset = create_ixp_dataset( @@ -876,6 +894,7 @@ fn unpack_ixp( instructions: def.instructions, }) .collect(), + default_label_group_instructions, ) .context("Could not create dataset")?; diff --git a/cli/src/commands/update/dataset.rs b/cli/src/commands/update/dataset.rs index 0d4bd199..39f81228 100644 --- a/cli/src/commands/update/dataset.rs +++ b/cli/src/commands/update/dataset.rs @@ -59,6 +59,7 @@ pub fn update(client: &Client, args: &UpdateDatasetArgs, printer: &Printer) -> R title: title.as_deref(), description: description.as_deref(), model_config: None, + default_label_group_instructions: None, entity_defs: Vec::new(), }, ) diff --git a/cli/tests/test_datasets.rs b/cli/tests/test_datasets.rs index f1f1704d..1cb53418 100644 --- a/cli/tests/test_datasets.rs +++ b/cli/tests/test_datasets.rs @@ -289,6 +289,7 @@ fn test_create_update_dataset_custom() { ], label_groups: vec![LabelGroup { name: LabelGroupName("default".to_owned()), + instructions: String::new(), label_defs: vec![ LabelDef { name: LabelName("bar".to_owned()),