Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
# Unreleased
- Fix `re package download`/`re package upload` silently dropping an IXP project's overall
extraction instruction, so an uploaded project ran a different prompt from the one it was
downloaded from and scored differently in Measure. `LabelGroup` gains an `instructions` field and
`UpdateDataset` a `default_label_group_instructions` field
- Breaking (`reinfer-client` API): `LabelGroup` and `UpdateDataset` each gain a field, so struct
literals of them need an extra initializer
- Update `--help` text, the READMEs and the crate descriptions to refer to UiPath IXP rather than
Re:infer, and print `re` rather than `reinfer-cli` in `re --help` and `re --version`. The config
file location (`~/.config/reinfer`), the `REINFER_CLI_NUM_THREADS` environment variable, the
Expand Down
59 changes: 59 additions & 0 deletions api/src/resources/dataset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,16 @@ pub struct UpdateDataset<'request> {
#[serde(rename = "_model_config", skip_serializing_if = "Option::is_none")]
pub model_config: Option<ModelConfig>,

/// The instructions for the dataset's `default` label group, surfaced in
/// IXP projects as the "overall extraction instruction". Applied to the
/// group that already exists on the dataset, so this does not need to be
/// sent after the group's label defs.
#[serde(
rename = "_default_label_group_instructions",
skip_serializing_if = "Option::is_none"
)]
pub default_label_group_instructions: Option<String>,

#[serde(skip_serializing_if = "Vec::is_empty")]
pub entity_defs: Vec<NewEntityDef>,
}
Expand Down Expand Up @@ -898,4 +908,53 @@ mod tests {
r#"{"kind":"gpt_ixp","model_version":"a_model_from_the_future","flags":["a_flag_from_the_future"],"attribution_method":"table_formatted_word_ids"}"#
);
}

/// The "overall extraction instruction" is fed to the model as part of its
/// prompt, so `re package download` has to capture it for
/// `re package upload` to be able to reproduce a project's scores.
#[test]
fn test_deserialize_default_label_group_instructions() {
let dataset: Dataset = serde_json::from_str(
r#"{"id":"aaaaaaaaaaaaaaaa","name":"ixp-one","owner":"proj","title":"IXP",
"description":"","created":"2026-01-01T00:00:00Z",
"last_modified":"2026-01-01T00:00:00Z","model_family":"english",
"source_ids":[],"has_sentiment":false,"entity_defs":[],"general_fields":[],
"label_defs":[],
"label_groups":[{"name":"default","instructions":"Extract from invoices only.",
"label_defs":[]}],
"_dataset_flags":["ixp"],"_model_config":{"kind":"gpt_ixp","flags":[]}}"#,
)
.expect("an ixp dataset must parse");

assert_eq!(
dataset.label_groups[0].instructions,
"Extract from invoices only."
);
}

/// Groups other than `default` have their instructions round-tripped by
/// the label defs themselves, so only the default group's are sent here,
/// and only when set — an absent field must leave the new dataset's own
/// default in place rather than blanking it.
#[test]
fn test_serialize_update_dataset_default_label_group_instructions() {
let with_instructions = UpdateDataset {
source_ids: None,
title: None,
description: None,
model_config: None,
default_label_group_instructions: Some("Extract from invoices only.".to_owned()),
entity_defs: Vec::new(),
};
assert_eq!(
serde_json::to_string(&with_instructions).unwrap(),
r#"{"_default_label_group_instructions":"Extract from invoices only."}"#
);

let without_instructions = UpdateDataset {
default_label_group_instructions: None,
..with_instructions
};
assert_eq!(serde_json::to_string(&without_instructions).unwrap(), "{}");
}
}
8 changes: 8 additions & 0 deletions api/src/resources/label_group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ pub static DEFAULT_LABEL_GROUP_NAME: Lazy<Name> = Lazy::new(|| Name("default".to
pub struct LabelGroup {
pub name: Name,

/// For the `default` group of an IXP project this is the "overall
/// extraction instruction", which is fed to the model as part of its
/// prompt. Set it with [`UpdateDataset::default_label_group_instructions`].
///
/// [`UpdateDataset::default_label_group_instructions`]: crate::UpdateDataset::default_label_group_instructions
#[serde(default)]
pub instructions: String,

#[serde(default)]
pub label_defs: Vec<LabelDef>,
}
Expand Down
19 changes: 19 additions & 0 deletions cli/src/commands/package/upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,28 @@ fn wait_until(
Err(anyhow!("Timeout waiting for {what}"))
}

/// The packaged dataset's "overall extraction instruction", or `None` when it
/// is unset so that we leave the new dataset's own default in place.
///
/// The instruction lives on the `default` label group rather than on the
/// dataset, so it is not covered by the label defs the group is created with.
fn default_label_group_instructions(dataset: &Dataset) -> Option<String> {
dataset
.label_groups
.iter()
.find(|label_group| label_group.name == *DEFAULT_LABEL_GROUP_NAME)
.map(|label_group| label_group.instructions.clone())
.filter(|instructions| !instructions.is_empty())
}

fn create_ixp_dataset(
name: DatasetName,
label_defs: Vec<LabelDef>,
client: &Client,
timeout_s: u64,
model_config: ModelConfig,
entity_defs: Vec<NewEntityDef>,
default_label_group_instructions: Option<String>,
) -> Result<Dataset> {
let mut new_label_defs = Vec::new();

Expand All @@ -154,6 +169,7 @@ fn create_ixp_dataset(
source_ids: None,
title: None,
description: None,
default_label_group_instructions,
entity_defs,
},
)?;
Expand Down Expand Up @@ -841,6 +857,8 @@ fn unpack_ixp(
let packaged_sources = get_ixp_source(&dataset, SourceProvider::Packaged(package))
.context("Could not get ixp source from package")?;

let default_label_group_instructions = default_label_group_instructions(&dataset);

// We use title here as the name will already have a hex appended, the api will normalize
// the title into an api name
let new_dataset = create_ixp_dataset(
Expand Down Expand Up @@ -876,6 +894,7 @@ fn unpack_ixp(
instructions: def.instructions,
})
.collect(),
default_label_group_instructions,
)
.context("Could not create dataset")?;

Expand Down
1 change: 1 addition & 0 deletions cli/src/commands/update/dataset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ pub fn update(client: &Client, args: &UpdateDatasetArgs, printer: &Printer) -> R
title: title.as_deref(),
description: description.as_deref(),
model_config: None,
default_label_group_instructions: None,
entity_defs: Vec::new(),
},
)
Expand Down
1 change: 1 addition & 0 deletions cli/tests/test_datasets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ fn test_create_update_dataset_custom() {
],
label_groups: vec![LabelGroup {
name: LabelGroupName("default".to_owned()),
instructions: String::new(),
label_defs: vec![
LabelDef {
name: LabelName("bar".to_owned()),
Expand Down
Loading