From 85a2a67f517ce532ef1556d5709c3dcdc027a8fd Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 30 May 2026 12:12:41 +0100 Subject: [PATCH 01/17] chore(refactor): add manifest list reader --- crates/iceberg/src/catalog/utils.rs | 6 ++- crates/iceberg/src/inspect/manifests.rs | 4 +- crates/iceberg/src/io/object_cache.rs | 11 +++--- crates/iceberg/src/spec/manifest_list.rs | 37 ++++++++++++++++++- crates/iceberg/src/spec/snapshot.rs | 1 + crates/iceberg/src/table.rs | 7 +++- crates/iceberg/src/transaction/append.rs | 14 +++---- crates/iceberg/src/transaction/mod.rs | 13 +++---- crates/iceberg/src/transaction/snapshot.rs | 6 ++- .../datafusion/src/physical_plan/commit.rs | 5 ++- 10 files changed, 73 insertions(+), 31 deletions(-) diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index d450f9df80..aa66f0f86f 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -23,7 +23,7 @@ use futures::{TryStreamExt, stream}; use crate::Result; use crate::io::FileIO; -use crate::spec::TableMetadata; +use crate::spec::{ManifestListReader, TableMetadata}; const DELETE_CONCURRENCY: usize = 10; @@ -47,7 +47,9 @@ pub async fn drop_table_data( // Load all manifest lists concurrently let results: Vec<_> = futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { - let manifest_list = snapshot.load_manifest_list(io, metadata).await?; + let manifest_list = ManifestListReader::new(snapshot, io, metadata) + .load() + .await?; Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) })) .await?; diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 4c30ca2ec5..a985786460 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -161,9 +161,7 @@ impl<'a> ManifestsTable<'a> { let mut partition_summaries = self.partition_summary_builder()?; if let Some(snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) - .await?; + let manifest_list = self.table.manifest_list_reader(snapshot).load().await?; for manifest in manifest_list.entries() { content.append_value(manifest.content as i32); path.append_value(manifest.manifest_path.clone()); diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 5de45e2acc..7e0e96f46d 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -20,7 +20,8 @@ use std::sync::Arc; use crate::io::FileIO; use crate::spec::{ - FormatVersion, Manifest, ManifestFile, ManifestList, SchemaId, SnapshotRef, TableMetadataRef, + FormatVersion, Manifest, ManifestFile, ManifestList, ManifestListReader, SchemaId, SnapshotRef, + TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -126,8 +127,8 @@ impl ObjectCache { table_metadata: &TableMetadataRef, ) -> Result> { if self.cache_disabled { - return snapshot - .load_manifest_list(&self.file_io, table_metadata) + return ManifestListReader::new(snapshot, &self.file_io, table_metadata) + .load() .await .map(Arc::new); } @@ -173,8 +174,8 @@ impl ObjectCache { snapshot: &SnapshotRef, table_metadata: &TableMetadataRef, ) -> Result { - let manifest_list = snapshot - .load_manifest_list(&self.file_io, table_metadata) + let manifest_list = ManifestListReader::new(snapshot, &self.file_io, table_metadata) + .load() .await?; Ok(CachedItem::ManifestList(Arc::new(manifest_list))) diff --git a/crates/iceberg/src/spec/manifest_list.rs b/crates/iceberg/src/spec/manifest_list.rs index baaab1f590..73d0ee607f 100644 --- a/crates/iceberg/src/spec/manifest_list.rs +++ b/crates/iceberg/src/spec/manifest_list.rs @@ -28,7 +28,7 @@ use serde_derive::{Deserialize, Serialize}; use self::_const_schema::{MANIFEST_LIST_AVRO_SCHEMA_V1, MANIFEST_LIST_AVRO_SCHEMA_V2}; use self::_serde::{ManifestFileV1, ManifestFileV2}; -use super::{FormatVersion, Manifest}; +use super::{FormatVersion, Manifest, Snapshot, TableMetadata}; use crate::error::Result; use crate::io::{FileIO, OutputFile}; use crate::spec::manifest_list::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V3; @@ -90,6 +90,41 @@ impl ManifestList { } } +/// A manifest list reader that encapsulates the logic for loading and parsing a [`ManifestList`] +/// from a snapshot. +pub struct ManifestListReader<'a> { + snapshot: &'a Snapshot, + file_io: &'a FileIO, + table_metadata: &'a TableMetadata, +} + +impl<'a> ManifestListReader<'a> { + pub(crate) fn new( + snapshot: &'a Snapshot, + file_io: &'a FileIO, + table_metadata: &'a TableMetadata, + ) -> Self { + Self { + snapshot, + file_io, + table_metadata, + } + } + + /// Loads and returns the [`ManifestList`] for this snapshot. + pub async fn load(&self) -> Result { + let manifest_list_content = self + .file_io + .new_input(self.snapshot.manifest_list())? + .read() + .await?; + ManifestList::parse_with_version( + &manifest_list_content, + self.table_metadata.format_version(), + ) + } +} + /// A manifest list writer. pub struct ManifestListWriter { format_version: FormatVersion, diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 3b8a3c934e..3894e3c329 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -195,6 +195,7 @@ impl Snapshot { } /// Load manifest list. + #[deprecated(since = "0.9.0", note = "Use `Table::manifest_list_reader()` instead")] pub async fn load_manifest_list( &self, file_io: &FileIO, diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index d2ba93f854..47234db33b 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -25,7 +25,7 @@ use crate::io::FileIO; use crate::io::object_cache::ObjectCache; use crate::runtime::Runtime; use crate::scan::TableScanBuilder; -use crate::spec::{SchemaRef, TableMetadata, TableMetadataRef}; +use crate::spec::{ManifestListReader, SchemaRef, Snapshot, TableMetadata, TableMetadataRef}; use crate::{Error, ErrorKind, Result, TableIdent}; /// Builder to create table scan. @@ -264,6 +264,11 @@ impl Table { self.metadata.current_schema().clone() } + /// Creates a [`ManifestListReader`] for the given snapshot. + pub fn manifest_list_reader<'a>(&'a self, snapshot: &'a Snapshot) -> ManifestListReader<'a> { + ManifestListReader::new(snapshot, &self.file_io, &self.metadata) + } + /// Create a reader for the table. pub fn reader_builder(&self) -> ArrowReaderBuilder { ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone()) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 08d4032409..ec4ceee277 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -128,11 +128,10 @@ impl SnapshotProduceOperation for FastAppendOperation { return Ok(vec![]); }; - let manifest_list = snapshot - .load_manifest_list( - snapshot_produce.table.file_io(), - &snapshot_produce.table.metadata_ref(), - ) + let manifest_list = snapshot_produce + .table + .manifest_list_reader(snapshot) + .load() .await?; Ok(manifest_list @@ -304,8 +303,9 @@ mod tests { } else { unreachable!() }; - let manifest_list = new_snapshot - .load_manifest_list(table.file_io(), table.metadata()) + let manifest_list = table + .manifest_list_reader(new_snapshot) + .load() .await .unwrap(); assert_eq!(1, manifest_list.entries().len()); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 04ee1997d0..5460919671 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -605,11 +605,10 @@ mod test_row_lineage { assert_eq!(table.metadata().next_row_id(), 30); // Check written manifest for first_row_id + let snapshot = table.metadata().current_snapshot().unwrap(); let manifest_list = table - .metadata() - .current_snapshot() - .unwrap() - .load_manifest_list(table.file_io(), table.metadata()) + .manifest_list_reader(snapshot) + .load() .await .unwrap(); @@ -634,10 +633,8 @@ mod test_row_lineage { // Check written manifest for first_row_id let manifest_list = table - .metadata() - .current_snapshot() - .unwrap() - .load_manifest_list(table.file_io(), table.metadata()) + .manifest_list_reader(snapshot) + .load() .await .unwrap(); assert_eq!(manifest_list.entries().len(), 2); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 8f643a7d1e..497467b37c 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -172,8 +172,10 @@ impl<'a> SnapshotProducer<'a> { let mut referenced_files = Vec::new(); if let Some(current_snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = current_snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) + let manifest_list = self + .table + .manifest_list_reader(current_snapshot) + .load() .await?; for manifest_list_entry in manifest_list.entries() { let manifest = manifest_list_entry diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index 3b3ff3d6b3..835c804908 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -504,8 +504,9 @@ mod tests { let current_snapshot = updated_table.metadata().current_snapshot().unwrap(); // Load the manifest list to verify the data files were added - let manifest_list = current_snapshot - .load_manifest_list(updated_table.file_io(), updated_table.metadata()) + let manifest_list = updated_table + .manifest_list_reader(current_snapshot) + .load() .await?; // There should be at least one manifest From 0bcc80a2cf0afd752232cf757229c49066f2a502 Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 30 May 2026 12:19:59 +0100 Subject: [PATCH 02/17] fmt --- crates/iceberg/src/transaction/mod.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 5460919671..b68a53e5e3 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -606,11 +606,7 @@ mod test_row_lineage { // Check written manifest for first_row_id let snapshot = table.metadata().current_snapshot().unwrap(); - let manifest_list = table - .manifest_list_reader(snapshot) - .load() - .await - .unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 1); let manifest_file = &manifest_list.entries()[0]; @@ -632,11 +628,7 @@ mod test_row_lineage { assert_eq!(table.metadata().next_row_id(), 30 + 17 + 11); // Check written manifest for first_row_id - let manifest_list = table - .manifest_list_reader(snapshot) - .load() - .await - .unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 2); let manifest_file = &manifest_list.entries()[1]; assert_eq!(manifest_file.first_row_id, Some(30)); From 3fd956b6e827440cd6f3aaf38f2d213469e95b12 Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 30 May 2026 12:25:34 +0100 Subject: [PATCH 03/17] public-api-lock --- crates/iceberg/public-api.txt | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index af74bff6fe..2bd888faa2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -7506,6 +7506,55 @@ impl bnum::cast::As for iceberg::spec::ManifestList pub fn iceberg::spec::ManifestList::as_(self) -> T where T: bnum::cast::CastFrom impl ppv_lite86::types::VZip for iceberg::spec::ManifestList where V: ppv_lite86::types::MultiLane pub fn iceberg::spec::ManifestList::vzip(self) -> V +pub struct iceberg::spec::ManifestListReader<'a> +impl<'a> iceberg::spec::ManifestListReader<'a> +pub async fn iceberg::spec::ManifestListReader<'a>::load(&self) -> iceberg::Result +impl<'a> core::marker::Freeze for iceberg::spec::ManifestListReader<'a> +impl<'a> core::marker::Send for iceberg::spec::ManifestListReader<'a> +impl<'a> core::marker::Sync for iceberg::spec::ManifestListReader<'a> +impl<'a> core::marker::Unpin for iceberg::spec::ManifestListReader<'a> +impl<'a> !core::panic::unwind_safe::RefUnwindSafe for iceberg::spec::ManifestListReader<'a> +impl<'a> !core::panic::unwind_safe::UnwindSafe for iceberg::spec::ManifestListReader<'a> +impl core::convert::Into for iceberg::spec::ManifestListReader<'a> where U: core::convert::From +pub fn iceberg::spec::ManifestListReader<'a>::into(self) -> U +impl core::convert::TryFrom for iceberg::spec::ManifestListReader<'a> where U: core::convert::Into +pub type iceberg::spec::ManifestListReader<'a>::Error = core::convert::Infallible +pub fn iceberg::spec::ManifestListReader<'a>::try_from(value: U) -> core::result::Result>::Error> +impl core::convert::TryInto for iceberg::spec::ManifestListReader<'a> where U: core::convert::TryFrom +pub type iceberg::spec::ManifestListReader<'a>::Error = >::Error +pub fn iceberg::spec::ManifestListReader<'a>::try_into(self) -> core::result::Result>::Error> +impl as_any::AsAny for iceberg::spec::ManifestListReader<'a> where T: core::any::Any +pub fn iceberg::spec::ManifestListReader<'a>::as_any(&self) -> &(dyn core::any::Any + 'static) +pub fn iceberg::spec::ManifestListReader<'a>::as_any_mut(&mut self) -> &mut (dyn core::any::Any + 'static) +pub fn iceberg::spec::ManifestListReader<'a>::type_name(&self) -> &'static str +impl as_any::Downcast for iceberg::spec::ManifestListReader<'a> where T: as_any::AsAny + ?core::marker::Sized +impl core::any::Any for iceberg::spec::ManifestListReader<'a> where T: 'static + ?core::marker::Sized +pub fn iceberg::spec::ManifestListReader<'a>::type_id(&self) -> core::any::TypeId +impl core::borrow::Borrow for iceberg::spec::ManifestListReader<'a> where T: ?core::marker::Sized +pub fn iceberg::spec::ManifestListReader<'a>::borrow(&self) -> &T +impl core::borrow::BorrowMut for iceberg::spec::ManifestListReader<'a> where T: ?core::marker::Sized +pub fn iceberg::spec::ManifestListReader<'a>::borrow_mut(&mut self) -> &mut T +impl core::convert::From for iceberg::spec::ManifestListReader<'a> +pub fn iceberg::spec::ManifestListReader<'a>::from(t: T) -> T +impl crossbeam_epoch::atomic::Pointable for iceberg::spec::ManifestListReader<'a> +pub type iceberg::spec::ManifestListReader<'a>::Init = T +pub const iceberg::spec::ManifestListReader<'a>::ALIGN: usize +pub unsafe fn iceberg::spec::ManifestListReader<'a>::deref<'a>(ptr: usize) -> &'a T +pub unsafe fn iceberg::spec::ManifestListReader<'a>::deref_mut<'a>(ptr: usize) -> &'a mut T +pub unsafe fn iceberg::spec::ManifestListReader<'a>::drop(ptr: usize) +pub unsafe fn iceberg::spec::ManifestListReader<'a>::init(init: ::Init) -> usize +impl either::into_either::IntoEither for iceberg::spec::ManifestListReader<'a> +impl tower_http::follow_redirect::policy::PolicyExt for iceberg::spec::ManifestListReader<'a> where T: ?core::marker::Sized +pub fn iceberg::spec::ManifestListReader<'a>::and(self, other: P) -> tower_http::follow_redirect::policy::and::And where T: tower_http::follow_redirect::policy::Policy, P: tower_http::follow_redirect::policy::Policy +pub fn iceberg::spec::ManifestListReader<'a>::or(self, other: P) -> tower_http::follow_redirect::policy::or::Or where T: tower_http::follow_redirect::policy::Policy, P: tower_http::follow_redirect::policy::Policy +impl tracing::instrument::Instrument for iceberg::spec::ManifestListReader<'a> +impl tracing::instrument::WithSubscriber for iceberg::spec::ManifestListReader<'a> +impl typenum::type_operators::Same for iceberg::spec::ManifestListReader<'a> +pub type iceberg::spec::ManifestListReader<'a>::Output = T +impl bnum::cast::As for iceberg::spec::ManifestListReader<'a> +pub fn iceberg::spec::ManifestListReader<'a>::as_(self) -> T where T: bnum::cast::CastFrom +impl ppv_lite86::types::VZip for iceberg::spec::ManifestListReader<'a> where V: ppv_lite86::types::MultiLane +pub fn iceberg::spec::ManifestListReader<'a>::vzip(self) -> V pub struct iceberg::spec::ManifestListWriter impl iceberg::spec::ManifestListWriter pub fn iceberg::spec::ManifestListWriter::add_manifests(&mut self, manifests: impl core::iter::traits::iterator::Iterator) -> iceberg::Result<()> @@ -11004,6 +11053,7 @@ pub fn iceberg::table::Table::current_schema_ref(&self) -> iceberg::spec::Schema pub fn iceberg::table::Table::file_io(&self) -> &iceberg::io::FileIO pub fn iceberg::table::Table::identifier(&self) -> &iceberg::TableIdent pub fn iceberg::table::Table::inspect(&self) -> iceberg::inspect::MetadataTable<'_> +pub fn iceberg::table::Table::manifest_list_reader<'a>(&'a self, snapshot: &'a iceberg::spec::Snapshot) -> iceberg::spec::ManifestListReader<'a> pub fn iceberg::table::Table::metadata(&self) -> &iceberg::spec::TableMetadata pub fn iceberg::table::Table::metadata_location(&self) -> core::option::Option<&str> pub fn iceberg::table::Table::metadata_location_result(&self) -> iceberg::Result<&str> From df5300d994948dd311858303a86fccba123b756e Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 10:44:49 +0100 Subject: [PATCH 04/17] Read encrypted manifest lists --- Cargo.lock | 507 ++++---------------- crates/iceberg/src/io/object_cache.rs | 25 +- crates/iceberg/src/spec/snapshot.rs | 136 +++++- crates/iceberg/src/spec/table_properties.rs | 10 + crates/iceberg/src/table.rs | 265 +++++++++- crates/iceberg/src/transaction/append.rs | 5 + 6 files changed, 527 insertions(+), 421 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 774f9c86c3..370fbf8a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,54 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - [[package]] name = "adler2" version = "2.0.1" @@ -305,7 +257,6 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", - "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", @@ -328,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" dependencies = [ "ahash", "arrow-buffer", @@ -347,9 +298,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" dependencies = [ "bytes", "half", @@ -396,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" dependencies = [ "arrow-buffer", "arrow-schema", @@ -449,9 +400,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" dependencies = [ "arrow-array", "arrow-buffer", @@ -460,18 +411,6 @@ dependencies = [ "arrow-select", ] -[[package]] -name = "arrow-pyarrow" -version = "58.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" -dependencies = [ - "arrow-array", - "arrow-data", - "arrow-schema", - "pyo3", -] - [[package]] name = "arrow-row" version = "58.1.0" @@ -487,20 +426,19 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" dependencies = [ - "bitflags", "serde_core", "serde_json", ] [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" dependencies = [ "ahash", "arrow-array", @@ -512,9 +450,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8" dependencies = [ "arrow-array", "arrow-buffer", @@ -533,18 +471,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "assert-json-diff" version = "2.0.2" @@ -579,15 +505,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-ffi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] - [[package]] name = "async-lock" version = "3.4.2" @@ -607,7 +524,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -618,7 +535,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1179,13 +1096,13 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.23.0", + "darling 0.20.11", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn", ] [[package]] @@ -1209,15 +1126,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bstr" version = "1.12.1" @@ -1252,7 +1160,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1404,7 +1312,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1443,7 +1351,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1581,21 +1489,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_extensions" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" -dependencies = [ - "core_extensions_proc_macros", -] - -[[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - [[package]] name = "countio" version = "0.3.0" @@ -1783,7 +1676,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn", ] [[package]] @@ -1796,7 +1689,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn", ] [[package]] @@ -1807,7 +1700,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1818,7 +1711,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2227,36 +2120,6 @@ dependencies = [ "paste", ] -[[package]] -name = "datafusion-ffi" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c" -dependencies = [ - "abi_stable", - "arrow", - "arrow-schema", - "async-ffi", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-proto", - "datafusion-proto-common", - "datafusion-session", - "futures", - "log", - "prost", - "semver", - "tokio", -] - [[package]] name = "datafusion-functions" version = "53.1.0" @@ -2401,7 +2264,7 @@ checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2531,45 +2394,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-proto" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" -dependencies = [ - "arrow", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-datasource", - "datafusion-datasource-arrow", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-table", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-proto-common", - "object_store", - "prost", - "rand 0.9.4", -] - -[[package]] -name = "datafusion-proto-common" -version = "53.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" -dependencies = [ - "arrow", - "datafusion-common", - "prost", -] - [[package]] name = "datafusion-pruning" version = "53.1.0" @@ -2732,7 +2556,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2742,7 +2566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.117", + "syn", ] [[package]] @@ -2781,7 +2605,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2792,7 +2616,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2858,7 +2682,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2899,7 +2723,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2949,7 +2773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3206,7 +3030,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3253,15 +3077,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if 1.0.4", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -3342,7 +3157,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3653,7 +3468,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -3738,7 +3553,6 @@ dependencies = [ "strum", "tempfile", "tokio", - "tracing", "typed-builder", "typetag", "url", @@ -4230,7 +4044,7 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4241,7 +4055,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4286,7 +4100,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn", ] [[package]] @@ -4305,7 +4119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4446,16 +4260,6 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if 1.0.4", - "winapi", -] - [[package]] name = "liblzma" version = "0.4.6" @@ -4726,7 +4530,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4800,7 +4604,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4826,7 +4630,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4890,7 +4694,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4984,7 +4788,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5573,7 +5377,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5732,7 +5536,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn", ] [[package]] @@ -5778,7 +5582,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.117", + "syn", "tempfile", ] @@ -5792,7 +5596,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5831,78 +5635,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "pyiceberg_core_rust" -version = "0.9.0" -dependencies = [ - "arrow", - "datafusion-ffi", - "iceberg", - "iceberg-datafusion", - "iceberg-storage-opendal", - "pyo3", - "tokio", -] - -[[package]] -name = "pyo3" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" -dependencies = [ - "libc", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", -] - -[[package]] -name = "pyo3-build-config" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" -dependencies = [ - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.28.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -5944,7 +5677,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -5982,9 +5715,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6122,7 +5855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6180,7 +5913,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6240,15 +5973,6 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - [[package]] name = "reqsign-aliyun-oss" version = "3.0.0" @@ -6498,7 +6222,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6557,7 +6281,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.117", + "syn", "unicode-ident", ] @@ -6599,7 +6323,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6657,7 +6381,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6790,7 +6514,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn", ] [[package]] @@ -6910,7 +6634,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6921,7 +6645,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6945,7 +6669,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -6966,7 +6690,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.117", + "syn", ] [[package]] @@ -6983,12 +6707,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19" dependencies = [ "base64", - "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -7003,14 +6726,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7292,7 +7015,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7354,7 +7077,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.117", + "syn", ] [[package]] @@ -7375,7 +7098,7 @@ dependencies = [ "sha2", "sqlx-core", "sqlx-sqlite", - "syn 2.0.117", + "syn", "tokio", "url", ] @@ -7552,7 +7275,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7585,7 +7308,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.117", + "syn", "typify", "walkdir", ] @@ -7602,17 +7325,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -7641,7 +7353,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7685,12 +7397,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "target-lexicon" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" - [[package]] name = "tempfile" version = "3.27.0" @@ -7701,7 +7407,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7736,7 +7442,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7747,7 +7453,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -7860,7 +7566,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -8058,7 +7764,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -8119,33 +7825,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typed-builder" version = "0.20.1" @@ -8163,7 +7848,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -8199,14 +7884,14 @@ checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] name = "typewit" -version = "1.15.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" +checksum = "bc19094686c694eb41b3b99dcc2f2975d4b078512fa22ae6c63f7ca318bdcff7" [[package]] name = "typify" @@ -8233,7 +7918,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.117", + "syn", "thiserror 2.0.18", "unicode-ident", ] @@ -8251,7 +7936,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.117", + "syn", "typify-impl", ] @@ -8575,7 +8260,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -8740,7 +8425,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -8802,7 +8487,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -8813,7 +8498,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -9149,7 +8834,7 @@ dependencies = [ "heck", "indexmap 2.13.0", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -9165,7 +8850,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -9401,7 +9086,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -9422,7 +9107,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -9442,7 +9127,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -9482,7 +9167,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 7e0e96f46d..c6526f7532 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -18,6 +18,7 @@ use std::mem::size_of_val; use std::sync::Arc; +use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ FormatVersion, Manifest, ManifestFile, ManifestList, ManifestListReader, SchemaId, SnapshotRef, @@ -45,20 +46,25 @@ pub struct ObjectCache { cache: moka::future::Cache, file_io: FileIO, cache_disabled: bool, + encryption_manager: Option>, } impl ObjectCache { /// Creates a new [`ObjectCache`] /// with the default cache size - pub(crate) fn new(file_io: FileIO) -> Self { - Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES) + pub(crate) fn new(file_io: FileIO, encryption_manager: Option>) -> Self { + Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES, encryption_manager) } /// Creates a new [`ObjectCache`] /// with a specific cache size - pub(crate) fn new_with_capacity(file_io: FileIO, cache_size_bytes: u64) -> Self { + pub(crate) fn new_with_capacity( + file_io: FileIO, + cache_size_bytes: u64, + encryption_manager: Option>, + ) -> Self { if cache_size_bytes == 0 { - Self::with_disabled_cache(file_io) + Self::with_disabled_cache(file_io, encryption_manager) } else { Self { cache: moka::future::Cache::builder() @@ -70,17 +76,22 @@ impl ObjectCache { .build(), file_io, cache_disabled: false, + encryption_manager, } } } /// Creates a new [`ObjectCache`] /// with caching disabled - pub(crate) fn with_disabled_cache(file_io: FileIO) -> Self { + pub(crate) fn with_disabled_cache( + file_io: FileIO, + encryption_manager: Option>, + ) -> Self { Self { cache: moka::future::Cache::new(0), file_io, cache_disabled: true, + encryption_manager, } } @@ -320,7 +331,7 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone()); + let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone(), None); let result_manifest_list = object_cache .get_manifest_list( @@ -353,7 +364,7 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::new(fixture.table.file_io().clone()); + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); // not in cache let result_manifest_list = object_cache diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 3894e3c329..aee2c56a54 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -26,6 +26,7 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::table_metadata::SnapshotLog; +use crate::encryption::{EncryptedInputFile, EncryptionManager}; use crate::error::{Result, timestamp_ms_to_utc}; use crate::io::FileIO; use crate::spec::{ManifestList, SchemaId, SchemaRef, TableMetadata}; @@ -200,8 +201,22 @@ impl Snapshot { &self, file_io: &FileIO, table_metadata: &TableMetadata, + encryption_manager: Option<&EncryptionManager>, ) -> Result { - let manifest_list_content = file_io.new_input(&self.manifest_list)?.read().await?; + let manifest_list_content = match (&self.encryption_key_id, encryption_manager) { + (Some(_), None) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snapshot has encryption_key_id but no EncryptionManager configured on Table", + )); + } + (Some(key_id), Some(em)) => { + let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; + let input = file_io.new_input(&self.manifest_list)?; + EncryptedInputFile::new(input, key_metadata).read().await? + } + (None, _) => file_io.new_input(&self.manifest_list)?.read().await?, + }; ManifestList::parse_with_version( &manifest_list_content, // TODO: You don't really need the version since you could just project any Avro in @@ -522,6 +537,7 @@ impl SnapshotRetention { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use chrono::{TimeZone, Utc}; @@ -730,4 +746,122 @@ mod tests { assert_eq!(v2_snapshot.parent_snapshot_id(), None); assert_eq!(v2_snapshot.schema_id(), None); } + + use bytes::Bytes; + + use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; + use crate::encryption::{EncryptionManager, StandardKeyMetadata}; + use crate::io::FileIO; + use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; + + const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + fn encryption_test_metadata() -> TableMetadata { + serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() + } + + fn encryption_test_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + fn encryption_test_manager() -> EncryptionManager { + EncryptionManager::builder() + .kms_client(encryption_test_kms()) + .table_key_id("master-1") + .build() + } + + async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { + Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(manifest_list_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_encryption_key_id(key_id) + .build() + } + + #[tokio::test] + async fn load_manifest_list_errors_when_encrypted_but_no_manager_configured() { + let io = FileIO::new_with_memory(); + let snapshot = snapshot_pointing_at( + "memory:///table/metadata/manifest-list-enc.avro", + Some("k1".to_string()), + ); + let metadata = encryption_test_metadata(); + + let err = snapshot + .load_manifest_list(&io, &metadata, None) + .await + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn load_manifest_list_decrypts_roundtrip() { + let io = FileIO::new_with_memory(); + let plain_path = "memory:///table/metadata/manifest-list-plain.avro"; + let encrypted_path = "memory:///table/metadata/manifest-list-enc.avro"; + + // Build raw manifest list bytes via the standard writer. + let raw_bytes = write_v3_manifest_list_bytes(&io, plain_path).await; + + // Encrypt those bytes to a second path and capture the file's key metadata. + let mgr = encryption_test_manager(); + let encrypted_output = mgr.encrypt(io.new_output(encrypted_path).unwrap()); + let std_key_metadata: StandardKeyMetadata = encrypted_output.key_metadata().clone(); + encrypted_output.write(raw_bytes).await.unwrap(); + + // Wrap the file's key metadata with a KEK and record the resulting wrapped + // entry's id on the snapshot. + let key_id = mgr + .encrypt_manifest_list_key_metadata(&std_key_metadata) + .await + .unwrap(); + + let snapshot = snapshot_pointing_at(encrypted_path, Some(key_id)); + let metadata = encryption_test_metadata(); + + let manifest_list: ManifestList = snapshot + .load_manifest_list(&io, &metadata, Some(&mgr)) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } } diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index dc21da565c..fecfebf9d4 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -128,6 +128,9 @@ pub struct TableProperties { pub cdc_max_chunk_size: usize, /// Content-defined chunking normalization level (gearhash bit adjustment). pub cdc_norm_level: i32, + /// The master key id used to encrypt this table's manifest list and data + /// files. `None` if `encryption.key-id` is not set. + pub encryption_key_id: Option, } impl TableProperties { @@ -253,6 +256,10 @@ impl TableProperties { "write.parquet.content-defined-chunking.norm-level"; /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; + + /// Property key for the master key id used to encrypt the table's manifest + /// list and data files. Matches Java's `TableProperties.ENCRYPTION_TABLE_KEY`. + pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; } impl TryFrom<&HashMap> for TableProperties { @@ -322,6 +329,9 @@ impl TryFrom<&HashMap> for TableProperties { TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL, TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, )?, + encryption_key_id: props + .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) + .cloned(), }) } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 47234db33b..1a8f82fb09 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -20,6 +20,8 @@ use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; +use crate::encryption::EncryptionManager; +use crate::encryption::kms::KeyManagementClient; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; @@ -34,6 +36,7 @@ pub struct TableBuilder { metadata_location: Option, metadata: Option, identifier: Option, + kms_client: Option>, readonly: bool, disable_cache: bool, cache_size_bytes: Option, @@ -47,6 +50,7 @@ impl TableBuilder { metadata_location: None, metadata: None, identifier: None, + kms_client: None, readonly: false, disable_cache: false, cache_size_bytes: None, @@ -104,6 +108,16 @@ impl TableBuilder { self } + /// optional - sets the KMS client used to unwrap keys for table encryption. + /// + /// If the table metadata has the `encryption.key-id` property set, a + /// [`KeyManagementClient`] must be provided here so the table can build + /// an [`EncryptionManager`]; otherwise [`Self::build`] will return an error. + pub fn kms_client(mut self, kms_client: Arc) -> Self { + self.kms_client = Some(kms_client); + self + } + /// build the Table pub fn build(self) -> Result { let Self { @@ -111,6 +125,7 @@ impl TableBuilder { metadata_location, metadata, identifier, + kms_client, readonly, disable_cache, cache_size_bytes, @@ -145,15 +160,24 @@ impl TableBuilder { )); }; + let encryption_manager = maybe_configure_encryption(kms_client.as_ref(), &metadata)?; + let object_cache = if disable_cache { - Arc::new(ObjectCache::with_disabled_cache(file_io.clone())) + Arc::new(ObjectCache::with_disabled_cache( + file_io.clone(), + encryption_manager.clone(), + )) } else if let Some(cache_size_bytes) = cache_size_bytes { Arc::new(ObjectCache::new_with_capacity( file_io.clone(), cache_size_bytes, + encryption_manager.clone(), )) } else { - Arc::new(ObjectCache::new(file_io.clone())) + Arc::new(ObjectCache::new( + file_io.clone(), + encryption_manager.clone(), + )) }; Ok(Table { @@ -164,6 +188,7 @@ impl TableBuilder { readonly, object_cache, runtime, + encryption_manager, }) } } @@ -178,6 +203,7 @@ pub struct Table { readonly: bool, object_cache: Arc, runtime: Runtime, + encryption_manager: Option>, } impl Table { @@ -238,6 +264,16 @@ impl Table { self.object_cache.clone() } + /// Returns the [`EncryptionManager`] for this table, if encryption is + /// configured. + /// + /// A manager is present iff the table metadata has the + /// `encryption.key-id` property set and a [`KeyManagementClient`] was + /// supplied to the [`TableBuilder`]. + pub fn encryption_manager(&self) -> Option<&EncryptionManager> { + self.encryption_manager.as_deref() + } + /// Creates a table scan. pub fn scan(&self) -> TableScanBuilder<'_> { TableScanBuilder::new(self) @@ -361,9 +397,54 @@ impl StaticTable { } } +/// If the table metadata sets the `encryption.key-id` property, build an +/// [`EncryptionManager`] for the table. +/// +/// Returns `Ok(None)` if the property is not set. Returns an error if the +/// property is set but no [`KeyManagementClient`] was provided. +fn maybe_configure_encryption( + kms_client: Option<&Arc>, + metadata: &TableMetadataRef, +) -> Result>> { + let Some(table_key_id) = metadata.table_properties()?.encryption_key_id else { + return Ok(None); + }; + + // Encryption is a v3 feature: `encryption-keys` table metadata and the + // snapshot `key-id` field are introduced in format version 3. + if metadata.format_version() < FormatVersion::V3 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Table encryption requires format version 3, found {}", + metadata.format_version() + ), + )); + } + + let kms_client = kms_client.ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + "Table has encryption.key-id set but no KeyManagementClient was provided to TableBuilder", + ) + })?; + + let em = EncryptionManager::builder() + .kms_client(Arc::clone(kms_client)) + .table_key_id(table_key_id) + .encryption_keys(metadata.encryption_keys.clone()) + .build(); + Ok(Some(Arc::new(em))) +} + #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; + use crate::encryption::StandardKeyMetadata; + use crate::encryption::kms::MemoryKeyManagementClient; + use crate::spec::{ManifestListWriter, Operation, Snapshot, Summary, TableProperties}; #[tokio::test] async fn test_static_table_from_file() { @@ -437,4 +518,184 @@ mod tests { assert!(!table.readonly()); assert_eq!(table.identifier.name(), "table"); } + + const V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + const V2_METADATA: &str = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {} + }"#; + + fn make_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + async fn write_empty_manifest_list_bytes(io: &FileIO, path: &str) -> bytes::Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + #[tokio::test] + async fn table_decrypts_manifest_list_via_object_cache() { + let io = FileIO::new_with_memory(); + let plain_path = "memory:///table/metadata/manifest-list-plain.avro"; + let encrypted_path = "memory:///table/metadata/manifest-list-enc.avro"; + + // Encrypt a real manifest list onto the encrypted path. + let raw = write_empty_manifest_list_bytes(&io, plain_path).await; + let kms = make_kms(); + let mgr = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id("master-1") + .build(); + let encrypted_output = mgr.encrypt(io.new_output(encrypted_path).unwrap()); + let std_km: StandardKeyMetadata = encrypted_output.key_metadata().clone(); + encrypted_output.write(raw).await.unwrap(); + let key_id = mgr + .encrypt_manifest_list_key_metadata(&std_km) + .await + .unwrap(); + + // Snapshot the wrapped keys (manifest-list entry + KEK) the manager produced. + let encryption_keys = mgr.with_encryption_keys(|keys| keys.clone()); + + // Build a TableMetadata with those keys, the encryption.key-id property, + // and a snapshot whose encryption_key_id points at the wrapped entry. + let mut metadata: TableMetadata = serde_json::from_str(V3_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + metadata.encryption_keys = encryption_keys; + + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(encrypted_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_schema_id(0) + .with_encryption_key_id(Some(key_id)) + .build(); + let snapshot_ref = Arc::new(snapshot); + metadata + .snapshots + .insert(snapshot_ref.snapshot_id(), snapshot_ref.clone()); + metadata.current_snapshot_id = Some(snapshot_ref.snapshot_id()); + + // Build the table with the KMS client, then read via the object cache. + let table = Table::builder() + .file_io(io) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(kms) + .build() + .unwrap(); + assert!(table.encryption_manager().is_some()); + + let manifest_list = table + .object_cache() + .get_manifest_list(&snapshot_ref, &table.metadata_ref()) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } + + #[tokio::test] + async fn table_builder_errors_when_encryption_key_id_set_but_no_kms() { + let mut metadata: TableMetadata = serde_json::from_str(V3_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + + let err = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .build() + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn table_builder_errors_when_encryption_set_on_pre_v3_table() { + // Encryption is a v3 spec feature; setting encryption.key-id on a v2 table + // must be rejected even when a KMS client is available. + let mut metadata: TableMetadata = serde_json::from_str(V2_METADATA).unwrap(); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + + let err = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(make_kms()) + .build() + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn table_builder_skips_encryption_when_property_absent() { + let metadata: TableMetadata = serde_json::from_str(V2_METADATA).unwrap(); + let table = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap()) + .kms_client(make_kms()) + .build() + .unwrap(); + assert!(table.encryption_manager().is_none()); + } } diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index ec4ceee277..01e1d617a3 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -303,9 +303,14 @@ mod tests { } else { unreachable!() }; +<<<<<<< HEAD let manifest_list = table .manifest_list_reader(new_snapshot) .load() +======= + let manifest_list = new_snapshot + .load_manifest_list(table.file_io(), table.metadata(), None) +>>>>>>> d0cfbb7d (Read encrypted manifest lists) .await .unwrap(); assert_eq!(1, manifest_list.entries().len()); From 9a93a515e8ff1ca4a52240a163cc257a933692a6 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 15:07:14 +0100 Subject: [PATCH 05/17] wire in purge_table --- crates/catalog/glue/src/catalog.rs | 1 + crates/catalog/hms/src/catalog.rs | 1 + crates/catalog/sql/src/catalog.rs | 1 + crates/iceberg/src/catalog/memory/catalog.rs | 1 + crates/iceberg/src/catalog/utils.rs | 2 ++ 5 files changed, 6 insertions(+) diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index c51f6a6a89..11a9ffd0bc 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -681,6 +681,7 @@ impl Catalog for GlueCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/catalog/hms/src/catalog.rs b/crates/catalog/hms/src/catalog.rs index d778a3d5fc..c76dab8ed3 100644 --- a/crates/catalog/hms/src/catalog.rs +++ b/crates/catalog/hms/src/catalog.rs @@ -628,6 +628,7 @@ impl Catalog for HmsCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index c7bf9d0cfd..4e6413fb46 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -778,6 +778,7 @@ impl Catalog for SqlCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index 3ae01a23df..2f2e2a215e 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -347,6 +347,7 @@ impl Catalog for MemoryCatalog { table_info.file_io(), table_info.metadata(), table_info.metadata_location(), + table_info.encryption_manager(), ) .await } diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index aa66f0f86f..c668d1a6ec 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -22,6 +22,7 @@ use std::collections::HashSet; use futures::{TryStreamExt, stream}; use crate::Result; +use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ManifestListReader, TableMetadata}; @@ -40,6 +41,7 @@ pub async fn drop_table_data( io: &FileIO, metadata: &TableMetadata, metadata_location: Option<&str>, + encryption_manager: Option<&EncryptionManager>, ) -> Result<()> { let mut manifest_lists_to_delete: HashSet = HashSet::new(); let mut manifests_to_delete: HashSet = HashSet::new(); From 371aca092a0e49480ebaeb4251ad8f11fbd7ab13 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 14 May 2026 15:23:51 +0100 Subject: [PATCH 06/17] fmt --- crates/iceberg/src/spec/table_properties.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index fecfebf9d4..5647e8113f 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -258,7 +258,7 @@ impl TableProperties { pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; /// Property key for the master key id used to encrypt the table's manifest - /// list and data files. Matches Java's `TableProperties.ENCRYPTION_TABLE_KEY`. + /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; } From 4415ba64d540e56170dae1b5a5f1daafb859c4ce Mon Sep 17 00:00:00 2001 From: Xander Date: Fri, 15 May 2026 11:02:26 +0100 Subject: [PATCH 07/17] key_size --- crates/iceberg/src/spec/table_properties.rs | 12 ++++++++++++ crates/iceberg/src/table.rs | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 5647e8113f..883f05e5a5 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -131,6 +131,8 @@ pub struct TableProperties { /// The master key id used to encrypt this table's manifest list and data /// files. `None` if `encryption.key-id` is not set. pub encryption_key_id: Option, + /// The encryption data encryption key length in bytes. + pub encryption_data_key_length: usize, } impl TableProperties { @@ -260,6 +262,11 @@ impl TableProperties { /// Property key for the master key id used to encrypt the table's manifest /// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/. pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id"; + + /// Property key for the encryption data encryption key (DEK) length in bytes. + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &str = "encryption.data-key-length"; + /// Default value for the encryption DEK length (16 bytes = AES-128). + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; } impl TryFrom<&HashMap> for TableProperties { @@ -332,6 +339,11 @@ impl TryFrom<&HashMap> for TableProperties { encryption_key_id: props .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) .cloned(), + encryption_data_key_length: parse_property( + props, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + )?, }) } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 1a8f82fb09..99d380b5a1 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -19,9 +19,11 @@ use std::sync::Arc; +use backon::BackoffBuilder; + use crate::arrow::ArrowReaderBuilder; -use crate::encryption::EncryptionManager; use crate::encryption::kms::KeyManagementClient; +use crate::encryption::{AesKeySize, EncryptionManager}; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; @@ -406,7 +408,8 @@ fn maybe_configure_encryption( kms_client: Option<&Arc>, metadata: &TableMetadataRef, ) -> Result>> { - let Some(table_key_id) = metadata.table_properties()?.encryption_key_id else { + let table_properties = metadata.table_properties()?; + let Some(table_key_id) = table_properties.encryption_key_id else { return Ok(None); }; @@ -433,6 +436,9 @@ fn maybe_configure_encryption( .kms_client(Arc::clone(kms_client)) .table_key_id(table_key_id) .encryption_keys(metadata.encryption_keys.clone()) + .key_size(AesKeySize::from_key_length( + table_properties.encryption_data_key_length, + )?) .build(); Ok(Some(Arc::new(em))) } From 88052dd316b1dad33c4fc1a71c030b8e2383ac90 Mon Sep 17 00:00:00 2001 From: Xander Date: Fri, 15 May 2026 13:24:37 +0100 Subject: [PATCH 08/17] Whoops --- crates/iceberg/src/table.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 99d380b5a1..f78cc88e72 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -19,8 +19,6 @@ use std::sync::Arc; -use backon::BackoffBuilder; - use crate::arrow::ArrowReaderBuilder; use crate::encryption::kms::KeyManagementClient; use crate::encryption::{AesKeySize, EncryptionManager}; From 90c6d46ffd326b24847c82b4653b57124eaf530a Mon Sep 17 00:00:00 2001 From: Xander Date: Tue, 19 May 2026 15:27:44 +0100 Subject: [PATCH 09/17] fix tests --- crates/iceberg/src/table.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index f78cc88e72..a183208283 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -641,6 +641,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) .kms_client(kms) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap(); assert!(table.encryption_manager().is_some()); @@ -665,6 +666,7 @@ mod tests { .file_io(FileIO::new_with_memory()) .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); @@ -685,6 +687,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); @@ -698,6 +701,7 @@ mod tests { .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap()) .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap(); assert!(table.encryption_manager().is_none()); From aaaa71f2e0e6cfac3cf2f5595e4b836b10c7efe9 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 18:58:24 +0100 Subject: [PATCH 10/17] Use precondition error --- crates/iceberg/src/spec/snapshot.rs | 2 +- crates/iceberg/src/table.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index aee2c56a54..624a5bd2eb 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -206,7 +206,7 @@ impl Snapshot { let manifest_list_content = match (&self.encryption_key_id, encryption_manager) { (Some(_), None) => { return Err(Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, "Snapshot has encryption_key_id but no EncryptionManager configured on Table", )); } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index a183208283..9cd1e842dc 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -415,7 +415,7 @@ fn maybe_configure_encryption( // snapshot `key-id` field are introduced in format version 3. if metadata.format_version() < FormatVersion::V3 { return Err(Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, format!( "Table encryption requires format version 3, found {}", metadata.format_version() @@ -425,7 +425,7 @@ fn maybe_configure_encryption( let kms_client = kms_client.ok_or_else(|| { Error::new( - ErrorKind::FeatureUnsupported, + ErrorKind::PreconditionFailed, "Table has encryption.key-id set but no KeyManagementClient was provided to TableBuilder", ) })?; From b159391bb63ed1a77d977927c4052727d0b785b1 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 19:33:55 +0100 Subject: [PATCH 11/17] hoist test functions and imports --- crates/iceberg/src/spec/snapshot.rs | 140 ++++++++++++++-------------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 624a5bd2eb..2bd7040770 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -539,12 +539,80 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + use bytes::Bytes; use chrono::{TimeZone, Utc}; + use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; + use crate::encryption::{EncryptionManager, StandardKeyMetadata}; + use crate::io::FileIO; use crate::spec::TableMetadata; + use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; use crate::spec::snapshot::_serde::SnapshotV1; use crate::spec::snapshot::{Operation, Snapshot, Summary}; + const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [{"type": "struct", "schema-id": 0, "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + "refs": {}, + "next-row-id": 0 + }"#; + + fn encryption_test_metadata() -> TableMetadata { + serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() + } + + fn encryption_test_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + fn encryption_test_manager() -> EncryptionManager { + EncryptionManager::builder() + .kms_client(encryption_test_kms()) + .table_key_id("master-1") + .build() + } + + async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { + let output = io.new_output(path).unwrap(); + let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); + writer.add_manifests(std::iter::empty()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap() + } + + fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { + Snapshot::builder() + .with_snapshot_id(1) + .with_sequence_number(0) + .with_timestamp_ms(0) + .with_manifest_list(manifest_list_path.to_string()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_encryption_key_id(key_id) + .build() + } + #[test] fn schema() { let record = r#" @@ -747,76 +815,6 @@ mod tests { assert_eq!(v2_snapshot.schema_id(), None); } - use bytes::Bytes; - - use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; - use crate::encryption::{EncryptionManager, StandardKeyMetadata}; - use crate::io::FileIO; - use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; - - const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ - "format-version": 3, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "memory:///table", - "last-sequence-number": 0, - "last-updated-ms": 1602638573590, - "last-column-id": 1, - "current-schema-id": 0, - "schemas": [{"type": "struct", "schema-id": 0, "fields": [ - {"id": 1, "name": "x", "required": true, "type": "long"} - ]}], - "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": []}], - "last-partition-id": 1000, - "default-sort-order-id": 0, - "sort-orders": [{"order-id": 0, "fields": []}], - "properties": {}, - "snapshots": [], - "snapshot-log": [], - "metadata-log": [], - "refs": {}, - "next-row-id": 0 - }"#; - - fn encryption_test_metadata() -> TableMetadata { - serde_json::from_str(ENCRYPTION_TEST_V3_METADATA).unwrap() - } - - fn encryption_test_kms() -> Arc { - let kms = MemoryKeyManagementClient::new(); - kms.add_master_key("master-1").unwrap(); - Arc::new(kms) - } - - fn encryption_test_manager() -> EncryptionManager { - EncryptionManager::builder() - .kms_client(encryption_test_kms()) - .table_key_id("master-1") - .build() - } - - async fn write_v3_manifest_list_bytes(io: &FileIO, path: &str) -> Bytes { - let output = io.new_output(path).unwrap(); - let mut writer = ManifestListWriter::v3(output, 1, None, 0, Some(0)); - writer.add_manifests(std::iter::empty()).unwrap(); - writer.close().await.unwrap(); - io.new_input(path).unwrap().read().await.unwrap() - } - - fn snapshot_pointing_at(manifest_list_path: &str, key_id: Option) -> Snapshot { - Snapshot::builder() - .with_snapshot_id(1) - .with_sequence_number(0) - .with_timestamp_ms(0) - .with_manifest_list(manifest_list_path.to_string()) - .with_summary(Summary { - operation: Operation::Append, - additional_properties: HashMap::new(), - }) - .with_encryption_key_id(key_id) - .build() - } - #[tokio::test] async fn load_manifest_list_errors_when_encrypted_but_no_manager_configured() { let io = FileIO::new_with_memory(); @@ -830,7 +828,7 @@ mod tests { .load_manifest_list(&io, &metadata, None) .await .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), crate::ErrorKind::PreconditionFailed); } #[tokio::test] From 1bb18cddc9d0e8c92fb715c96f5f7750ed1337f1 Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 22:33:15 +0100 Subject: [PATCH 12/17] fix tests --- crates/iceberg/src/table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 9cd1e842dc..a86cc8716f 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -669,7 +669,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); } #[tokio::test] @@ -690,7 +690,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); } #[tokio::test] From e2466bc590c5ec31729d632bbfe570d653c7c35f Mon Sep 17 00:00:00 2001 From: Xander Date: Wed, 20 May 2026 22:48:11 +0100 Subject: [PATCH 13/17] fix different test --- crates/iceberg/src/table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index a86cc8716f..b6707f6a2d 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -669,7 +669,7 @@ mod tests { .runtime(Runtime::try_current().unwrap()) .build() .unwrap_err(); - assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); } #[tokio::test] From bf6ac34407d9cf7dda3a269b8218a0dcd9de2acb Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 21 May 2026 10:09:47 +0100 Subject: [PATCH 14/17] Add manifest list loader --- crates/iceberg/src/catalog/utils.rs | 2 +- crates/iceberg/src/io/object_cache.rs | 4 +-- crates/iceberg/src/spec/manifest_list.rs | 29 +++++++++++++--- crates/iceberg/src/spec/snapshot.rs | 43 ++++++------------------ crates/iceberg/src/table.rs | 12 +++++-- crates/iceberg/src/transaction/append.rs | 5 --- 6 files changed, 48 insertions(+), 47 deletions(-) diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs index c668d1a6ec..22f491a9dd 100644 --- a/crates/iceberg/src/catalog/utils.rs +++ b/crates/iceberg/src/catalog/utils.rs @@ -49,7 +49,7 @@ pub async fn drop_table_data( // Load all manifest lists concurrently let results: Vec<_> = futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { - let manifest_list = ManifestListReader::new(snapshot, io, metadata) + let manifest_list = ManifestListReader::new(snapshot, io, metadata, encryption_manager) .load() .await?; Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index c6526f7532..cf06c97def 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -138,7 +138,7 @@ impl ObjectCache { table_metadata: &TableMetadataRef, ) -> Result> { if self.cache_disabled { - return ManifestListReader::new(snapshot, &self.file_io, table_metadata) + return ManifestListReader::new(snapshot, &self.file_io, table_metadata, self.encryption_manager.as_deref()) .load() .await .map(Arc::new); @@ -185,7 +185,7 @@ impl ObjectCache { snapshot: &SnapshotRef, table_metadata: &TableMetadataRef, ) -> Result { - let manifest_list = ManifestListReader::new(snapshot, &self.file_io, table_metadata) + let manifest_list = ManifestListReader::new(snapshot, &self.file_io, table_metadata, self.encryption_manager.as_deref()) .load() .await?; diff --git a/crates/iceberg/src/spec/manifest_list.rs b/crates/iceberg/src/spec/manifest_list.rs index 73d0ee607f..b3cf1d7422 100644 --- a/crates/iceberg/src/spec/manifest_list.rs +++ b/crates/iceberg/src/spec/manifest_list.rs @@ -29,6 +29,7 @@ use serde_derive::{Deserialize, Serialize}; use self::_const_schema::{MANIFEST_LIST_AVRO_SCHEMA_V1, MANIFEST_LIST_AVRO_SCHEMA_V2}; use self::_serde::{ManifestFileV1, ManifestFileV2}; use super::{FormatVersion, Manifest, Snapshot, TableMetadata}; +use crate::encryption::{EncryptedInputFile, EncryptionManager}; use crate::error::Result; use crate::io::{FileIO, OutputFile}; use crate::spec::manifest_list::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V3; @@ -96,6 +97,7 @@ pub struct ManifestListReader<'a> { snapshot: &'a Snapshot, file_io: &'a FileIO, table_metadata: &'a TableMetadata, + encryption_manager: Option<&'a EncryptionManager>, } impl<'a> ManifestListReader<'a> { @@ -103,21 +105,38 @@ impl<'a> ManifestListReader<'a> { snapshot: &'a Snapshot, file_io: &'a FileIO, table_metadata: &'a TableMetadata, + encryption_manager: Option<&'a EncryptionManager>, ) -> Self { Self { snapshot, file_io, table_metadata, + encryption_manager, } } /// Loads and returns the [`ManifestList`] for this snapshot. pub async fn load(&self) -> Result { - let manifest_list_content = self - .file_io - .new_input(self.snapshot.manifest_list())? - .read() - .await?; + let manifest_list_content = + match (self.snapshot.encryption_key_id(), self.encryption_manager) { + (Some(_), None) => { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "Snapshot has encryption_key_id but no EncryptionManager configured on Table", + )); + } + (Some(key_id), Some(em)) => { + let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; + let input = self.file_io.new_input(self.snapshot.manifest_list())?; + EncryptedInputFile::new(input, key_metadata).read().await? + } + (None, _) => { + self.file_io + .new_input(self.snapshot.manifest_list())? + .read() + .await? + } + }; ManifestList::parse_with_version( &manifest_list_content, self.table_metadata.format_version(), diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 2bd7040770..6945207644 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -26,10 +26,9 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::table_metadata::SnapshotLog; -use crate::encryption::{EncryptedInputFile, EncryptionManager}; use crate::error::{Result, timestamp_ms_to_utc}; use crate::io::FileIO; -use crate::spec::{ManifestList, SchemaId, SchemaRef, TableMetadata}; +use crate::spec::{ManifestList, ManifestListReader, SchemaId, SchemaRef, TableMetadata}; use crate::{Error, ErrorKind}; /// The ref name of the main branch of the table. @@ -201,28 +200,10 @@ impl Snapshot { &self, file_io: &FileIO, table_metadata: &TableMetadata, - encryption_manager: Option<&EncryptionManager>, ) -> Result { - let manifest_list_content = match (&self.encryption_key_id, encryption_manager) { - (Some(_), None) => { - return Err(Error::new( - ErrorKind::PreconditionFailed, - "Snapshot has encryption_key_id but no EncryptionManager configured on Table", - )); - } - (Some(key_id), Some(em)) => { - let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; - let input = file_io.new_input(&self.manifest_list)?; - EncryptedInputFile::new(input, key_metadata).read().await? - } - (None, _) => file_io.new_input(&self.manifest_list)?.read().await?, - }; - ManifestList::parse_with_version( - &manifest_list_content, - // TODO: You don't really need the version since you could just project any Avro in - // the version that you'd like to get (probably always the latest) - table_metadata.format_version(), - ) + ManifestListReader::new(self, file_io, table_metadata, None) + .load() + .await } #[allow(dead_code)] @@ -545,7 +526,7 @@ mod tests { use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; use crate::encryption::{EncryptionManager, StandardKeyMetadata}; use crate::io::FileIO; - use crate::spec::TableMetadata; + use crate::spec::{ManifestListReader, TableMetadata}; use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; use crate::spec::snapshot::_serde::SnapshotV1; use crate::spec::snapshot::{Operation, Snapshot, Summary}; @@ -824,10 +805,9 @@ mod tests { ); let metadata = encryption_test_metadata(); - let err = snapshot - .load_manifest_list(&io, &metadata, None) - .await - .unwrap_err(); + let err = ManifestListReader::new(&snapshot, &io, &metadata, None) + .load() + .await.unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::PreconditionFailed); } @@ -856,10 +836,9 @@ mod tests { let snapshot = snapshot_pointing_at(encrypted_path, Some(key_id)); let metadata = encryption_test_metadata(); - let manifest_list: ManifestList = snapshot - .load_manifest_list(&io, &metadata, Some(&mgr)) - .await - .unwrap(); + let manifest_list: ManifestList = ManifestListReader::new(&snapshot, &io, &metadata, Some(&mgr)) + .load() + .await.unwrap(); assert_eq!(manifest_list.entries().len(), 0); } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index b6707f6a2d..eb4248e9e6 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -27,7 +27,9 @@ use crate::io::FileIO; use crate::io::object_cache::ObjectCache; use crate::runtime::Runtime; use crate::scan::TableScanBuilder; -use crate::spec::{ManifestListReader, SchemaRef, Snapshot, TableMetadata, TableMetadataRef}; +use crate::spec::{ + FormatVersion, ManifestListReader, SchemaRef, Snapshot, TableMetadata, TableMetadataRef, +}; use crate::{Error, ErrorKind, Result, TableIdent}; /// Builder to create table scan. @@ -302,7 +304,12 @@ impl Table { /// Creates a [`ManifestListReader`] for the given snapshot. pub fn manifest_list_reader<'a>(&'a self, snapshot: &'a Snapshot) -> ManifestListReader<'a> { - ManifestListReader::new(snapshot, &self.file_io, &self.metadata) + ManifestListReader::new( + snapshot, + &self.file_io, + &self.metadata, + self.encryption_manager.as_deref(), + ) } /// Create a reader for the table. @@ -311,6 +318,7 @@ impl Table { } } + /// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog. /// It can only be used to read metadata and for table scan. /// # Examples diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 01e1d617a3..ec4ceee277 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -303,14 +303,9 @@ mod tests { } else { unreachable!() }; -<<<<<<< HEAD let manifest_list = table .manifest_list_reader(new_snapshot) .load() -======= - let manifest_list = new_snapshot - .load_manifest_list(table.file_io(), table.metadata(), None) ->>>>>>> d0cfbb7d (Read encrypted manifest lists) .await .unwrap(); assert_eq!(1, manifest_list.entries().len()); From 26267103cf2172fb4b1dc96c63a2e4c653b7aef6 Mon Sep 17 00:00:00 2001 From: Xander Date: Thu, 21 May 2026 10:28:35 +0100 Subject: [PATCH 15/17] fix lock --- Cargo.lock | 723 +++-------------------------------------------------- 1 file changed, 41 insertions(+), 682 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 370fbf8a31..19ac987caa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cipher", "cpufeatures 0.2.17", ] @@ -49,7 +49,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "const-random", "getrandom 0.3.4", "once_cell", @@ -206,15 +206,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "approx" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" -dependencies = [ - "num-traits", -] - [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -1047,7 +1038,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.4", + "cfg-if", "constant_time_eq", "cpufeatures 0.2.17", ] @@ -1126,17 +1117,6 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" -dependencies = [ - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -1218,12 +1198,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.4" @@ -1242,7 +1216,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.3.0", "rand_core 0.10.0", ] @@ -1442,37 +1416,12 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "const-str" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" - -[[package]] -name = "const_panic" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" -dependencies = [ - "typewit", -] - [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -1489,15 +1438,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "countio" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" -dependencies = [ - "futures-io", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1546,7 +1486,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -1720,7 +1660,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -2794,7 +2734,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "home", "windows-sys 0.48.0", ] @@ -2866,7 +2806,7 @@ version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "rustix", "windows-sys 0.59.0", ] @@ -3068,15 +3008,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gearhash" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" -dependencies = [ - "cfg-if 0.1.10", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -3093,10 +3024,10 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -3106,7 +3037,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", "r-efi 5.3.0", @@ -3120,14 +3051,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if 1.0.4", - "js-sys", + "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] @@ -3140,26 +3069,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "git-version" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" -dependencies = [ - "git-version-macro", -] - -[[package]] -name = "git-version-macro" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "glob" version = "0.3.3" @@ -3204,7 +3113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "bytemuck", - "cfg-if 1.0.4", + "cfg-if", "crunchy", "num-traits", "zerocopy", @@ -3259,12 +3168,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "heapify" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" - [[package]] name = "heck" version = "0.5.0" @@ -3277,28 +3180,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hf-xet" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" -dependencies = [ - "async-trait", - "bytes", - "http 1.4.0", - "more-asserts", - "serde", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "uuid", - "xet-client", - "xet-core-structures", - "xet-data", - "xet-runtime", -] - [[package]] name = "hive_metastore" version = "0.2.0" @@ -3469,11 +3350,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.5.10", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -3777,7 +3656,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "cfg-if 1.0.4", + "cfg-if", "futures", "iceberg", "iceberg_test_utils", @@ -4079,7 +3958,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "combine", "jni-macros", "jni-sys", @@ -4159,23 +4038,6 @@ dependencies = [ "simple_asn1", ] -[[package]] -name = "konst" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" -dependencies = [ - "const_panic", - "konst_proc_macros", - "typewit", -] - -[[package]] -name = "konst_proc_macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" - [[package]] name = "lazy_static" version = "1.5.0" @@ -4405,22 +4267,13 @@ dependencies = [ "serde", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "md-5" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "digest", ] @@ -4503,7 +4356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.61.2", ] @@ -4513,7 +4366,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "downcast", "fragile", "mockall_derive", @@ -4527,7 +4380,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "proc-macro2", "quote", "syn", @@ -4578,12 +4431,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "more-asserts" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" - [[package]] name = "motore" version = "0.4.1" @@ -4661,7 +4508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "cfg_aliases", "libc", "memoffset", @@ -4674,20 +4521,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "cfg_aliases", "libc", ] -[[package]] -name = "ntapi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" -dependencies = [ - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4791,34 +4629,6 @@ dependencies = [ "syn", ] -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags", -] - -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-system-configuration" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" -dependencies = [ - "objc2-core-foundation", -] - [[package]] name = "object" version = "0.37.3" @@ -4879,12 +4689,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oneshot" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" - [[package]] name = "opaque-debug" version = "0.3.1" @@ -4906,7 +4710,6 @@ dependencies = [ "opendal-service-azdls", "opendal-service-fs", "opendal-service-gcs", - "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", ] @@ -5046,23 +4849,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "opendal-service-hf" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2ab7a2a8a11dfe257ef4db5c0de798acbcd0d6429c37382dad2154bc06a388" -dependencies = [ - "bytes", - "hf-xet", - "http 1.4.0", - "log", - "opendal-core", - "percent-encoding", - "reqwest 0.13.3", - "serde", - "serde_json", -] - [[package]] name = "opendal-service-oss" version = "0.56.0" @@ -5143,15 +4929,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" -dependencies = [ - "memchr", -] - [[package]] name = "outref" version = "0.5.2" @@ -5186,7 +4963,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "libc", "redox_syscall 0.5.18", "smallvec", @@ -5448,7 +5225,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -5858,15 +5635,6 @@ dependencies = [ "syn", ] -[[package]] -name = "redb" -version = "3.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -6152,8 +5920,6 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", - "serde", - "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -6168,20 +5934,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "reqwest-middleware" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" -dependencies = [ - "anyhow", - "async-trait", - "http 1.4.0", - "reqwest 0.13.3", - "thiserror 2.0.18", - "tower-service", -] - [[package]] name = "ring" version = "0.17.14" @@ -6189,7 +5941,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if 1.0.4", + "cfg-if", "getrandom 0.2.17", "libc", "untrusted 0.9.0", @@ -6273,7 +6025,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "glob", "proc-macro-crate", "proc-macro2", @@ -6291,7 +6043,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "ordered-multimap", ] @@ -6369,7 +6121,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "jni", "log", @@ -6415,7 +6167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", - "cfg-if 1.0.4", + "cfg-if", "clipboard-win", "fd-lock", "home", @@ -6436,12 +6188,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "safe-transmute" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" - [[package]] name = "salsa20" version = "0.10.2" @@ -6541,7 +6287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -6755,7 +6501,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "digest", ] @@ -6766,19 +6512,9 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures 0.2.17", "digest", - "sha2-asm", -] - -[[package]] -name = "sha2-asm" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" -dependencies = [ - "cc", ] [[package]] @@ -6790,17 +6526,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shellexpand" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" -dependencies = [ - "bstr", - "dirs", - "os_str_bytes", -] - [[package]] name = "shlex" version = "1.3.0" @@ -6920,7 +6645,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5661364b38abad49cf1ade6631fcc35d2ccf882a7d68616b4228b7717feb5fba" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -6931,7 +6656,7 @@ checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" dependencies = [ "bumpalo", "bytes", - "cfg-if 1.0.4", + "cfg-if", "faststr", "itoa", "ref-cast", @@ -6949,7 +6674,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f944718c33623919878cf74b4c9361eb3024f635733922b26722b14cd3f8cc" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -7141,7 +6866,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami 1.6.1", + "whoami", ] [[package]] @@ -7178,7 +6903,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami 1.6.1", + "whoami", ] [[package]] @@ -7218,28 +6943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", - "cfg-if 1.0.4", + "cfg-if", "libc", "psm", "windows-sys 0.59.0", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "statrs" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" -dependencies = [ - "approx", - "num-traits", -] - [[package]] name = "stringprep" version = "0.1.5" @@ -7319,12 +7028,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "symlink" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" - [[package]] name = "syn" version = "2.0.117" @@ -7356,41 +7059,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sysinfo" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tagptr" version = "0.2.0" @@ -7462,7 +7130,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -7569,17 +7237,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-retry" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640" -dependencies = [ - "pin-project-lite", - "rand 0.10.1", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -7743,19 +7400,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" -dependencies = [ - "crossbeam-channel", - "symlink", - "thiserror 2.0.18", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.31" @@ -7788,35 +7432,18 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-serde" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" -dependencies = [ - "serde", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ - "matchers", "nu-ansi-term", - "once_cell", - "regex-automata", - "serde", - "serde_json", "sharded-slab", "smallvec", "thread_local", - "tracing", "tracing-core", "tracing-log", - "tracing-serde", ] [[package]] @@ -7887,12 +7514,6 @@ dependencies = [ "syn", ] -[[package]] -name = "typewit" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc19094686c694eb41b3b99dcc2f2975d4b078512fa22ae6c63f7ca318bdcff7" - [[package]] name = "typify" version = "0.5.0" @@ -8172,15 +7793,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -8205,22 +7817,13 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" -[[package]] -name = "wasite" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" -dependencies = [ - "wasi 0.14.7+wasi-0.2.4", -] - [[package]] name = "wasm-bindgen" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -8233,7 +7836,7 @@ version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "futures-util", "js-sys", "once_cell", @@ -8387,38 +7990,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite 0.1.0", -] - -[[package]] -name = "whoami" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" -dependencies = [ - "libc", - "libredox", - "objc2-system-configuration", - "wasite 1.0.2", - "web-sys", + "wasite", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -8428,33 +8002,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" -dependencies = [ - "windows-core", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -8468,17 +8015,6 @@ dependencies = [ "windows-strings", ] -[[package]] -name = "windows-future" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - [[package]] name = "windows-implement" version = "0.60.2" @@ -8507,27 +8043,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" -dependencies = [ - "windows-core", - "windows-link", -] - -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -8639,15 +8154,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows-threading" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" -dependencies = [ - "windows-link", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8908,153 +8414,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "xet-client" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "bytes", - "clap", - "crc32fast", - "futures", - "http 1.4.0", - "hyper", - "lazy_static", - "more-asserts", - "rand 0.10.1", - "redb", - "reqwest 0.13.3", - "reqwest-middleware", - "serde", - "serde_json", - "serde_repr", - "statrs", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-retry", - "tracing", - "tracing-subscriber", - "url", - "urlencoding", - "web-time", - "xet-core-structures", - "xet-runtime", -] - -[[package]] -name = "xet-core-structures" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" -dependencies = [ - "async-trait", - "base64", - "blake3", - "bytemuck", - "bytes", - "clap", - "countio", - "csv", - "futures", - "futures-util", - "getrandom 0.4.2", - "heapify", - "itertools 0.14.0", - "lazy_static", - "lz4_flex", - "more-asserts", - "rand 0.10.1", - "regex", - "safe-transmute", - "serde", - "static_assertions", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "uuid", - "web-time", - "xet-runtime", -] - -[[package]] -name = "xet-data" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "chrono", - "clap", - "gearhash", - "http 1.4.0", - "itertools 0.14.0", - "lazy_static", - "more-asserts", - "rand 0.10.1", - "serde", - "serde_json", - "sha2", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "url", - "uuid", - "walkdir", - "xet-client", - "xet-core-structures", - "xet-runtime", -] - -[[package]] -name = "xet-runtime" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "chrono", - "colored", - "const-str", - "ctor", - "dirs", - "futures", - "git-version", - "humantime", - "konst", - "lazy_static", - "libc", - "more-asserts", - "oneshot", - "pin-project", - "rand 0.10.1", - "reqwest 0.13.3", - "serde", - "serde_json", - "shellexpand", - "sysinfo", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "tracing-appender", - "tracing-subscriber", - "whoami 2.1.1", - "winapi", -] - [[package]] name = "xmlparser" version = "0.13.6" From 67c94f691ba976ec3c4dfa706bd8ecc7ac0396ea Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 30 May 2026 13:34:20 +0100 Subject: [PATCH 16/17] revert cargo change --- Cargo.lock | 1232 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 1094 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ac987caa..774f9c86c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,54 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "abi_stable" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" +dependencies = [ + "abi_stable_derive", + "abi_stable_shared", + "const_panic", + "core_extensions", + "crossbeam-channel", + "generational-arena", + "libloading", + "lock_api", + "parking_lot", + "paste", + "repr_offset", + "rustc_version", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "abi_stable_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" +dependencies = [ + "abi_stable_shared", + "as_derive_utils", + "core_extensions", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "typed-arena", +] + +[[package]] +name = "abi_stable_shared" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" +dependencies = [ + "core_extensions", +] + [[package]] name = "adler2" version = "2.0.1" @@ -24,7 +72,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] @@ -49,7 +97,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "const-random", "getrandom 0.3.4", "once_cell", @@ -206,6 +254,15 @@ dependencies = [ "zstd", ] +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -248,6 +305,7 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", + "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", @@ -270,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash", "arrow-buffer", @@ -289,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -338,9 +396,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -391,9 +449,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,6 +460,18 @@ dependencies = [ "arrow-select", ] +[[package]] +name = "arrow-pyarrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + [[package]] name = "arrow-row" version = "58.1.0" @@ -417,19 +487,20 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ + "bitflags", "serde_core", "serde_json", ] [[package]] name = "arrow-select" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", "arrow-array", @@ -441,9 +512,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ "arrow-array", "arrow-buffer", @@ -462,6 +533,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" +[[package]] +name = "as_derive_utils" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" +dependencies = [ + "core_extensions", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -496,6 +579,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" +dependencies = [ + "abi_stable", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -515,7 +607,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -526,7 +618,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1038,7 +1130,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.4", "constant_time_eq", "cpufeatures 0.2.17", ] @@ -1087,13 +1179,13 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.117", ] [[package]] @@ -1117,6 +1209,26 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1140,7 +1252,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1198,6 +1310,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1216,7 +1334,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "rand_core 0.10.0", ] @@ -1286,7 +1404,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1325,7 +1443,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1416,12 +1534,37 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1438,6 +1581,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_extensions" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" +dependencies = [ + "core_extensions_proc_macros", +] + +[[package]] +name = "core_extensions_proc_macros" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" + +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1486,7 +1653,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -1616,7 +1783,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -1629,7 +1796,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -1640,7 +1807,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1651,7 +1818,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1660,7 +1827,7 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -2060,6 +2227,36 @@ dependencies = [ "paste", ] +[[package]] +name = "datafusion-ffi" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c" +dependencies = [ + "abi_stable", + "arrow", + "arrow-schema", + "async-ffi", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto", + "datafusion-proto-common", + "datafusion-session", + "futures", + "log", + "prost", + "semver", + "tokio", +] + [[package]] name = "datafusion-functions" version = "53.1.0" @@ -2204,7 +2401,7 @@ checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2334,6 +2531,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +dependencies = [ + "arrow", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "object_store", + "prost", + "rand 0.9.4", +] + +[[package]] +name = "datafusion-proto-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "53.1.0" @@ -2496,7 +2732,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2506,7 +2742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.117", ] [[package]] @@ -2545,7 +2781,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2556,7 +2792,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2622,7 +2858,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2663,7 +2899,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2713,7 +2949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2734,7 +2970,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "home", "windows-sys 0.48.0", ] @@ -2806,7 +3042,7 @@ version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "rustix", "windows-sys 0.59.0", ] @@ -2970,7 +3206,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3008,6 +3244,24 @@ dependencies = [ "slab", ] +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "generational-arena" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" +dependencies = [ + "cfg-if 1.0.4", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3024,10 +3278,10 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -3037,7 +3291,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi 5.3.0", @@ -3051,12 +3305,14 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -3069,6 +3325,26 @@ dependencies = [ "polyval", ] +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "glob" version = "0.3.3" @@ -3113,7 +3389,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "bytemuck", - "cfg-if", + "cfg-if 1.0.4", "crunchy", "num-traits", "zerocopy", @@ -3168,6 +3444,12 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + [[package]] name = "heck" version = "0.5.0" @@ -3180,6 +3462,28 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-xet" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "more-asserts", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + [[package]] name = "hive_metastore" version = "0.2.0" @@ -3349,10 +3653,12 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -3432,6 +3738,7 @@ dependencies = [ "strum", "tempfile", "tokio", + "tracing", "typed-builder", "typetag", "url", @@ -3656,7 +3963,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "cfg-if", + "cfg-if 1.0.4", "futures", "iceberg", "iceberg_test_utils", @@ -3923,7 +4230,7 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3934,7 +4241,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3958,7 +4265,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "combine", "jni-macros", "jni-sys", @@ -3979,7 +4286,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -3998,7 +4305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4038,6 +4345,23 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + [[package]] name = "lazy_static" version = "1.5.0" @@ -4122,6 +4446,16 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if 1.0.4", + "winapi", +] + [[package]] name = "liblzma" version = "0.4.6" @@ -4268,13 +4602,22 @@ dependencies = [ ] [[package]] -name = "md-5" -version = "0.10.6" +name = "matchers" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "cfg-if", - "digest", + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if 1.0.4", + "digest", ] [[package]] @@ -4356,7 +4699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -4366,7 +4709,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "downcast", "fragile", "mockall_derive", @@ -4380,10 +4723,10 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4431,6 +4774,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "motore" version = "0.4.1" @@ -4451,7 +4800,7 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4477,7 +4826,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4508,7 +4857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "cfg_aliases", "libc", "memoffset", @@ -4521,18 +4870,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "cfg_aliases", "libc", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4626,7 +4984,35 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", ] [[package]] @@ -4689,6 +5075,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -4710,6 +5102,7 @@ dependencies = [ "opendal-service-azdls", "opendal-service-fs", "opendal-service-gcs", + "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", ] @@ -4849,6 +5242,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-hf" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2ab7a2a8a11dfe257ef4db5c0de798acbcd0d6429c37382dad2154bc06a388" +dependencies = [ + "bytes", + "hf-xet", + "http 1.4.0", + "log", + "opendal-core", + "percent-encoding", + "reqwest 0.13.3", + "serde", + "serde_json", +] + [[package]] name = "opendal-service-oss" version = "0.56.0" @@ -4929,6 +5339,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + [[package]] name = "outref" version = "0.5.2" @@ -4963,7 +5382,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "redox_syscall 0.5.18", "smallvec", @@ -5154,7 +5573,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5225,7 +5644,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -5313,7 +5732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -5359,7 +5778,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.117", "tempfile", ] @@ -5373,7 +5792,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5412,7 +5831,78 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "pyiceberg_core_rust" +version = "0.9.0" +dependencies = [ + "arrow", + "datafusion-ffi", + "iceberg", + "iceberg-datafusion", + "iceberg-storage-opendal", + "pyo3", + "tokio", +] + +[[package]] +name = "pyo3" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.117", ] [[package]] @@ -5454,7 +5944,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -5492,9 +5982,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5632,7 +6122,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", ] [[package]] @@ -5681,7 +6180,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5741,6 +6240,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" +[[package]] +name = "repr_offset" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" +dependencies = [ + "tstr", +] + [[package]] name = "reqsign-aliyun-oss" version = "3.0.0" @@ -5920,6 +6428,8 @@ dependencies = [ "rustls", "rustls-pki-types", "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -5934,6 +6444,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http 1.4.0", + "reqwest 0.13.3", + "thiserror 2.0.18", + "tower-service", +] + [[package]] name = "ring" version = "0.17.14" @@ -5941,7 +6465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.17", "libc", "untrusted 0.9.0", @@ -5974,7 +6498,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6025,7 +6549,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "glob", "proc-macro-crate", "proc-macro2", @@ -6033,7 +6557,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn", + "syn 2.0.117", "unicode-ident", ] @@ -6043,7 +6567,7 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "ordered-multimap", ] @@ -6075,7 +6599,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6121,7 +6645,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -6133,7 +6657,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6167,7 +6691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.4", "clipboard-win", "fd-lock", "home", @@ -6188,6 +6712,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + [[package]] name = "salsa20" version = "0.10.2" @@ -6260,7 +6790,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -6287,7 +6817,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6380,7 +6910,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6391,7 +6921,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6415,7 +6945,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6436,7 +6966,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.117", ] [[package]] @@ -6453,11 +6983,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.19.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ "base64", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -6472,14 +7003,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.19.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6501,7 +7032,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest", ] @@ -6512,9 +7043,19 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest", + "sha2-asm", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] [[package]] @@ -6526,6 +7067,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + [[package]] name = "shlex" version = "1.3.0" @@ -6645,7 +7197,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5661364b38abad49cf1ade6631fcc35d2ccf882a7d68616b4228b7717feb5fba" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6656,7 +7208,7 @@ checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" dependencies = [ "bumpalo", "bytes", - "cfg-if", + "cfg-if 1.0.4", "faststr", "itoa", "ref-cast", @@ -6674,7 +7226,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f944718c33623919878cf74b4c9361eb3024f635733922b26722b14cd3f8cc" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6740,7 +7292,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6802,7 +7354,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.117", ] [[package]] @@ -6823,7 +7375,7 @@ dependencies = [ "sha2", "sqlx-core", "sqlx-sqlite", - "syn", + "syn 2.0.117", "tokio", "url", ] @@ -6866,7 +7418,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -6903,7 +7455,7 @@ dependencies = [ "stringprep", "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -6943,12 +7495,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "libc", "psm", "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -6984,7 +7552,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7017,7 +7585,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.117", "typify", "walkdir", ] @@ -7028,6 +7596,23 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -7056,7 +7641,42 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -7065,6 +7685,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + [[package]] name = "tempfile" version = "3.27.0" @@ -7075,7 +7701,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7110,7 +7736,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7121,7 +7747,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7130,7 +7756,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -7234,7 +7860,18 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "tokio-retry" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640" +dependencies = [ + "pin-project-lite", + "rand 0.10.1", + "tokio", ] [[package]] @@ -7400,6 +8037,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -7408,7 +8058,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7432,18 +8082,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -7452,12 +8119,33 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tstr" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" +dependencies = [ + "tstr_proc_macros", +] + +[[package]] +name = "tstr_proc_macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" + [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typed-builder" version = "0.20.1" @@ -7475,7 +8163,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7511,9 +8199,15 @@ checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" + [[package]] name = "typify" version = "0.5.0" @@ -7539,7 +8233,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.117", "thiserror 2.0.18", "unicode-ident", ] @@ -7557,7 +8251,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.117", "typify-impl", ] @@ -7793,6 +8487,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -7817,13 +8520,22 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -7836,7 +8548,7 @@ version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "futures-util", "js-sys", "once_cell", @@ -7863,7 +8575,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -7990,16 +8702,72 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", +] + +[[package]] +name = "whoami" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite 1.0.2", + "web-sys", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", ] [[package]] @@ -8015,6 +8783,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -8023,7 +8802,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -8034,7 +8813,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -8043,6 +8822,27 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -8154,6 +8954,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -8340,7 +9149,7 @@ dependencies = [ "heck", "indexmap 2.13.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -8356,7 +9165,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -8414,6 +9223,153 @@ dependencies = [ "rustix", ] +[[package]] +name = "xet-client" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "clap", + "crc32fast", + "futures", + "http 1.4.0", + "hyper", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "redb", + "reqwest 0.13.3", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +dependencies = [ + "async-trait", + "base64", + "blake3", + "bytemuck", + "bytes", + "clap", + "countio", + "csv", + "futures", + "futures-util", + "getrandom 0.4.2", + "heapify", + "itertools 0.14.0", + "lazy_static", + "lz4_flex", + "more-asserts", + "rand 0.10.1", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "clap", + "gearhash", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "more-asserts", + "rand 0.10.1", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "url", + "uuid", + "walkdir", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor", + "dirs", + "futures", + "git-version", + "humantime", + "konst", + "lazy_static", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.1", + "reqwest 0.13.3", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-appender", + "tracing-subscriber", + "whoami 2.1.1", + "winapi", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -8445,7 +9401,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -8466,7 +9422,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -8486,7 +9442,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -8526,7 +9482,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] From 8710d4f8d18f0531c22d3d27f689c03949da297d Mon Sep 17 00:00:00 2001 From: Xander Date: Sat, 30 May 2026 13:45:33 +0100 Subject: [PATCH 17/17] comments --- crates/iceberg/src/io/object_cache.rs | 24 ++++++++++---- crates/iceberg/src/spec/manifest_list.rs | 42 +++++++++++++----------- crates/iceberg/src/spec/snapshot.rs | 13 +++++--- crates/iceberg/src/table.rs | 39 ++++++++++------------ 4 files changed, 65 insertions(+), 53 deletions(-) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index cf06c97def..7b028b7470 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -138,10 +138,15 @@ impl ObjectCache { table_metadata: &TableMetadataRef, ) -> Result> { if self.cache_disabled { - return ManifestListReader::new(snapshot, &self.file_io, table_metadata, self.encryption_manager.as_deref()) - .load() - .await - .map(Arc::new); + return ManifestListReader::new( + snapshot, + &self.file_io, + table_metadata, + self.encryption_manager.as_deref(), + ) + .load() + .await + .map(Arc::new); } let key = CachedObjectKey::ManifestList(( @@ -185,9 +190,14 @@ impl ObjectCache { snapshot: &SnapshotRef, table_metadata: &TableMetadataRef, ) -> Result { - let manifest_list = ManifestListReader::new(snapshot, &self.file_io, table_metadata, self.encryption_manager.as_deref()) - .load() - .await?; + let manifest_list = ManifestListReader::new( + snapshot, + &self.file_io, + table_metadata, + self.encryption_manager.as_deref(), + ) + .load() + .await?; Ok(CachedItem::ManifestList(Arc::new(manifest_list))) } diff --git a/crates/iceberg/src/spec/manifest_list.rs b/crates/iceberg/src/spec/manifest_list.rs index b3cf1d7422..15dbda2b99 100644 --- a/crates/iceberg/src/spec/manifest_list.rs +++ b/crates/iceberg/src/spec/manifest_list.rs @@ -117,26 +117,28 @@ impl<'a> ManifestListReader<'a> { /// Loads and returns the [`ManifestList`] for this snapshot. pub async fn load(&self) -> Result { - let manifest_list_content = - match (self.snapshot.encryption_key_id(), self.encryption_manager) { - (Some(_), None) => { - return Err(Error::new( - ErrorKind::PreconditionFailed, - "Snapshot has encryption_key_id but no EncryptionManager configured on Table", - )); - } - (Some(key_id), Some(em)) => { - let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; - let input = self.file_io.new_input(self.snapshot.manifest_list())?; - EncryptedInputFile::new(input, key_metadata).read().await? - } - (None, _) => { - self.file_io - .new_input(self.snapshot.manifest_list())? - .read() - .await? - } - }; + let manifest_list_content = match ( + self.snapshot.encryption_key_id(), + self.encryption_manager, + ) { + (Some(_), None) => { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "Snapshot has encryption_key_id but no EncryptionManager configured on Table", + )); + } + (Some(key_id), Some(em)) => { + let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; + let input = self.file_io.new_input(self.snapshot.manifest_list())?; + EncryptedInputFile::new(input, key_metadata).read().await? + } + (None, _) => { + self.file_io + .new_input(self.snapshot.manifest_list())? + .read() + .await? + } + }; ManifestList::parse_with_version( &manifest_list_content, self.table_metadata.format_version(), diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 6945207644..87adf7c3c5 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -526,10 +526,10 @@ mod tests { use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; use crate::encryption::{EncryptionManager, StandardKeyMetadata}; use crate::io::FileIO; - use crate::spec::{ManifestListReader, TableMetadata}; use crate::spec::manifest_list::{ManifestList, ManifestListWriter}; use crate::spec::snapshot::_serde::SnapshotV1; use crate::spec::snapshot::{Operation, Snapshot, Summary}; + use crate::spec::{ManifestListReader, TableMetadata}; const ENCRYPTION_TEST_V3_METADATA: &str = r#"{ "format-version": 3, @@ -807,7 +807,8 @@ mod tests { let err = ManifestListReader::new(&snapshot, &io, &metadata, None) .load() - .await.unwrap_err(); + .await + .unwrap_err(); assert_eq!(err.kind(), crate::ErrorKind::PreconditionFailed); } @@ -836,9 +837,11 @@ mod tests { let snapshot = snapshot_pointing_at(encrypted_path, Some(key_id)); let metadata = encryption_test_metadata(); - let manifest_list: ManifestList = ManifestListReader::new(&snapshot, &io, &metadata, Some(&mgr)) - .load() - .await.unwrap(); + let manifest_list: ManifestList = + ManifestListReader::new(&snapshot, &io, &metadata, Some(&mgr)) + .load() + .await + .unwrap(); assert_eq!(manifest_list.entries().len(), 0); } } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index eb4248e9e6..efd7df3234 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -318,7 +318,6 @@ impl Table { } } - /// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog. /// It can only be used to read metadata and for table scan. /// # Examples @@ -408,29 +407,27 @@ impl StaticTable { /// If the table metadata sets the `encryption.key-id` property, build an /// [`EncryptionManager`] for the table. /// -/// Returns `Ok(None)` if the property is not set. Returns an error if the -/// property is set but no [`KeyManagementClient`] was provided. +/// Returns `Ok(None)` if the format version is below v3 or the property is +/// not set. Returns an error if the property is set but no +/// [`KeyManagementClient`] was provided. fn maybe_configure_encryption( kms_client: Option<&Arc>, metadata: &TableMetadataRef, ) -> Result>> { + if metadata.format_version() < FormatVersion::V3 { + return Ok(None); + } + let table_properties = metadata.table_properties()?; let Some(table_key_id) = table_properties.encryption_key_id else { + if kms_client.is_some() { + tracing::warn!( + "KeyManagementClient provided but table does not have encryption.key-id set" + ); + } return Ok(None); }; - // Encryption is a v3 feature: `encryption-keys` table metadata and the - // snapshot `key-id` field are introduced in format version 3. - if metadata.format_version() < FormatVersion::V3 { - return Err(Error::new( - ErrorKind::PreconditionFailed, - format!( - "Table encryption requires format version 3, found {}", - metadata.format_version() - ), - )); - } - let kms_client = kms_client.ok_or_else(|| { Error::new( ErrorKind::PreconditionFailed, @@ -681,24 +678,24 @@ mod tests { } #[tokio::test] - async fn table_builder_errors_when_encryption_set_on_pre_v3_table() { - // Encryption is a v3 spec feature; setting encryption.key-id on a v2 table - // must be rejected even when a KMS client is available. + async fn table_builder_skips_encryption_on_pre_v3_table() { + // Encryption is a v3 spec feature; pre-v3 tables silently skip + // encryption even if encryption.key-id is set. let mut metadata: TableMetadata = serde_json::from_str(V2_METADATA).unwrap(); metadata.properties.insert( TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), "master-1".to_string(), ); - let err = Table::builder() + let table = Table::builder() .file_io(FileIO::new_with_memory()) .metadata(metadata) .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) .kms_client(make_kms()) .runtime(Runtime::try_current().unwrap()) .build() - .unwrap_err(); - assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + .unwrap(); + assert!(table.encryption_manager().is_none()); } #[tokio::test]