diff --git a/docs/specs/file-format.md b/docs/specs/file-format.md index b9ce8da2eed..4a977365cb8 100644 --- a/docs/specs/file-format.md +++ b/docs/specs/file-format.md @@ -54,6 +54,13 @@ The postscript contains the locations of: 2. a `layout` segment containing the root `Layout` 3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning) 4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes +5. up to 16 user-defined `metadata` segments, each identified by a unique, non-empty UTF-8 key of at most 64 bytes + +The postscript carries a locator (offset, length, and alignment) for each metadata segment that is +present; a file written without user metadata (and any file predating this feature) carries none. +Readers do not load the opaque metadata values by default. Opt-in metadata reads resolve each +locator separately, allowing values outside the initial file-tail read to be fetched without reading +the intervening file contents. :::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs :start-after: [postscript] diff --git a/vortex-buffer/src/alignment.rs b/vortex-buffer/src/alignment.rs index 691d10eeb45..0e32c1f31b7 100644 --- a/vortex-buffer/src/alignment.rs +++ b/vortex-buffer/src/alignment.rs @@ -117,9 +117,14 @@ impl Alignment { /// /// ## Panics /// - /// Panics if `1 << exponent` overflows `usize`. + /// Panics if `1 << exponent` overflows `usize`. Use [`Self::try_from_exponent`] when parsing + /// untrusted input. #[inline] pub const fn from_exponent(exponent: u8) -> Self { + assert!( + (exponent as u32) < usize::BITS, + "Alignment exponent must fit in usize" + ); Self::new(1 << exponent) } diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs index 2fc27039c06..739d7c9fd8b 100644 --- a/vortex-file/src/file.rs +++ b/vortex-file/src/file.rs @@ -15,6 +15,7 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_layout::LayoutReader; use vortex_layout::scan::layout::LayoutReaderDataSource; @@ -23,6 +24,7 @@ use vortex_layout::scan::split_by::SplitBy; use vortex_layout::segments::SegmentSource; use vortex_scan::DataSourceRef; use vortex_session::VortexSession; +use vortex_utils::aliases::hash_map::HashMap; use crate::FileStatistics; use crate::footer::Footer; @@ -42,6 +44,8 @@ pub struct VortexFile { segment_source: Arc, /// The Vortex session used to open this file. session: VortexSession, + /// User-defined metadata values resolved for this file open. + metadata: Arc>, /// None id LayoutReader caching is turned off layout_reader_cache: Option>>, } @@ -78,10 +82,16 @@ impl VortexFile { footer, segment_source, session, + metadata: Arc::new(HashMap::new()), layout_reader_cache: None, } } + pub(crate) fn with_metadata(mut self, metadata: Arc>) -> Self { + self.metadata = metadata; + self + } + /// Enable layout reader caching. /// /// Repeated calls to [`layout_reader`](Self::layout_reader), [`scan`](Self::scan), and @@ -91,6 +101,7 @@ impl VortexFile { footer: self.footer, segment_source: self.segment_source, session: self.session, + metadata: self.metadata, layout_reader_cache: Some(OnceLock::new()), } } @@ -117,6 +128,22 @@ impl VortexFile { self.footer.statistics() } + /// Returns the user-defined metadata segments loaded for this file. + /// + /// Metadata is only loaded when requested during open. Iteration order is unspecified. + pub fn metadata_segments(&self) -> impl Iterator { + self.metadata + .iter() + .map(|(key, metadata)| (key.as_str(), metadata)) + } + + /// Returns the loaded user-defined metadata segment for the given key. + /// + /// Returns `None` when the key is absent or metadata was not loaded. + pub fn metadata_segment(&self, key: &str) -> Option<&ByteBuffer> { + self.metadata.get(key) + } + /// Create a new segment source for reading from the file. /// /// This may spawn a background I/O driver that will exit when the returned segment source diff --git a/vortex-file/src/footer/deserializer.rs b/vortex-file/src/footer/deserializer.rs index 9fb29db12fd..992070fb0a6 100644 --- a/vortex-file/src/footer/deserializer.rs +++ b/vortex-file/src/footer/deserializer.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::Arc; + use flatbuffers::root; use vortex_array::dtype::DType; use vortex_buffer::ByteBuffer; @@ -18,6 +20,7 @@ use crate::Footer; use crate::MAGIC_BYTES; use crate::VERSION; use crate::footer::FileStatistics; +use crate::footer::SegmentSpec; use crate::footer::postscript::Postscript; use crate::footer::postscript::PostscriptSegment; @@ -177,14 +180,50 @@ impl FooterDeserializer { ) }) .transpose()?; + let metadata = postscript + .metadata + .iter() + .map(|metadata| { + let segment = SegmentSpec { + offset: metadata.segment.offset, + length: metadata.segment.length, + alignment: metadata.segment.alignment, + }; + let end = segment + .offset + .checked_add(u64::from(segment.length)) + .ok_or_else(|| { + vortex_err!("Metadata segment {} range overflowed u64", metadata.key) + })?; + if end > file_size { + vortex_bail!( + "Metadata segment {} range {}..{} exceeds file size {}", + metadata.key, + segment.offset, + end, + file_size + ); + } + let offset = usize::try_from(segment.offset)?; + if !segment.alignment.is_offset_aligned(offset) { + vortex_bail!( + "Metadata segment {} offset {} is not aligned to {}", + metadata.key, + segment.offset, + segment.alignment + ); + } + Ok((metadata.key.clone(), segment)) + }) + .collect::>>()?; Ok(DeserializeStep::Done(self.parse_footer( initial_offset, &self.buffer, - &postscript.footer, - &postscript.layout, + postscript, dtype, file_stats, + metadata, )?)) } @@ -269,19 +308,29 @@ impl FooterDeserializer { &self, initial_offset: u64, initial_read: &[u8], - footer_segment: &PostscriptSegment, - layout_segment: &PostscriptSegment, + postscript: &Postscript, dtype: DType, file_stats: Option, + metadata: Arc<[(String, SegmentSpec)]>, ) -> VortexResult