diff --git a/astro.config.mjs b/astro.config.mjs index a826d986..bd7ee5f3 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -213,6 +213,17 @@ export default defineConfig({ items: [ { slug: "developers/stone/v1" }, { slug: "developers/stone/v1/header" }, + { slug: "developers/stone/v1/payload-subheader" }, + { + label: "Records", + items: [ + { slug: "developers/stone/v1/record/attribute" }, + { slug: "developers/stone/v1/record/content" }, + { slug: "developers/stone/v1/record/index/index" }, + { slug: "developers/stone/v1/record/layout" }, + { slug: "developers/stone/v1/record/meta" }, + ], + }, ], }, ], diff --git a/src/content/docs/Developers/Stone/V1/Record/attribute.mdx b/src/content/docs/Developers/Stone/V1/Record/attribute.mdx new file mode 100644 index 00000000..c895c509 --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/Record/attribute.mdx @@ -0,0 +1,16 @@ +--- +title: Attribute +lastUpdated: 2026-04-22T00:00:00Z +description: The format of the Attribute record +--- + +The Attribute record is a general-purpose key-value pair. The format of the key and the value is undetermined: +applications will use them according to the context they operate in. The Attribute record has no fixed length, +but it is at least 16 bytes long. + +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| key_size | uint | 8 | Size, in bytes, of the `key` field. | +| value_size | uint | 8 | Size, in bytes, of the `value` field. +| key | blob | Variable | General-purpose key. | +| value | blob | Variable | General-purpose value. | diff --git a/src/content/docs/Developers/Stone/V1/Record/content.mdx b/src/content/docs/Developers/Stone/V1/Record/content.mdx new file mode 100644 index 00000000..c292dae1 --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/Record/content.mdx @@ -0,0 +1,8 @@ +--- +title: Content +lastUpdated: 2026-04-22T00:00:00Z +description: The format of the Content record +--- + +The Content record contains a blob. Its size is indicated in the Payload sub-header, as this kind +of record spans the entire payload. There must be exactly one Content record inside a payload. diff --git a/src/content/docs/Developers/Stone/V1/Record/index.mdx b/src/content/docs/Developers/Stone/V1/Record/index.mdx new file mode 100644 index 00000000..116adf1f --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/Record/index.mdx @@ -0,0 +1,63 @@ +--- +title: Index +lastUpdated: 2026-04-22T00:00:00Z +description: The format of the Index record +slug: developers/stone/v1/record/index/index +--- + +The Index record isolates a region of a Content record. This is especially useful +when the Content's blob is composed of heterogenous data that must be managed separately. + +The Index record does not specify which Content record it refers to; applications +may use custom logic to disambiguate the correct record (e.g., a *stone* may be encoded +with Index records followed by the Content they refer to). The trivial case is a *stone* +with only one Content record. + +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| start | uint | 8 | The offset in the Content blob where this region begins. This is a zero-based index: the first byte is addressed with 0. | +| end | uint | 8 | The offset in the Content blob where this region ends. This last byte is included in the range. This is a zero-based index: the first byte is addressed with 0. | +| hash | blob | 16 | Checksum of the content of the region, computed using XXH3_128bits. | + +Illustrated below is an example of two Index records addressing the same Content. + +```mermaid +block + columns 3 + +block +columns 1 + block:Index1:1 + columns 1 + start1["start = 1"] + end1["end = 3"] + cap1["hash"] + end +space + block:Index2:1 +columns 1 + start2["start = 1"] + end2["end = 6"] + cap2["hash"] + end + end + +space + + block:Content + columns 1 + c0["0x41"] + c1["0x65"] + c2["0x72"] + c3["0x79"] + c4["0x6E"] + c5["0x4F"] + c6["0x53"] + end + + start1 --> c1 + end1 --> c3 + +start2 --> c1 +end2 --> c6 +``` diff --git a/src/content/docs/Developers/Stone/V1/Record/layout.mdx b/src/content/docs/Developers/Stone/V1/Record/layout.mdx new file mode 100644 index 00000000..f9cdc670 --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/Record/layout.mdx @@ -0,0 +1,89 @@ +--- +title: Layout +lastUpdated: 2026-04-22T00:00:00Z +description: The format of the Layout record +--- + +The Layout record contains metadata of a file or directory that should be written to the mass memory. +When combined with a Content record (and possibly an Index record), it is possible to write the file +or directory on disk in a reproducible way. + +It is composed of a 32 bytes long header-like section and two fields of variable length. + +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| uid | uint | 4 | User ID of the owner of the file. | +| gid | uint | 4 | Group ID of the owner of the file. | +| mode | uint | 4 | File's mode and permission bits. | +| tag | blob | 4 | Unused. | +| source_length | uint | 2 | Length of the `source` blob. | +| target_length | uint | 2 | Length of the `target` string. | +| file_type | uint | 1 | Type of file to be written. | +| padding | blob | 11 | Unused. | +| source | blob | Specified by `source_length` | "source" of the file. Its meaning varies depending on `file_type`. | +| target | str | Specified by `target_length` | Path where the file should be written to. | + +### file_type + +`file_type` is an enum that specifies the type of file that should be written on disk. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueNameDescriptionMeaning of source
1RegularRegular file of data. + XXH3_128bits hash of the file. +
The source is a hash because our package manager, Moss, uses a + CAS to store files. + This hash may be also used to match an Index record, in order to identify a region of a Content + record to read. In fact, the Index record also contains a XXH3_128bits-based hash. +
2SymlinkSymbolic link to another file.String. Original file path to link somewhere else.
3DirectoryEmpty directory.Unused.
4CharacterDeviceCharacter device.Unused.
5BlockDeviceBlock device.Unused.
6FIFOPOSIX FIFO.Unused.
7SocketPOSIX communication socket.Unused.
diff --git a/src/content/docs/Developers/Stone/V1/Record/meta.mdx b/src/content/docs/Developers/Stone/V1/Record/meta.mdx new file mode 100644 index 00000000..338d8dee --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/Record/meta.mdx @@ -0,0 +1,92 @@ +--- +title: Meta +lastUpdated: 2026-04-22T00:00:00Z +description: The format of the Meta record +--- + +The Meta record carries information about the *stone* +itself or other binary packages, hence the name "meta". +It is composed of a 8 bytes long header-like section, plus a value of variable length. +The information it carries is strongly typed. + +Some of its fields can only be found in Repository *stones*, since they are information about packages to download. + +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| length | uint | 4 | Length, in bytes, of the value that follows the header-like section. | +| tag | uint | 2 | What information this record carries about the *stone*. | +| kind | uint | 1 | The base type of the information, for proper encoding and decoding. | +| padding | blob | 1 | Unused. | +| value | blob | `length` | Value of the information. The encoding depends on the value of `kind`. | + +### tag + +`tag` is an enum. + +|Value|Name|Description| +|---|---|---| +| 1 | Name | Name of the *stone*, which is typically the name of a package to install or remove. | +| 2 | Architecture | Target hardware architecture of the package. | +| 3 | Version | Version of the package. | +| 4 | Summary | Succinct description of a package. | +| 5 | Description | Full length description of the package. | +| 6 | Homepage | Web homepage of the project being packaged. | +| 7 | SourceID | ID of the source package, used for grouping. | +| 8 | Depends | One dependency of the package. | +| 9 | Provides | One capability, or the name, of the package. See further paragraphs for details. | +| 10 | Conflicts | One capability, or name of another package, that conflicts with this package. | +| 11 | Release | Release number of the package. It differs from Version in that Version identifies a state of the upstream project. Release is an increasing number that relates to the package. | +| 12 | License | One license under which the upstream project is released, in [SPDX format](https://spdx.org/licenses). | +| 13 | BuildRelease | Uniquely identifies a rebuild of a *stone*. It is an increasing number. | +| 14 | PackageURI | URL relative to the base repository URL, where to download a package from. It is specific to a Repository *stone*. | +| 15 | PackageHash | SHA-256 hash of a whole package *stone*. It is specific to a Repository *stone*. | +| 16 | PackageSize | Size, in bytes, of a package when installed on disk (assumed when the CAS is empty, thus without de-duplication). It is specific to a Repository *stone*. | +| 17 | BuildDepends | Build-time dependency of a package. | +| 18 | SourceURI | Upstream URL for the source archive of a package. | +| 19 | SourcePath | Relative path for the source within the upstream URL. | +| 20 | SourceRef | Git commit or ref of the upstream source. | + +### kind + +`kind` is an enum. + +|Value|Name|Description| +|---|---|---| +| 1 | Int8 | int, 1 byte long. | +| 2 | Uint8 | uint, 1 byte long. | +| 3 | Int16 | int, 2 bytes long. | +| 4 | Uint16 | uint, 2 bytes long. | +| 5 | Int32 | int, 4 bytes long. | +| 6 | Uint32 | uint, 4 bytes long. | +| 7 | Int64 | int, 8 bytes long. | +| 8 | Uint64 | uint, 8 bytes long. | +| 9 | String | String. | +| 10 | Dependency | PackageReference structure. | +| 11 | Provider | PackageReference structure. | + +### PackageReference + +As explained before, the encoding format of `value` is explicated by the `kind` enum. +Most of the kinds are a characterization of a base format, +but there is one outstanding kind: PackageReference. Below is the format of PackageReference. + +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| kind | uint | 1 | Kind of package one depends on, or provides. | +| name | str | `length` - 1 | Value of the reference. | + +#### kind + +`PackageReference`'s `kind` field is an enum. + +|Value|Name|Description| +|---|---|---| +| 0 | PackageName | A package depends on, or provides, a package name. | +| 1 | SharedLibrary | A package depends on, or provides, a shared library (`*.so` file). | +| 2 | PkgConfig | A package depends on, or provides, a [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config) target. | +| 3 | Interpreter | A package depends on, or provides, a program interpreter (i.e. an ELF binary with `PT_INTERP` set). | +| 4 | CMake | A package depends on, or provides, a [CMake](https://cmake.org) target. | +| 5 | Python | A package depends on, or provides, a Python module. | +| 6 | BinaryDep | A package depends on, or provides, a binary file in `/usr/bin`. | +| 7 | SystemBinary | A package depends on, or provides, a binary file in `/usr/sbin`. | +| 8 | PkgConfig32 | Like `PkgConfig`, but emul32-compatible. | diff --git a/src/content/docs/Developers/Stone/V1/header.mdx b/src/content/docs/Developers/Stone/V1/header.mdx index 5451eeb0..234c950f 100644 --- a/src/content/docs/Developers/Stone/V1/header.mdx +++ b/src/content/docs/Developers/Stone/V1/header.mdx @@ -1,34 +1,33 @@ --- -title: V1 Header -lastUpdated: 2025-03-28T05:28:00Z -description: The v1 header of the stone format +title: Header +lastUpdated: 2026-04-22T00:00:00Z +description: The header of the Stone v1 format --- -The v1 header contains 3 fields to denote the type of the `.stone` file as well as a count of the payloads. These are contained within the 24-byte `data` field of the agnostic header. +The v1 header is contained in Prelude's 24-byte Data field. +It contains fields to denote the type of the *stone* as well as the number of payloads. ## Fields - | Field | Size | Description | - |---------------|----------|-------------------------------------------| - | num_payloads | 2 bytes | Number of all payloads within the archive | - | padding_chk | 21 bytes | Simple corruption check (fixed content) | - | type | 1 byte | Denotes the archive type | +| Field | Type | Size (bytes) | Description | +|---|---|---|---| +| num_payloads | uint | 2 | Number of payloads within the *stone*. | +| padding_chk | blob | 21 | Corruption check (fixed content). | +| type | uint | 1 | Denotes the *stone*'s type. | -## The padding check +### padding_chk -While building the stone format, we built-in the `.data` field to permit future extensions in subsequent stone versions. As of v1, the `.padding_chk` field contains a statically initialized array as a mild corruption check. +`padding_chk` contains a fixed array as a mild corruption check: +`[0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 5, 0, 0, 6, 0, 0, 7]`. -```rust -const INTEGRITY_CHECK: [u8; 21] = [ - 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 5, 0, 0, 6, 0, 0, 7, -]; -``` +### type -## Types +`type` denotes the intended use of a *stone*. Applications may consider +the value of this field when executing logic, but it doesn't affect the way a *stone* is encoded or decoded. - | Name | Value | Description | - |---------------|-------|---------------------------------------------------------------------| - | Binary | 1 | Standard package | - | Delta | 2 | currently unused | - | Repository | 3 | A package repository index | - | BuildManifest | 4 | A build-time artefact containing the _yield potential_ of a package | +| Value | Type | Description | +|---|---|---| +| 1 | Binary | Standard package. | +| 2 | Delta | Currently unused. | +| 3 | Repository | Index of a repository of packages. | +| 4 | BuildManifest | Build-time artefact containing the _yield potential_ of a package. | diff --git a/src/content/docs/Developers/Stone/V1/index.mdx b/src/content/docs/Developers/Stone/V1/index.mdx index d63fe49c..28548767 100644 --- a/src/content/docs/Developers/Stone/V1/index.mdx +++ b/src/content/docs/Developers/Stone/V1/index.mdx @@ -1,10 +1,46 @@ --- -title: V1 Stone -lastUpdated: 2025-03-28T05:28:00Z -description: V1 Stone format +title: Overview +lastUpdated: 2026-04-22T00:00:00Z +description: Overview of the Stone v1 format --- import DirectoryList from '@components/DirectoryList.astro'; -The V1 Stone Format is the format currently employed by AerynOS, and is the first revision of our format. Over time we will continue to enhance the format and introduce new features, gated explicitly to a version. +The v1 of the Stone format is the version currently employed by AerynOS, and is the first revision of our format. + +:::tip +The contents below extend the version-agnostic components of the Stone format. +Readers are encouraged to read the Stone format overview first. +::: + +v1 revolves around the concept of records.\ +A payload contains one or more records, all of the same type, which specified in the payload's sub-header; +the record's type describes the information it carries.\ +Records within a payload may be compressed as a whole archive (not individually) using [Zstandard](https://facebook.github.io/zstd/).\ +With the exception of the Content record, all records have a fixed size determined by their type, or have a preamble that reveals the final size. +The Content record is unique in that it spans the entire payload and must be the only record it contains. + +```mermaid +%%{ + init: { + 'treemap': { + 'showValues': false + } + } +}%% +treemap-beta +"Payload 1" + "Header": 32 + "Records" + "Record 1": 8 + "Record 3": 8 + "Record 2": 8 + "Record 4": 8 +"Payload 2" + "Header": 32 + "Records" + "Record 1": 8 + "Record 2": 8 +"…": 47 +``` diff --git a/src/content/docs/Developers/Stone/V1/payload-subheader.mdx b/src/content/docs/Developers/Stone/V1/payload-subheader.mdx new file mode 100644 index 00000000..a347af41 --- /dev/null +++ b/src/content/docs/Developers/Stone/V1/payload-subheader.mdx @@ -0,0 +1,79 @@ +--- +title: Payload's sub-header +lastUpdated: 2026-04-22T00:00:00Z +description: The content of Payload's sub-header +--- + +Described below is the format of the 32-byte-long payload sub-header. This format is unique to v1. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldTypeSize (bytes)Description
stored_sizeuint8 + Compressed size, in bytes, of the records. + If records are not compressed, stored_size is equal to plain_size. +
plain_sizeuint8Uncompressed size, in bytes, of records.
checksumblob8Checksum of records, computed using XXH3_64bits.
num_recordsuint4Number of records contained in the payload.
versionuint2Version of the payload data format. Unused.
record_kinduint1Kind of records that the payload contains.
compressionuint1Compression algorithm used for the records.
+ +### record_kind + +`record_kind` is an enum. The following table lists all possible values: + +|Value|Name| +|---|---| +|1|Meta| +|2|Content| +|3|Layout| +|4|Index| +|5|Attributes + +### compression +|Value|Name| +|---|---| +|1|Uncompressed| +|2|Zstd| diff --git a/src/content/docs/Developers/Stone/index.mdx b/src/content/docs/Developers/Stone/index.mdx index ccace43b..5606b708 100644 --- a/src/content/docs/Developers/Stone/index.mdx +++ b/src/content/docs/Developers/Stone/index.mdx @@ -1,6 +1,6 @@ --- title: Stone Format -lastUpdated: 2026-04-19T00:00:00Z +lastUpdated: 2026-04-22T00:00:00Z description: An overview of the Stone format --- import DirectoryList from '@components/DirectoryList.astro'; @@ -38,16 +38,20 @@ packet +32: "[...]" ``` +The first payload, if present, immediately follows the Prelude. + The fundamental types of the Stone format are integer numbers and strings. -| Type | Format | Length (in bytes) | Description | Abbreviation| +| Type | Format | Size (bytes) | Description | Abbreviation| |---|---|---|---|---| -| Signed integer | Big-endian | Various | Integer number with a sign. When negative, two's complement is used. The range of values goes from -2^(n-1) to 2^(n-1)-1, where `n` is the number of bytes. |uint| -| Unsigned integer | Big-endian | Various | Integer number without a sign. The range of values goes from 0 to 2^(n)-1, where `n` is the number of bytes. |int| -| String | UTF-8 | Various | A string of text, without the NULL termination. |str| - -Types with a variable length are always combined with a field that reveals the actual length. -The position of such field depends on the version of the Stone format in use. +| Signed integer | Big-endian | Variable | Integer number with a sign. When negative, two's complement is used. The range of values goes from -2^(n-1) to 2^(n-1)-1, where `n` is the number of bytes. |uint| +| Unsigned integer | Big-endian | Variable | Integer number without a sign. The range of values goes from 0 to 2^(n)-1, where `n` is the number of bytes. |int| +| String | UTF-8 | Variable | String of text, without the NULL termination. |str| +| Blob | Undetermined | Variable | Meaningless or context-dependent array of bytes. | blob + +The length of a field is documented where it is known a priori. +When this is not possible, it is always paired with another field that reveals its actual length. +The position of the latter depends on the version of the Stone format in use. The documentation will delve into the details of each section in the next pages. diff --git a/src/content/docs/Developers/Stone/prelude.mdx b/src/content/docs/Developers/Stone/prelude.mdx index 8c2c3e3a..ba7754f8 100644 --- a/src/content/docs/Developers/Stone/prelude.mdx +++ b/src/content/docs/Developers/Stone/prelude.mdx @@ -9,17 +9,17 @@ fields can be handled separately from version and format detection. This is a 32 ## Fields -| Field | Type | Size | Description | +| Field | Type | Size (bytes) | Description | |---|---|---|---| -| Magic | str | 4 | Always `0x006d6f73` | -| Data | Version-dependent | 24 | Version-specific header of the *stone* | -| Version | uint | 4 | Version number, i.e. `1` | +| magic | str | 4 | Always `0x006d6f73`. | +| data | Version-dependent | 24 | Version-specific header of the *stone*. | +| version | uint | 4 | Version number, i.e. `1`. | -### Magic +### magic It's the [magic number](https://en.wikipedia.org/wiki/Magic_number_(programming)) of the Stone format. -The Magic field always contains `['\0', 'M', 'O', 'S']`. +The `magic` field always contains `['\0', 'M', 'O', 'S']`. It is defined after AerynOS's package manager: `moss`. In the Rust language it is defined as: @@ -28,11 +28,11 @@ In the Rust language it is defined as: pub const STONE_MAGIC: &[u8; 4] = b"\0mos"; ``` -### Data +### data -The content of the Data field depends on the Version field. +The content of the `data` field depends on the Version field. Documentation about Stone versions is available in the next pages. -### Version +### version A number that uniquely identifies the version of the Stone format in use.