diff --git a/_ext/document_authors.py b/_ext/document_authors.py new file mode 100644 index 00000000..6903b973 --- /dev/null +++ b/_ext/document_authors.py @@ -0,0 +1,85 @@ +import yaml +from docutils import nodes +from docutils.parsers.rst import Directive + +ORCID_ICON = "https://orcid.org/assets/vectors/orcid.logo.icon.svg" +GITHUB_ICON = "https://github.githubassets.com/favicons/favicon.svg" + + +def _icon_link(uri, src, alt): + """A hyperlink wrapping a small inline image.""" + ref = nodes.reference("", "", refuri=uri) + img = nodes.image( + uri=src, + alt=alt, + classes=["rfc-author-icon"], + ) + ref += img + return ref + + +class DocumentAuthors(Directive): + def run(self): + env = self.state.document.settings.env + src = env.doc2path(env.docname) + with open(src, encoding="utf-8") as f: + text = f.read() + + parts = text.split("---", 2) + if len(parts) < 3: + raise self.error("rfc-authors: no YAML front matter found") + meta = yaml.safe_load(parts[1]) or {} + + authors = meta.get("authors", []) + if not authors: + raise self.error("rfc-authors: no 'authors' in front matter") + + # Number unique affiliations in first-seen order + affils, order = {}, [] + for a in authors: + aff = a.get("affiliation") + if aff and aff not in affils: + order.append(aff) + affils[aff] = len(order) + + para = nodes.paragraph(classes=["rfc-authors"]) + for i, a in enumerate(authors): + if i: + para += nodes.Text(" and " if i == len(authors) - 1 else ", ") + + para += nodes.Text(a["name"]) + + aff = a.get("affiliation") + if aff: + para += nodes.superscript(text=str(affils[aff])) + + orcid = a.get("orcid") + if orcid: + uri = ( + orcid + if str(orcid).startswith("http") + else f"https://orcid.org/{orcid}" + ) + para += nodes.Text(" ") + para += _icon_link(uri, ORCID_ICON, "ORCID") + + gh = a.get("github") + if gh: + uri = gh if str(gh).startswith("http") else f"https://github.com/{gh}" + para += nodes.Text(" ") + para += _icon_link(uri, GITHUB_ICON, "GitHub") + + result = [para] + + for aff in order: + p = nodes.paragraph(classes=["rfc-affiliation"]) + p += nodes.superscript(text=str(affils[aff])) + p += nodes.Text(" " + aff) + result.append(p) + + return result + + +def setup(app): + app.add_directive("document-authors", DocumentAuthors) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/_ext/rfc_status.py b/_ext/rfc_status.py new file mode 100644 index 00000000..36571690 --- /dev/null +++ b/_ext/rfc_status.py @@ -0,0 +1,218 @@ +import os +import yaml +from docutils import nodes +from docutils.parsers.rst import Directive + + +def _read_front_matter(path): + try: + with open(path, encoding="utf-8") as f: + text = f.read() + except OSError: + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + try: + return yaml.safe_load(parts[1]) or {} + except yaml.YAMLError: + return {} + + +def _numbered_subdirs(base): + if not os.path.isdir(base): + return + for e in os.listdir(base): + p = os.path.join(base, e) + if os.path.isdir(p) and e != "index": + yield e, p + + +def _folder_sort_key(label): + """'1' -> (1, ''), '1b' -> (1, 'b'), so rounds sort after their round 1.""" + num = "".join(c for c in label if c.isdigit()) + suffix = "".join(c for c in label if not c.isdigit()) + return (int(num) if num else 0, suffix) + + +def _thread_stem(label): + """'1b' -> '1', '12c' -> '12'. Used to count threads, not documents.""" + i = len(label) + while i > 0 and label[i - 1].isalpha(): + i -= 1 + return label[:i] or label + + +def _collect_section(rfc_dir, section): + """Return list of (label, meta), sorted by folder key (round-aware).""" + base = os.path.join(rfc_dir, section) + rows = [] + for label, subdir in _numbered_subdirs(base): + index_path = os.path.join(subdir, "index.md") + if not os.path.isfile(index_path): + continue + rows.append((label, _read_front_matter(index_path))) + rows.sort(key=lambda r: _folder_sort_key(r[0])) + return rows + + +def _thread_count(rows): + return len({_thread_stem(label) for label, _ in rows}) + + +def _count_versions(rfc_dir): + base = os.path.join(rfc_dir, "versions") + return sum(1 for _ in _numbered_subdirs(base)) if os.path.isdir(base) else 0 + + +class RFCStatus(Directive): + SECTION_LABELS = { + "reviews": ("Reviewer", "Review"), + "comments": ("Commenter", "Comment"), + "responses": ("Author", "Response"), + } + + def run(self): + env = self.state.document.settings.env + src = env.doc2path(env.docname) + rfc_dir = os.path.dirname(src) + central = _read_front_matter(src) + + reviews = _collect_section(rfc_dir, "reviews") + comments = _collect_section(rfc_dir, "comments") + responses = _collect_section(rfc_dir, "responses") + n_versions = _count_versions(rfc_dir) + + all_dates = [ + str(m.get("date", "")) + for _, m in (reviews + comments + responses) + if m.get("date") + ] + last_update = max(all_dates) if all_dates else str(central.get("date", "")) + + result = [] + + summary = nodes.paragraph(classes=["rfc-status-summary"]) + summary += nodes.Text( + f"As of the last update, {last_update}: " + f"{_thread_count(comments)} comments, " + f"{_thread_count(reviews)} reviews, " + f"{_thread_count(responses)} responses, " + ) + result.append(summary) + result.append(nodes.Text("\n Authors and editors:")) + result.append(self._people_table(central)) + result.append(nodes.Text("\n Reviews, comments, and responses:")) + result.append(self._activity_table(reviews, comments, responses)) + return result + + # ---- Table 1: Authors + Editors ---- + + def _people_table(self, central): + cols = ["Role", "Name", "GitHub", "Institution", "Date", "Status"] + table, tbody = self._new_table(cols, (10, 22, 20, 22, 10, 16)) + for person in central.get("authors", []): + tbody += self._person_row(person, "Author") + for person in central.get("editors", []): + tbody += self._person_row(person, "Editor") + return table + + def _person_row(self, person, role): + row = nodes.row() + row += self._text_entry(role) + row += self._text_entry(person.get("name", "")) + gh = person.get("github") + row += self._github_entry([gh] if gh else []) + row += self._text_entry(person.get("affiliation", "")) + row += self._text_entry(str(person.get("date", ""))) + row += self._text_entry(person.get("role", "")) + return row + + # ---- Table 2: Reviews + Comments + Responses (one row per round) ---- + + def _activity_table(self, reviews, comments, responses): + cols = ["Link", "Name", "GitHub", "Institution", "Date", "Rec."] + table, tbody = self._new_table(cols, (12, 22, 18, 20, 12, 16)) + for section, rows in ( + ("reviews", reviews), + ("comments", comments), + ("responses", responses), + ): + _, link_label = self.SECTION_LABELS[section] + for label, meta in rows: + tbody += self._activity_row( + meta, + f"{link_label}\u00a0{label}", # non-breaking space + f"./{section}/{label}/index", + ) + return table + + def _activity_row(self, meta, link_text, link_target): + authors = meta.get("authors", []) + names = ", ".join(a.get("name", "") for a in authors if a.get("name")) + handles = [a["github"] for a in authors if a.get("github")] + affils = [] + for a in authors: + aff = a.get("affiliation") + if aff and aff not in affils: + affils.append(aff) + recommendation = str(meta.get("recommendation", "")).replace("_", " ") + + row = nodes.row() + row += self._status_entry(link_text, link_target) + row += self._text_entry(names) + row += self._github_entry(handles) + row += self._text_entry(", ".join(affils)) + row += self._text_entry(str(meta.get("date", ""))) + row += self._text_entry(recommendation) + return row + + # ---- shared builders ---- + + def _new_table(self, cols, widths): + table = nodes.table() + tgroup = nodes.tgroup(cols=len(cols)) + table += tgroup + for w in widths: + tgroup += nodes.colspec(colwidth=w) + thead = nodes.thead() + tgroup += thead + hrow = nodes.row() + for c in cols: + hrow += self._text_entry(c) + thead += hrow + tbody = nodes.tbody() + tgroup += tbody + return table, tbody + + def _text_entry(self, text): + entry = nodes.entry() + para = nodes.paragraph() + para += nodes.Text(text or "") + entry += para + return entry + + def _github_entry(self, handles): + entry = nodes.entry() + para = nodes.paragraph() + for i, gh in enumerate(handles): + if i: + para += nodes.Text(", ") + para += nodes.reference("", gh, refuri=f"https://github.com/{gh}") + entry += para + return entry + + def _status_entry(self, text, target): + entry = nodes.entry() + para = nodes.paragraph() + if target: + para += nodes.reference("", text, refuri=target) + else: + para += nodes.Text(text or "") + entry += para + return entry + + +def setup(app): + app.add_directive("rfc-status", RFCStatus) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/_static/document_authors.css b/_static/document_authors.css new file mode 100644 index 00000000..72aae27a --- /dev/null +++ b/_static/document_authors.css @@ -0,0 +1,17 @@ +.rfc-author-icon { + height: 1em; + width: 1em; + vertical-align: -0.15em; + margin: 0 0.05em; + display: inline; +} + +.rfc-authors { + font-size: 1.05em; +} + +.rfc-affiliation { + font-size: 0.85em; + color: var(--pst-color-text-muted, #666); + margin: 0.1em 0; +} diff --git a/conf.py b/conf.py index 55152f5e..414148c7 100644 --- a/conf.py +++ b/conf.py @@ -6,6 +6,8 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os + project = "NGFF" copyright = "2020-2025, NGFF Community" author = "NGFF Community" @@ -13,17 +15,25 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +# Needed for custom document_authors extension to be found in _ext +import os +import sys + +sys.path.insert(0, os.path.abspath("_ext")) + extensions = [ "myst_parser", "sphinx_reredirects", "sphinx_design", "sphinxcontrib.bibtex", + "document_authors", + "rfc_status", ] bibtex_bibfiles = ["references.bib"] source_suffix = [".rst", ".md"] myst_heading_anchors = 5 -myst_enable_extensions = ["deflist", "strikethrough", "colon_fence"] - +myst_enable_extensions = ["deflist", "strikethrough", "colon_fence", "substitution"] templates_path = ["_templates"] exclude_patterns = [ "_build", @@ -81,6 +91,7 @@ html_css_files = [ "https://cdn.datatables.net/v/dt/dt-1.11.5/datatables.min.css", + "document_authors.css", ] html_js_files = [ diff --git a/requirements.txt b/requirements.txt index 19336c7f..2e1e8b74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ testresources sphinx-reredirects sphinxcontrib-bibtex sphinx-design +pyyaml diff --git a/rfc/1/comments/1/index.md b/rfc/1/comments/1/index.md index 336343cf..ae7508bd 100644 --- a/rfc/1/comments/1/index.md +++ b/rfc/1/comments/1/index.md @@ -1,9 +1,17 @@ +--- +authors: + - name: Wouter-Michiel Vierdag + affiliation: EMBL + github: melonora + - name: Luca Marconato + affiliation: EMBL + github: LucaMarconato +--- # RFC-1: Comment 1 -| Name | GitHub Handle | Institution | -|------------------------|---------------|----------------------| -| Wouter-Michiel Vierdag | melonora | EMBL | -| Luca Marconato | LucaMarconato | EMBL | +```{document-authors} + +``` ## Comments on implementations diff --git a/rfc/1/index.md b/rfc/1/index.md index ee7fba5f..2e29be03 100644 --- a/rfc/1/index.md +++ b/rfc/1/index.md @@ -1,5 +1,4 @@ -RFC-1: RFC Process -================== +# RFC-1: RFC Process Definition of the NGFF “Request for Comment” (RFC) process @@ -15,6 +14,10 @@ versions/index ## Status +```{rfc-status} + +``` + This RFC has been adopted (S4). ```{list-table} Record @@ -33,7 +36,7 @@ This RFC has been adopted (S4). - [joshmoore](https://github.com/joshmoore) - [German BioImaging, e.V.](https://ror.org/05tpnw772) - 2023-12-23 - - Author ([PR](https://github.com/ome/ngff/pull/222)) + - Author ([PR](https://github.com/ome/ngff/pull/222)) * - Reviewer - Davis Bennett, John Bogovic, Michael Innerberger, Mark Kittisopikul, Virginia Scarlett, Yurii Zubov - [d-v-b](https://github.com/d-v-b), [bogovicj](https://github.com/bogovicj), [minnerbe](https://github.com/minnerbe), [mkitti](https://github.com/mkitti), [virginiascarlett](https://github.com/virginiascarlett), [yuriyzubov](https://github.com/yuriyzubov) @@ -154,16 +157,16 @@ which originated in the Internet Engineering Task Force (IETF), for use in the NGFF community as has been done in a number of other communities ([Rust](https://github.com/rust-lang/rfcs/blob/master/0000-template.md), [Hashicorp](https://works.hashicorp.com/articles/rfc-template), [Tensorflow](https://github.com/tensorflow/community/blob/master/rfcs/yyyymmdd-rfc-template.md), etc.) More information can be found under: -- [https://en.wikipedia.org/wiki/Internet\_Standard#Standardization\_process](https://en.wikipedia.org/wiki/Internet_Standard#Standardization_process) -- [https://en.wikipedia.org/wiki/Request\_for\_Comments](https://en.wikipedia.org/wiki/Request_for_Comments) +- [https://en.wikipedia.org/wiki/Internet_Standard#Standardization_process](https://en.wikipedia.org/wiki/Internet_Standard#Standardization_process) +- [https://en.wikipedia.org/wiki/Request_for_Comments](https://en.wikipedia.org/wiki/Request_for_Comments) ## Proposal -Requests for Comment (RFCs) are intended to structure high-level discussions on changes within the NGFF community and record outcomes including key opinions, actions, and decisions. The overall goal of the process is timely and transparent decision-making for a stable and trusted community specification. It should be clear after reading the RFC which stakeholder (**Author**, **Reviewer**, **Editor**, etc.) is responsible for each step of the process, what options are available to the decision makers, and how much time the community can expect that decision to take. +Requests for Comment (RFCs) are intended to structure high-level discussions on changes within the NGFF community and record outcomes including key opinions, actions, and decisions. The overall goal of the process is timely and transparent decision-making for a stable and trusted community specification. It should be clear after reading the RFC which stakeholder (**Author**, **Reviewer**, **Editor**, etc.) is responsible for each step of the process, what options are available to the decision makers, and how much time the community can expect that decision to take. ![Simplified drawing of the RFC process](./drawing.png) -**Figure 1. Simplified drawing of the RFC process** An RFC draft (1) is +**Figure 1. Simplified drawing of the RFC process** An RFC draft (1) is proposed by **Authors** who would like to see some change in the NGFF community. There is a period of gathering _[endorsements](#def-endorsement)_ (2) which will be listed in the RFC itself. This gives future readers and @@ -178,7 +181,7 @@ with the RFC changes (5). After a minimum number of implementations have been achieved, the RFC is considered _adopted_ (6). The RFC process functions by encouraging submissions from the -community that are recorded for posterity *even if not adopted*. +community that are recorded for posterity _even if not adopted_. Descriptive and complete comments from both **Authors** and **Reviewers** are critical to have a clear understanding of what decisions have been made. Goals of this process include maintaining a public record of @@ -214,14 +217,14 @@ draft has reached a stage where it is ready for review, **Editors** will merge it as a record of the fact that the suggestion has been made, and it will then become available on https://ngff.openmicroscopy.org. -**Endorsers** are non-**Author** supporters of an RFC, listed in a table within the RFC. -**Reviewers** who have given an "Accept" recommendation and **Implementers** are also considered **Endorsers**. +**Endorsers** are non-**Author** supporters of an RFC, listed in a table within the RFC. +**Reviewers** who have given an "Accept" recommendation and **Implementers** are also considered **Endorsers**. **Editors** are responsible for facilitating all parts of the RFC process. They identify whether a PR should or should not follow the RFC process, and choose when a draft is ready to become an RFC. They also choose appropriate **Reviewers** for an RFC and manage the communication between -**Authors** and **Reviewers**. +**Authors** and **Reviewers**. **Implementers** are responsible for an implementation of the NGFF specification in one or more programming languages. It is critical that specification RFCs have been evaluated by **Implementers** which is often best done in the implementation rather than a review. Therefore, statements that an RFC is “planned”, “begun”, or “complete” for an implementation will be given similar weight to an endorsement or positive review. @@ -252,8 +255,8 @@ Identifiers such as "D1", "R2", "S3", refer to individual steps. Notes regarding specific requirements are called out throughout the text with the following symbols: -> * 🕑 The clock symbol specifies definitive wait times within the process. -> * 📂 The folder symbol specifies requirements on additions to the repository, +> - 🕑 The clock symbol specifies definitive wait times within the process. +> - 📂 The folder symbol specifies requirements on additions to the repository, > for example an implementation or failing test. ### Phases @@ -324,36 +327,37 @@ should only be used when necessary.) (rfc-recommendations)= Possible recommendations from **Reviewers** in ascending order of support are: -* “Reject” suggests that a **Reviewer** considers there to be no merit to an +- “Reject” suggests that a **Reviewer** considers there to be no merit to an RFC. This should be a last recourse. Instead, suggestions in a “Major changes” recommendation might include attempting an Extension rather than an RFC so that not all implementations need concern themselves with the matter. -* “Major changes” suggests that a **Reviewer** sees the potential value of an +- “Major changes” suggests that a **Reviewer** sees the potential value of an RFC but will require significant changes before being convinced. Suggestions SHOULD be provided on how to concretely improve the proposal in order to make it acceptable and change the **Reviewer**’s recommendation. -* “Minor changes” suggests that if the described changes are made, that +- “Minor changes” suggests that if the described changes are made, that **Editors** can move forward with an RFC without a further review. -* “Accept” is a positive vote and no text review is strictly necessary, though +- “Accept” is a positive vote and no text review is strictly necessary, though may be provided to add context to the written record. A **Reviewer** who accepts an RFC is joining the list of endorsements. Three additional versions of the "Accept" recommendation are available for **Reviewers** who additionally maintain an implementation of the NGFF specification to express further support: -* “Plan to implement” with an estimated timeline -* “Implementation begun” with an estimated timeline -* “Implementation complete” with a link to the available code + +- “Plan to implement” with an estimated timeline +- “Implementation begun” with an estimated timeline +- “Implementation complete” with a link to the available code Where a review is required, **Reviewers** are free to structure the text in the most useful way. A [template markdown file](templates/review_template) is available but not mandatory. Useful sections include: -* Summary -* Conflicts of interest (if they exist) -* Significant comments and questions -* Minor comments and questions -* Recommendation +- Summary +- Conflicts of interest (if they exist) +- Significant comments and questions +- Minor comments and questions +- Recommendation The tone of a review should be cordial and professional. The goal is to communicate to the **Authors** what it would take to make the RFC acceptable. @@ -367,11 +371,11 @@ contact **Reviewers** to see if their recommendations have changed. > 🕑 Authors responses to Reviewers should be returned to the Editors in less than two weeks. -(anchor-rebuttal-r6)= This brings a critical, and possibly iterative, decision point (R6). If all **Reviewers** `approve` and there are no further changes needed, the RFC can progress to S1 as soon as there are two in-progress implementations. If the **Reviewers** do _not_ approve, then the **Editors** will make one of three decisions (R7): +(anchor-rebuttal-r6)= This brings a critical, and possibly iterative, decision point (R6). If all **Reviewers** `approve` and there are no further changes needed, the RFC can progress to S1 as soon as there are two in-progress implementations. If the **Reviewers** do _not_ approve, then the **Editors** will make one of three decisions (R7): -* The **Editors** MAY provide **Authors** a list of necessary changes. These will be based on the **Reviewers** suggestions but possibly modified, e.g., to remove contradictions. -* The **Editors** MAY decide that the RFC is to be closed (R9). This is the decision that SHOULD be chosen if there is a unanimous `Reject` recommendation. The **Authors** MAY then decide to re-draft a new RFC (D2). -* Finally, the **Editors** MAY decide that no further changes are necessary (S0). +- The **Editors** MAY provide **Authors** a list of necessary changes. These will be based on the **Reviewers** suggestions but possibly modified, e.g., to remove contradictions. +- The **Editors** MAY decide that the RFC is to be closed (R9). This is the decision that SHOULD be chosen if there is a unanimous `Reject` recommendation. The **Authors** MAY then decide to re-draft a new RFC (D2). +- Finally, the **Editors** MAY decide that no further changes are necessary (S0). If the **Editors** decide to override the recommendations of the **Reviewers** (R7) the **Editors** MUST include a response (S0). This may occur, for example, if consent between the reviewers cannot be reached. In the case of a unanimous `Reject`, the **Editors** SHOULD attempt to find at least one additional, approving **Reviewer** . @@ -419,7 +423,7 @@ listed, the specification will be considered "adopted". The adopted specification will be slotted into a release version by the **Editors** and the **Authors** are encouraged to be involved in that release. -> 📂 Two released implementations required for being adopted. +> 📂 Two released implementations required for being adopted. ## Policies @@ -429,8 +433,9 @@ This section defines several concrete aspects of the RFC process not directly re Unless otherwise specified in the text, the following considerations are taken into account when making decisions regarding RFCs: - - **prefer working examples**: whether an implementation of an RFC or a failing test which exposes an issue in a proposal, working examples will tend to carry more weight in decision making. - - **technical expertise**: all other considerations being equal, feedback from stakeholders with more technical expertise in a matter under consideration will tend to carry more weight in decision making. + +- **prefer working examples**: whether an implementation of an RFC or a failing test which exposes an issue in a proposal, working examples will tend to carry more weight in decision making. +- **technical expertise**: all other considerations being equal, feedback from stakeholders with more technical expertise in a matter under consideration will tend to carry more weight in decision making. - **newcomer advantage**: care will be taken not to let existing implementations overly dictate the future strategic direction of NGFF in order to avoid premature calcification. ### RFC Prioritization @@ -446,7 +451,7 @@ the community. Which cross-sections are chosen MAY depend on a given RFC but might include geographic distributions, the variety of imaging modalities, and/or programming languages of the expected implementations. An attempt MUST also be made to select both supporting and dissenting voices from the community. -*Editors* and *Reviewers* should proactively disclose any potential conflicts +_Editors_ and _Reviewers_ should proactively disclose any potential conflicts of interest to ensure a transparent review process. ### Deadline enforcement @@ -454,10 +459,11 @@ of interest to ensure a transparent review process. In the absence of concrete mechanisms for deadline enforcement (penalties, etc), all members of the NGFF community and especially the **Editors** SHOULD strive to prevent the specification process from becoming blocked. The **Editors**, however will endeavor to: -* keep a record of all communications to identify bottlenecks and improve the RFC process; -* frequently contact **Authors** and **Reviewers** regarding approaching deadlines; -* find new **Reviewers** when it becomes clear that the current slate is overextended; -* and proactively mark RFCs as inactive if it becomes clear that progress has stalled. + +- keep a record of all communications to identify bottlenecks and improve the RFC process; +- frequently contact **Authors** and **Reviewers** regarding approaching deadlines; +- find new **Reviewers** when it becomes clear that the current slate is overextended; +- and proactively mark RFCs as inactive if it becomes clear that progress has stalled. **Authors** and **Reviewers** are encouraged to be open and honest, both with themselves and the other members of the process, on available time. A short message stating that an edit or a review will not occur on deadline or even at all is preferable to silence. @@ -467,7 +473,7 @@ The process description describes “sufficient endorsement” in two locations, Under RFC-0, three implementation languages — Javascript, Python, and Java — were considered “reference”, or “required”, for a specification to be complete. This proved a difficult barrier since the implementation teams were not directly funded for work on NGFF. -RFC-1 has chosen to start with a simpler requirement: **two** separate implementations MUST be _begun_ to enter the SPEC phase and **two** separate implementations (they need not be the same ones) MUST be _released_ to be considered adopted. In both cases, at least **one** of those implementations MUST come from an **Implementer** who is not among the **Authors**. Additionally, data written by both implementations MUST be readable (and therefore validatable) by at least **one** of the implementations. +RFC-1 has chosen to start with a simpler requirement: **two** separate implementations MUST be _begun_ to enter the SPEC phase and **two** separate implementations (they need not be the same ones) MUST be _released_ to be considered adopted. In both cases, at least **one** of those implementations MUST come from an **Implementer** who is not among the **Authors**. Additionally, data written by both implementations MUST be readable (and therefore validatable) by at least **one** of the implementations. It is also strongly encouraged that for each specification change, the [ome-ngff-validator](https://github.com/ome/ome-ngff-validator) additionally be updated. The validator will not fully test the readability of a dataset since it has limited IO capabilities, but it is the most complete tool for validating the metadata associated with a dataset. @@ -480,6 +486,7 @@ This policy does not yet specify whether parts of an RFC may be considered _opti The IETF RFC process disallows edits to published RFCs. (In the extreme case, a single word change has resulted in a new RFC number.) Though this ensures a unique interpretation of any RFC number, it would also lead to significant duplication of content and _churn_ in the NGFF community. Though this decision may be reviewed in the future, RFCs MAY be edited, but **Editors** SHOULD limit modifications to _adopted_ RFCs only for: + - clarification: additional text and examples which simplify the implementation of specifications are welcome; - deprecation: where sections are no longer accurate and especially when they have been replaced by a new RFC, the existing text can be marked and a link to the updated information provided; - and extension: references to new RFCs can be added throughout an existing RFC to provide simpler reading for **Implementers**. @@ -487,10 +494,11 @@ Though this decision may be reviewed in the future, RFCs MAY be edited, but **Ed In writing RFCs, **Authors** SHOULD attempt to clearly identify sections which may be deprecated or extended in the future. Before an RFC is _adopted_ there are a number of versions of an RFC which are produced during the editing and revision process. This RFC does not try to specify how those versions are managed. The **Editors** are encouraged to layout a best practice as described under “Workflow” that simplifies the review process. Possible solutions include: -* using commit numbers version -* making hard-copies of versions under review -* creating a separate repository per RFC -* opening a long-lived “review PR” with a dedicated URL + +- using commit numbers version +- making hard-copies of versions under review +- creating a separate repository per RFC +- opening a long-lived “review PR” with a dedicated URL ### Specification Versions @@ -515,7 +523,7 @@ an expedited process. A similar model is in use within the IETF community. If th ### Handling Disagreements -The OME community is open to everybody and built upon mutual respect. Nevertheless, disagreements do occur. +The OME community is open to everybody and built upon mutual respect. Nevertheless, disagreements do occur. All activities within the NGFF community are conducted under the OME [Code of Conduct](https://github.com/ome/.github/blob/master/CODE_OF_CONDUCT.md#when-something-happens). If you feel that your objections are not being considered, please follow the steps outlined under “When Something Happens”. @@ -551,7 +559,7 @@ on GitHub does not provide the editorial functions that one would want, such as deferring and collecting comments, nor do the conversations provide a consistent whole when revisited after the work on a specification. Additionally, **Authors** have complained of the burden of managing responses. -So there's a need for *something*, but does this proposal go too far in the +So there's a need for _something_, but does this proposal go too far in the other direction? It is certainly true that the formality of the responses asked of the @@ -601,7 +609,7 @@ using adaptions of the RFC process which will not be re-listed here. However, there are also other enhancement processes which are closely related to the NGFF RFC. Most closely, is the Zarr Enhancement Proposals (ZEP) process within the Zarr community. Based originally on a combination of the PEP, NEP, and STAC -processes, the ZEP process uses a council of the implementations (ZIC) +processes, the ZEP process uses a council of the implementations (ZIC) ## Future possibilities @@ -627,12 +635,12 @@ Definitions for terms used throughout this RFC have been collected below. (def-accepted)= **Accepted** : Specifies that an RFC has passed review and all implementers should begin - implementation if they have not done so already. +implementation if they have not done so already. (def-adopted)= **Adopted** : An RFC that has been sufficiently implemented to be considered - as active within the community. +as active within the community. (def-author)= **Author** @@ -641,8 +649,8 @@ Definitions for terms used throughout this RFC have been collected below. (def-comment)= **Comment** : Documents that are included with the RFC discussing the pros and - cons of the proposal in a structured way. Comments from reviewers are - additionally referred to as "reviews". +cons of the proposal in a structured way. Comments from reviewers are +additionally referred to as "reviews". (def-draft)= **Draft** @@ -659,8 +667,8 @@ Definitions for terms used throughout this RFC have been collected below. (def-rfc)= **RFC** ("Request for Comment") : A formal proposal following a standardized - template that is made to the NGFF repository. The proposal need not be - accepted to be published online. +template that is made to the NGFF repository. The proposal need not be +accepted to be published online. (def-pr)= **PR** diff --git a/rfc/1/reviews/1/index.md b/rfc/1/reviews/1/index.md index 154273eb..9ee5c5bd 100644 --- a/rfc/1/reviews/1/index.md +++ b/rfc/1/reviews/1/index.md @@ -1,3 +1,13 @@ +--- +authors: + - name: Joel Lüthi + - name: Virginie Uhlmann + affiliation: BiovisionCenter + - name: Kevin Yamauchi + affiliation: ETH +recommendation: major_changes +date: 2024-03-05 +--- # RFC-1: Review 1 ## Review authors diff --git a/rfc/1/reviews/1b/index.md b/rfc/1/reviews/1b/index.md index c5fc4e44..313dd9d8 100644 --- a/rfc/1/reviews/1b/index.md +++ b/rfc/1/reviews/1b/index.md @@ -1,3 +1,11 @@ +--- +authors: + - name: Joel Lüthi + - name: Virginie Uhlmann + - name: Kevin Yamauchi +recommendation: accept +date: 2024-10-03 +--- # RFC-1: Review 1 Round 2 ## Review authors diff --git a/rfc/1/reviews/2/index.md b/rfc/1/reviews/2/index.md index 738475ab..66fd99e9 100644 --- a/rfc/1/reviews/2/index.md +++ b/rfc/1/reviews/2/index.md @@ -1,3 +1,13 @@ +--- +authors: + - name: Davis Bennett + - name: John Bogovic + - name: Michael Innerberger + - name: Mark Kittisopikul + - name: Virginia Scarlett + - name: Yurii Zubov +recommendation: major_changes +--- # RFC-1: Review 2 ## Contributors diff --git a/rfc/1/reviews/2b/index.md b/rfc/1/reviews/2b/index.md index 91d57e71..2b21d64c 100644 --- a/rfc/1/reviews/2b/index.md +++ b/rfc/1/reviews/2b/index.md @@ -1,3 +1,10 @@ +--- +authors: + - name: John Bogovic + - name: Michael Innerberger + - name: Virginia Scarlett +recommendation: accept +--- # RFC-1: Review 2b ## Contributors diff --git a/rfc/1/reviews/3/index.md b/rfc/1/reviews/3/index.md index 902ffe5f..2975961a 100644 --- a/rfc/1/reviews/3/index.md +++ b/rfc/1/reviews/3/index.md @@ -1,5 +1,18 @@ +--- +authors: + - name: Matthew Hartley + affiliation: BioImage Archive, EMBL-EBI + github: matthewh-ebi +recommendation: accept +--- + # RFC-1: Review 3 +```{document-authors} + +``` + + This review submitted by Matthew Hartley, on behalf on EMBL-EBI's imaging data resources (BioImage Archive, EMPIAR and EMDB). ## Summary diff --git a/rfc/9/comments/1/index.md b/rfc/9/comments/1/index.md index a8807c58..912affd5 100644 --- a/rfc/9/comments/1/index.md +++ b/rfc/9/comments/1/index.md @@ -1,10 +1,20 @@ +--- +authors: + - name: Matt McCormick + affiliation: Fideus Labs LLC + github: thewtex +date: 2025-11-15 +--- + # RFC-9: Comment 1 (rfcs:rfc9:comment1)= ## Comment authors -This comment was written by: Matt McCormick, Fideus Labs LLC. +```{document-authors} + +``` ## Conflicts of interest (optional) @@ -23,6 +33,7 @@ We have implemented the specification, including reading, writing, and all the r We recommend including a concrete example of the expected file order for clarification. This would help implementers understand exactly how to order `zarr.json` files in breadth-first order. For instance, given a hierarchy like: + ``` / ├── zarr.json (root) @@ -37,6 +48,7 @@ For instance, given a hierarchy like: ``` The recommended ZIP entry order would be: + 1. `zarr.json` (root) 2. `image/zarr.json` 3. `labels/zarr.json` diff --git a/rfc/9/comments/2/index.md b/rfc/9/comments/2/index.md index 9ff9f1c0..aa9004aa 100644 --- a/rfc/9/comments/2/index.md +++ b/rfc/9/comments/2/index.md @@ -1,10 +1,20 @@ +--- +authors: + - name: Joost de Folter + affiliation: BioImaging-NL + github: folterj +date: 2025-12-03 +--- + # RFC-9: Comment 2 (rfcs:rfc9:comment2)= ## Comment authors -This comment was written by: Joost de Folter, BioImaging-NL +```{document-authors} + +``` ## Conflicts of interest (optional) diff --git a/rfc/9/comments/3/index.md b/rfc/9/comments/3/index.md index d0f80c2c..5f4c8351 100644 --- a/rfc/9/comments/3/index.md +++ b/rfc/9/comments/3/index.md @@ -1,10 +1,20 @@ +--- +authors: + - name: Chris Barnes + affiliation: German BioImaging + github: clbarnes +date: 2025-12-12 +--- + # RFC-9: Comment 3 (rfcs:rfc9:comment2)= ## Comment authors -This comment was written by: Chris Barnes, German BioImaging +```{document-authors} + +``` ## Conflicts of interest (optional) diff --git a/rfc/9/comments/4/index.md b/rfc/9/comments/4/index.md index e2bfe508..79a55c8d 100644 --- a/rfc/9/comments/4/index.md +++ b/rfc/9/comments/4/index.md @@ -1,12 +1,21 @@ +--- +authors: + - name: Lenard Spiecker + affiliation: Miltenyi Biotec B.V. & Co. KG + - name: Matthias Grunwald + affiliation: Miltenyi Biotec B.V. & Co. KG +date: 2026-01-09 +--- + # RFC-9: Comment 4 (rfcs:rfc9:comment4)= ## Comment authors -This comment was written by: Lenard Spiecker1 and Matthias Grunwald1 +```{document-authors} -1 Miltenyi Biotec B.V. & Co. KG +``` ## Conflicts of interest @@ -22,7 +31,7 @@ We support standardizing single-file OME-Zarr via ZIP (.ozx). In our context at - **CRC/hash requirement:** ZIP requires CRC32 for file entries, which is useful for integrity verification, but it burdens implementations, especially for partial writes and reads. With sharding, recomputing CRCs for sub-ranges or appends is tricky; clarifying recommended strategies (e.g., validating at shard or chunk granularity and deferring CRC checks for in-flight writes) would help implementers. Implementers should also note that there is no support for CRC32(B) SIMD instructions on x86_64 (SSE4.2 only supports CRC32(C)). This point could be added under the drawback section in the RFC. -- **Ordering of zarr.json first:** While placing the root and all other `zarr.json` files at the beginning of the archive potentially aids discovery and streaming access, practical implementations may still read the ZIP comment together with the central directory first. The main reason is that the first `zarr.json` can become obsolete, rendering streaming access inefficient compared to seeking. We also observed that strict file ordering cannot be maintained when appending a new `zarr.json` (e.g., adding labels) to an existing .ozx file. Furthermore, we encounter cases where metadata is generated during acquisition; therefore, we lean toward writing data first and metadata second to avoid writing it twice. For the stated reasons, we will likely not produce .ozx files with `zarr.json` files ordered first. +- **Ordering of zarr.json first:** While placing the root and all other `zarr.json` files at the beginning of the archive potentially aids discovery and streaming access, practical implementations may still read the ZIP comment together with the central directory first. The main reason is that the first `zarr.json` can become obsolete, rendering streaming access inefficient compared to seeking. We also observed that strict file ordering cannot be maintained when appending a new `zarr.json` (e.g., adding labels) to an existing .ozx file. Furthermore, we encounter cases where metadata is generated during acquisition; therefore, we lean toward writing data first and metadata second to avoid writing it twice. For the stated reasons, we will likely not produce .ozx files with `zarr.json` files ordered first. - **Ordering of zarr.json in central directory:** This is reasonable for discoverability, especially for the root `zarr.json` if consolidated metadata is present. For all other `zarr.json` files, and for a root `zarr.json` without consolidated metadata, this seems less relevant for us and depends on the number of file entries. Therefore in our use case we might omit it for now and introduce it later if needed. @@ -30,7 +39,7 @@ We support standardizing single-file OME-Zarr via ZIP (.ozx). In our context at - **ZIP disadvantage in performance:** Compared to a directory store, file content is not necessarily stored page-aligned. In our implementation, we observed a significant performance impact for both reading and writing when using unbuffered, page-aligned I/O. To avoid read-modify-write cycles, we allocated a separate page for each local file header and kept partially filled pages empty. We also ensured this for chunks inside shards as well as the shard index. Unfortunately, due to the local file header, this results in memory overhead, though this is acceptable when sharding is turned on and chunksize is not too small. This point could be added under the drawback section in the RFC. -- **Split archives:** Field realities sometimes require multi-volume transport. Although splitting (e.g., channels or a measurement series) into smaller datasets is often possible — and recommended for other non-splittable file formats like .czi and .ims — we see use cases where archive-level splitting would be beneficial, particularly from a user-experience perspective. However, we acknowledge that this adds complexity to implementations, and support this decision. +- **Split archives:** Field realities sometimes require multi-volume transport. Although splitting (e.g., channels or a measurement series) into smaller datasets is often possible — and recommended for other non-splittable file formats like .czi and .ims — we see use cases where archive-level splitting would be beneficial, particularly from a user-experience perspective. However, we acknowledge that this adds complexity to implementations, and support this decision. - **Thumbnails:** Applications might benefit from pre-rendered thumbnails. As there is no standardized way to store thumbnails for Zarr and OME-Zarr it might be a question if this should be a topic to be addressed by zipped OME-Zarr separately or if this is out of scope for this RFC. As an example many ZIP based formats (e.g. docx, 3mf) follow the Open Packaging Conventions to store thumbnails in a standardized way. diff --git a/rfc/9/comments/5/index.md b/rfc/9/comments/5/index.md index d8a8c64c..77a3ae77 100644 --- a/rfc/9/comments/5/index.md +++ b/rfc/9/comments/5/index.md @@ -1,14 +1,28 @@ +--- +authors: + - name: Anna Kreshuk + orcid: 0000-0003-1334-6388 + affiliation: ilastik + - name: Dominik Kutra + github: k-dominik + orcid: 0000-0003-4202-3908 + affiliation: ilastik + - name: Benedikt Best + github: btbest + orcid: 0000-0001-6965-1117 + affiliation: ilastik +date: 2026-02-05 +--- + # RFC-9: Comment 5 (rfcs:rfc9:comment5)= ## Comment authors -This comment was written by the ilastik team: +```{document-authors} -* Anna Kreshuk, https://orcid.org/0000-0003-1334-6388 -* Dominik Kutra, https://orcid.org/0000-0003-4202-3908 -* Benedikt Best, https://orcid.org/0000-0001-6965-1117 +``` ## Conflicts of interest (optional) @@ -75,8 +89,8 @@ Alternatively, one could make this clear by adding an observation like the follo ## Minor comments and questions -* The proposed new section of the specification uses the term "SHALL", which is so far not used elsewhere in the specification. Since according to IETF RFC 2119, SHALL is synonymous to MUST, and MUST is the term used in the rest of the specification, this should be replaced. -* Duplication of "the" in "The ZIP file MUST contain the the OME-Zarr’s root-level zarr.json." +- The proposed new section of the specification uses the term "SHALL", which is so far not used elsewhere in the specification. Since according to IETF RFC 2119, SHALL is synonymous to MUST, and MUST is the term used in the rest of the specification, this should be replaced. +- Duplication of "the" in "The ZIP file MUST contain the the OME-Zarr’s root-level zarr.json." ## Recommendation diff --git a/rfc/9/comments/6/index.md b/rfc/9/comments/6/index.md index 19689c6c..5f4cd195 100644 --- a/rfc/9/comments/6/index.md +++ b/rfc/9/comments/6/index.md @@ -1,10 +1,20 @@ +--- +authors: + - name: Assa Diabira + affiliation: Institut Cochin (IMAG'IC / CID), Université Paris Cité, France + github: assadiab +date: 2026-06-22 +--- + # RFC-9: Comment 6 (rfcs:rfc9:comment6)= ## Comment authors -This comment was written by: Assa Diabira, Institut Cochin (IMAG'IC / Cochin Image Database), Université Paris Cité, Paris, France. +```{document-authors} + +``` ## Conflicts of interest (optional) diff --git a/rfc/9/index.md b/rfc/9/index.md index 768d7bbf..22cdfc4f 100644 --- a/rfc/9/index.md +++ b/rfc/9/index.md @@ -1,3 +1,30 @@ +--- +authors: + - name: Jonas Windhager + github: jwindhager + affiliation: SciLifeLab / Uppsala University, Sweden + role: Corresponding Author + date: "2025-07-02" + - name: Norman Rzepka + github: normanrz + affiliation: scalable minds GmbH, Germany + role: Co-author + date: "2025-08-27" + - name: Mark Kittisopikul + github: mkitti + affiliation: HHMI Janelia, United States + role: Co-author + date: "2025-08-27" +editors: + - name: Josh Moore + github: joshmoore + affiliation: German BioImaging e.V. + role: Editor + date: "2025-11-05" +reference_pr: https://github.com/ome/ngff/pull/316 +date: 2025-07-02 +--- + # RFC-9: Zipped OME-Zarr ```{toctree} @@ -15,21 +42,9 @@ Add a specification for storing an OME-Zarr hierarchy within a ZIP archive. This RFC is currently in state `R2` (waiting on reviewers). -| Role | Name | GitHub Handle | Institution | Date | Status | -| --------- | ------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------- | ---------- | ---------------------------------------------------------------- | -| Author | Jonas Windhager | [jwindhager](https://github.com/jwindhager) | SciLifeLab / Uppsala University, Sweden | 2025-07-02 | Corresponding Author [PR](https://github.com/ome/ngff/pull/316) | -| Author | Norman Rzepka | [normanrz](https://github.com/normanrz) | scalable minds GmbH, Germany | 2025-08-27 | Co-author [PR](https://github.com/ome/ngff/pull/316) | -| Author | Mark Kittisopikul | [mkitti](https://github.com/mkitti) | HHMI Janelia, United States | 2025-08-27 | Co-author [PR](https://github.com/ome/ngff/pull/316) | -| Editor | Josh Moore | [joshmoore](https://github.com/joshmoore) | German BioImaging e.V. | 2025-11-05 | Editor | -| Reviewer | Pete Bankhead | [petebankhead](https://github.com/petebankhead) | University of Edinburgh, United Kingdom | 2026-01-26 | [Review](./reviews/1/index) | -| Reviewer | Kola Babalola, Matthew Hartley | [kbab](https://github.com/kbab), [matthewh-ebi](https://github.com/matthewh-ebi) | BioImage Archive, EMBL-EBI | 2026-01-29 | [Review](./reviews/2/index) | -| Reviewer | Curtis Rueden | [ctrueden](https://github.com/ctrueden) | University of Wisconsin-Madison, United States | 2026-01-30 | [Review](./reviews/3/index) | -| Commenter | Matt McCormick | [thewtex](https://github.com/thewtex) | Fideus Labs LLC | 2025-11-15 | [Comment](./comments/1/index) | -| Commenter | Joost de Folter | [folterj](https://github.com/folterj) | BioImaging-NL | 2025-12-03 | [Comment](./comments/2/index) | -| Commenter | Chris Barnes | [clbarnes](https://github.com/clbarnes) | German BioImaging | 2025-12-12 | [Comment](./comments/3/index) | -| Commenter | Anna Kreshuk, Dominik Kutra, Benedikt Best | [k-dominik](https://github.com/k-dominik), [btbest](https://github.com/btbest) | ilastik | 2026-01-09 | [Comment](./comments/5/index) | -| Commenter | Lenard Spiecker, Matthias Grunwald | [l-spiecker](https://github.com/l-spiecker) | Miltenyi Biotec B.V. & Co. KG | 2026-02-05 | [Comment](./comments/4/index) | -| Commenter | Assa Diabira | [assadiab](https://github.com/assadiab) | Institut Cochin (IMAG'IC / CID), Université Paris Cité, France | 2026-06-22 | [Comment](./comments/6/index) | +```{rfc-status} + +``` ## Overview @@ -166,13 +181,14 @@ The `centralDirectory` attribute MAY contain the following key: - `jsonFirst`: If `true`, this indicates that the `zarr.json` files are ordered breadth-first in the central directory and precede other content, as recommended above. This allows the hierarchical structure of the contents to be discovered without parsing the entire central directory, which could contain many entries of Zarr chunks. Implementations MAY assume that no further `zarr.json` files exist beyond the first non-`zarr.json` file if `jsonFirst` is `true`. If `jsonFirst` is omitted, the value defaults to `false`. For example, + ```json { "ome": { "version": "XX.YY", "zipFile": { "centralDirectory": { - "jsonFirst": true, + "jsonFirst": true } } } @@ -217,8 +233,6 @@ Socialization: see Prior art and references; the draft was further discussed amo - A [neuroglancer view](https://neuroglancer-demo.appspot.com/#!%7B%22dimensions%22:%7B%22x%22:%5B3.6039815346402084e-7%2C%22m%22%5D%2C%22y%22:%5B3.6039815346402084e-7%2C%22m%22%5D%2C%22z%22:%5B5.002025531914894e-7%2C%22m%22%5D%7D%2C%22position%22:%5B135%2C137%2C118%5D%2C%22crossSectionScale%22:1%2C%22projectionScale%22:512%2C%22layers%22:%5B%7B%22type%22:%22image%22%2C%22source%22:%22https://static.webknossos.org/misc/6001240.ozx%7Czip:%7Czarr3:%22%2C%22localDimensions%22:%7B%22c%27%22:%5B1%2C%22%22%5D%7D%2C%22localPosition%22:%5B0%5D%2C%22tab%22:%22source%22%2C%22opacity%22:1%2C%22blend%22:%22additive%22%2C%22shader%22:%22#uicontrol%20invlerp%20contrast%5Cn#uicontrol%20vec3%20color%20color%5Cnvoid%20main%28%29%20%7B%5Cn%20%20float%20contrast_value%20=%20contrast%28%29%3B%5Cn%20%20if%20%28VOLUME_RENDERING%29%20%7B%5Cn%20%20%20%20emitRGBA%28vec4%28color%20%2A%20contrast_value%2C%20contrast_value%29%29%3B%5Cn%20%20%7D%5Cn%20%20else%20%7B%5Cn%20%20%20%20emitRGB%28color%20%2A%20contrast_value%29%3B%5Cn%20%20%7D%5Cn%7D%5Cn%22%2C%22shaderControls%22:%7B%22contrast%22:%7B%22range%22:%5B7%2C927%5D%2C%22window%22:%5B0%2C1159%5D%7D%2C%22color%22:%22#ff0000%22%7D%2C%22volumeRenderingDepthSamples%22:256%2C%22name%22:%226001240.ozx%20c-0.5%22%7D%2C%7B%22type%22:%22image%22%2C%22source%22:%22https://static.webknossos.org/misc/6001240.ozx%7Czip:%7Czarr3:%22%2C%22localDimensions%22:%7B%22c%27%22:%5B1%2C%22%22%5D%7D%2C%22localPosition%22:%5B1%5D%2C%22tab%22:%22source%22%2C%22opacity%22:1%2C%22blend%22:%22additive%22%2C%22shader%22:%22#uicontrol%20invlerp%20contrast%5Cn#uicontrol%20vec3%20color%20color%5Cnvoid%20main%28%29%20%7B%5Cn%20%20float%20contrast_value%20=%20contrast%28%29%3B%5Cn%20%20if%20%28VOLUME_RENDERING%29%20%7B%5Cn%20%20%20%20emitRGBA%28vec4%28color%20%2A%20contrast_value%2C%20contrast_value%29%29%3B%5Cn%20%20%7D%5Cn%20%20else%20%7B%5Cn%20%20%20%20emitRGB%28color%20%2A%20contrast_value%29%3B%5Cn%20%20%7D%5Cn%7D%5Cn%22%2C%22shaderControls%22:%7B%22contrast%22:%7B%22range%22:%5B25%2C824%5D%2C%22window%22:%5B0%2C1025%5D%7D%2C%22color%22:%22#00ff00%22%7D%2C%22volumeRenderingDepthSamples%22:256%2C%22name%22:%226001240.ozx%20c0.5%22%7D%5D%2C%22selectedLayer%22:%7B%22visible%22:true%2C%22layer%22:%226001240.ozx%20c-0.5%22%7D%2C%22layout%22:%224panel-alt%22%2C%22helpPanel%22:%7B%22row%22:2%7D%2C%22settingsPanel%22:%7B%22row%22:3%7D%2C%22toolPalettes%22:%7B%22Shader%20controls%22:%7B%22side%22:%22left%22%2C%22row%22:1%2C%22query%22:%22type:shaderControl%22%7D%7D%7D) of the [generated data](https://static.webknossos.org/misc/6001240.ozx) has kindly been [made available](https://github.com/ome/ngff/pull/316#issuecomment-3302595684) by Davis Bennett. - [ozx-tck](https://github.com/clbarnes/ozx-tck) is a toolkit to validate existing .ozx files and generate valid, warning, and error test cases. - - ## Drawbacks, risks, alternatives, and unknowns Drawbacks: diff --git a/rfc/9/reviews/1/index.md b/rfc/9/reviews/1/index.md index cb081f6a..c2955dd2 100644 --- a/rfc/9/reviews/1/index.md +++ b/rfc/9/reviews/1/index.md @@ -1,10 +1,21 @@ +--- +authors: + - name: Pete Bankhead + affiliation: University of Edinburgh + github: petebankhead +date: 2026-01-26 +recommendation: accept +--- + # RFC-9: Review 1 (rfcs:rfc9:review1)= ## Comment authors -This comment was written by: Pete Bankhead, University of Edinburgh +```{document-authors} + +``` ## Conflicts of interest (optional) @@ -23,7 +34,7 @@ I am strongly in favor of standardized single-file support, which I think will m I'm not familiar enough with ZIP to understand the rationale for this recommendation or how straightforward it would be to follow. Specifically for Java, Zip files can be written with [`ZipFile`](https://docs.oracle.com/en/java/javase/25/docs/api/java.base/java/util/zip/ZipFile.html) or the [optional Zip file system module](https://docs.oracle.com/en/java/javase/25/docs/api/jdk.zipfs/module-summary.html). -I believe both support ZIP64, but I do not see an API to request that it is always used, including for smaller files. Apache Commons Compress [provides more ZIP64 control](https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.html#setUseZip64(org.apache.commons.compress.archivers.zip.Zip64Mode)), at the expense of requiring an extra dependency. +I believe both support ZIP64, but I do not see an API to request that it is always used, including for smaller files. Apache Commons Compress [provides more ZIP64 control](), at the expense of requiring an extra dependency. The source for OpenJDK's `ZipFileSystem` [mentions a `"forceZIP64End"` property](https://github.com/openjdk/jdk/blob/master/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java#L179), but this appears to be undocumented. @@ -44,7 +55,6 @@ Under **User experience-related challenges**: The 'preview' aspect makes it tempting to want to embed a thumbnail, which could be supported by some applications or operating system plugins. Should this be explicitly forbidden / discouraged / encouraged in a standard way? - ## Recommendation Adopt, with some clarification around ZIP64 to ensure the recommendation is justified and readily achievable across most relevant programming languages. diff --git a/rfc/9/reviews/2/index.md b/rfc/9/reviews/2/index.md index 23f1e9d0..441f5551 100644 --- a/rfc/9/reviews/2/index.md +++ b/rfc/9/reviews/2/index.md @@ -1,10 +1,24 @@ +--- +authors: + - name: Kola Babalola + affiliation: BioImage Archive, EMBL-EBI + github: kbab + - name: Matthew Hartley + affiliation: BioImage Archive, EMBL-EBI + github: matthewh-ebi +date: 2026-01-29 +recommendation: minor_changes +--- + # RFC-9: Review 2 (rfcs:rfc9:review2)= ## Review authors -Kola Babalola, Matthew Hartley, the BioImage Archive, EMBL-EBI. +```{document-authors} + +``` ## Conflicts of interest @@ -17,10 +31,12 @@ This RFC represents highly valuable work, and we are in favour of adoption. The ## Significant comments and questions ### Versions of OME Zarr + The RFC only applies to OME-Zarrs with metadata in zarr.json (not .zattr / .zarray) which implies at least OME Zarr v0.5. Is it worth explicitly mentioning this in the RFC? This might make sense in the ‘Compatibility’ section. ### Recommendations on maximum archive size -The RFC in places alludes to the size of OME Zarrs and explicitly mentions 4GiB in the recommendation to use ZIP64 in the Proposal section. In the User experience-related challenges subsection of the Background section “a few small images” is mentioned. Additionally, the recommendations prohibit multi-volume archives. + +The RFC in places alludes to the size of OME Zarrs and explicitly mentions 4GiB in the recommendation to use ZIP64 in the Proposal section. In the User experience-related challenges subsection of the Background section “a few small images” is mentioned. Additionally, the recommendations prohibit multi-volume archives. However, no explicit guidance is given on size limitations associated with the single file format. Presumably the upper limit of size of a single file on filesystems is a hard limit, but there are practical limitations to the storage and transfer of extremely large files below the filesystem-imposed limit. Since the RFC explicitly prohibits multi-part archives, it would be useful to include a brief discussion of the limitations this imposes, and guidance for users with OME-Zarrs above this size. diff --git a/rfc/9/reviews/3/index.md b/rfc/9/reviews/3/index.md index 6227b45f..46565a2d 100644 --- a/rfc/9/reviews/3/index.md +++ b/rfc/9/reviews/3/index.md @@ -1,10 +1,21 @@ +--- +authors: + - name: Curtis Rueden + affiliation: University of Wisconsin-Madison + github: ctrueden +date: 2026-01-30 +recommendation: major_changes +--- + # RFC-9: Review 3 (rfcs:rfc9:review3)= ## Review authors -Curtis Rueden, University of Wisconsin-Madison. +```{document-authors} + +``` ## Conflicts of interest @@ -27,6 +38,7 @@ The proposal should articulate the technical requirements of a single-file OME-Z #### Conventional use cases The phrase "conventional use cases" is frequently used, but not fully defined. Examples are given: + - Reasonably small images stored on the local desktop file system. - associate an OME-Zarr file type with their favorite image viewer (“double click” functionality) - effortlessly use their OME-Zarr images with existing file-centric tooling @@ -38,6 +50,7 @@ But a clear bullet-list of community-gathered use cases would be clarifying. #### Streaming There is only one mention of streaming. Is ozx intended to be streamable from a remote source? + - If not: this should be stated explicitly in the RFC that streamability is a non-goal. - Or if so: ZIP is not ideal out of the box, due to the central directory being at the end of the file. - From the beginning of the ZIP, you don't know how many ZIP entries you are going to receive. @@ -49,7 +62,7 @@ There is only one mention of streaming. Is ozx intended to be streamable from a A core use case of ZIP in general is the ability to modify the archive, adding and removing files after initial creation. Are the contents of a zipped OME-Zarr file intended to be mutable? Given that three of the five points in "Disadvantages of the ZIP archive file format" are about mutability concerns, I will assume yes -- although my tentative recommendation would actually be to disallow ZIP-specific mutation actions on .ozx files in favor of simply rewriting them cleanly when changes are needed, similar to most other image file formats. Of course, it ultimately depends on the community requirements around OME-Zarr, but for finite mutation-oriented scenarios, one can imagine extracting the ZIP contents, operating on the unzipped OME-Zarr directory structure, and then zipping it again after mutations are complete. -If zipped OME-Zarr *is* intended to be mutable, that should be explicitly stated as a requirement, and the ramifications of that decision should be discussed on more depth. For example, mutation of data or metadata may necessitate corresponding modifications to the zarr.json entry. According to the ZIP specification, modifying zarr.json in this way will orphan the entry at the head of the file and append the revised version to the tail, spoiling the file's "zarr.json files come first" recommendation, and also potentially impacting streamability (see also "Streaming" above). Small adjustments to the proposal could potentially mitigate these issues: e.g., the addition of an optional fixed-sized header file as first entry with directory tree mutations directly overwriting those header bytes in place, or the inclusion of trailing padding to all ome.zarr entries like how the ID3v2 tag format supports a padded leading header to allow metadata room to grow. Of course, such mitigations also complicate the mutation operation logic, since general-purpose ZIP libraries do not normally perform such bookkeeping. +If zipped OME-Zarr _is_ intended to be mutable, that should be explicitly stated as a requirement, and the ramifications of that decision should be discussed on more depth. For example, mutation of data or metadata may necessitate corresponding modifications to the zarr.json entry. According to the ZIP specification, modifying zarr.json in this way will orphan the entry at the head of the file and append the revised version to the tail, spoiling the file's "zarr.json files come first" recommendation, and also potentially impacting streamability (see also "Streaming" above). Small adjustments to the proposal could potentially mitigate these issues: e.g., the addition of an optional fixed-sized header file as first entry with directory tree mutations directly overwriting those header bytes in place, or the inclusion of trailing padding to all ome.zarr entries like how the ID3v2 tag format supports a padded leading header to allow metadata room to grow. Of course, such mitigations also complicate the mutation operation logic, since general-purpose ZIP libraries do not normally perform such bookkeeping. #### Encryption @@ -71,10 +84,10 @@ Is fast performance a goal of this format? If so, how fast? In my view, efficien It would be good for the RFC to break this down more explicitly, with a short discussion of each of ZIP's relevant advantages. That is: how good are each of these advantages in practice for OME Zarr? For example: -* Is it desirable/intended that users can feed a .ozx file to a general-purpose unzipping tool to produce a normal (non-zipped) ome-zarr dataset on disk? -* Is it desirable/intended that zipped ome-zarr files have integrity checksums (CRC32) for file contents? -* Is it desirable/intended that zipped ome-zarr files can be modified after initial creation? (See also "Mutability" above.) -* How +- Is it desirable/intended that users can feed a .ozx file to a general-purpose unzipping tool to produce a normal (non-zipped) ome-zarr dataset on disk? +- Is it desirable/intended that zipped ome-zarr files have integrity checksums (CRC32) for file contents? +- Is it desirable/intended that zipped ome-zarr files can be modified after initial creation? (See also "Mutability" above.) +- How > Simplicity @@ -87,6 +100,7 @@ If simplicity is a key requirement, ozx files might be better served defining th > Widespread adoption Why does this matter for OME-Zarr? + - As a binary file, the .ozx file will be opaque to most users. - Savvy users could work with the file as a ZIP file, using ZIP-compatible tools. - But should they? Any naive modification to the ozx file would corrupt it, as discussed above. @@ -99,6 +113,7 @@ Why does this matter for OME-Zarr? > on-board tooling of various operating systems When would this tooling come into play for users? + - They could rename the file from .ozx to .zip and then use on-board tooling to extract the contents. - Any other benefits? Reconstruction of damaged/incomplete archives? Would users do this often? @@ -117,10 +132,11 @@ For future-proofing, I suggest generalizing this field beyond only a boolean. It > This allows the hierarchical structure of the contents to be discovered without parsing the entire central directory, which could contain many entries of Zarr chunks. It would be helpful for the proposal to give an example of central directory size vs zarr.json size, to give a sense of performance gains here. + - For the example ozx file? - For a larger file, e.g. tubhiswt.ome.tif converted to ozx with a reasonable sharding structure? -Regardless: knowing the hierarchical structure unfortunately does not help with random access, due to compressed block size variability -- in what scenarios *does* it help to discover that structure up front? +Regardless: knowing the hierarchical structure unfortunately does not help with random access, due to compressed block size variability -- in what scenarios _does_ it help to discover that structure up front? ## Minor comments and questions @@ -157,6 +173,7 @@ And less crucial but still beneficial to the proposal: My specific recommendation would be to add language like the following to the RFC: > The following ZIP features MUST NOT be used in ozx files: +> > - Encryption (any method) > - Compression at ZIP level (STORE method only) > - Extra fields containing non-Zarr data @@ -176,7 +193,7 @@ Finally, as food for thought, here is my devil's-advocate pitch for a minimal bi - Existence of such files will necessitate demand for image software to support them. -- To support them while also supporting recommendation-compliant ozx files efficiently, ozx readers will need to implement (at least) two branches of case logic: one case achieving good performance for the well-performing ozx files, and another more general case achieving support *at all* for the noncompliant files. +- To support them while also supporting recommendation-compliant ozx files efficiently, ozx readers will need to implement (at least) two branches of case logic: one case achieving good performance for the well-performing ozx files, and another more general case achieving support _at all_ for the noncompliant files. - Such case logic will complexify ome-zarr reader implementations to such an extent that the gains in simplicity from using ZIP become outweighed by the losses (code complexity, maintainability) incurred by the case logic.