From a3ea9be9a4d42dc0558e9b40f13f3160ca3a6918 Mon Sep 17 00:00:00 2001
From: Jayashanker Padishala <jayashankar.pjs@gmail.com>
Date: Thu, 23 Jul 2026 03:01:49 -0700
Subject: [PATCH] feat: support URLs in forecast submissions

Add an optional `url` field to forecast submission files so organizations
can provide their website directly in the submission:

- read_forecast_file() validates the field when present (must be http(s))
  and rejects non-anonymous submissions without one when the forecast due
  date is on or after URL_REQUIRED_AS_OF_DUE_DATE (2026-08-01), making the
  field required going forward while older forecast sets keep processing
  unchanged.
- Anonymous submissions (organization "Anonymous"/"Anonymous N") are
  exempt.
- The field is passed through to the processed forecast set via
  FORECAST_FILE_METADATA_FIELDS; it is not displayed on the website.

Closes #254
---
 src/orchestration/_io.py                      | 38 ++++++++++++
 src/orchestration/func_resolve/main.py        |  1 +
 .../orchestration/test_forecast_file_io.py    | 59 +++++++++++++++++++
 3 files changed, 98 insertions(+)

diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py
index c7d6cc8a..92a349bd 100644
--- a/src/orchestration/_io.py
+++ b/src/orchestration/_io.py
@@ -37,6 +37,14 @@
 )
 QUESTION_SET_READ_TIMEOUT_SECONDS = 30
 
+# Submissions with a forecast due date on or after this date must provide a `url`
+# field with the submitting organization's website, unless the submission is
+# anonymous. Forecast sets due before this date predate the requirement and are
+# still processed without one.
+URL_REQUIRED_AS_OF_DUE_DATE = "2026-08-01"
+
+ANONYMOUS_ORGANIZATION_RE = re.compile(r"^anonymous(\s+\d+)?$", re.IGNORECASE)
+
 
 # ---------------------------------------------------------------------------
 # Question bank loading
@@ -457,6 +465,10 @@ def get_valid_forecast_files_and_dates(
 def read_forecast_file(filename: str, f: TextIO | None = None) -> dict | None:
     """Read a forecast JSON file and validate its content.
 
+    The optional `url` field, when present, must be an http(s) URL. It is required
+    for non-anonymous submissions with a forecast due date on or after
+    URL_REQUIRED_AS_OF_DUE_DATE.
+
     Args:
         filename: Path to the forecast JSON file.
         f: Open file handle. If None, filename will be opened.
@@ -493,6 +505,32 @@ def read_forecast_file(filename: str, f: TextIO | None = None) -> dict | None:
         )
         return None
 
+    url = data.get("url")
+    if url is not None and (
+        not isinstance(url, str) or not url.startswith(("http://", "https://"))
+    ):
+        logger.error(
+            colored(
+                f"Problem processing {filename}. `url` must be an http(s) URL, got: {url}.",
+                "yellow",
+            )
+        )
+        return None
+
+    if (
+        not url
+        and forecast_due_date >= URL_REQUIRED_AS_OF_DUE_DATE
+        and not ANONYMOUS_ORGANIZATION_RE.match(organization.strip())
+    ):
+        logger.error(
+            colored(
+                f"Problem processing {filename}. Missing required field `url`; it is "
+                "required for non-anonymous submissions.",
+                "yellow",
+            )
+        )
+        return None
+
     df = pd.DataFrame(forecasts)
     if df.empty:
         logger.error(
diff --git a/src/orchestration/func_resolve/main.py b/src/orchestration/func_resolve/main.py
index 2a49badb..7a5c6771 100644
--- a/src/orchestration/func_resolve/main.py
+++ b/src/orchestration/func_resolve/main.py
@@ -42,6 +42,7 @@
     "model_run_slug",
     "forecast_variant_key",
     "uses_freeze_values",
+    "url",
 ]
 
 
diff --git a/src/tests/orchestration/test_forecast_file_io.py b/src/tests/orchestration/test_forecast_file_io.py
index 2d70c3db..df78cc97 100644
--- a/src/tests/orchestration/test_forecast_file_io.py
+++ b/src/tests/orchestration/test_forecast_file_io.py
@@ -1,11 +1,70 @@
 """Tests for processed forecast file IO."""
 
+import io
 import json
 
 from helpers import env
 from orchestration import _io
 
 
+def _forecast_file_handle(**overrides) -> io.StringIO:
+    """Return a file handle for a forecast submission with sensible defaults."""
+    data = {
+        "organization": "Test Org",
+        "model": "Test Model",
+        "model_organization": "Test Model Org",
+        "question_set": "2026-09-01-llm.json",
+        "forecasts": [{"id": "q1", "forecast": 0.5}],
+    }
+    data.update(overrides)
+    data = {k: v for k, v in data.items() if v is not None}
+    return io.StringIO(json.dumps(data))
+
+
+def test_read_forecast_file_passes_url_through():
+    data = _io.read_forecast_file(
+        "forecast.json",
+        f=_forecast_file_handle(url="https://example.org"),
+    )
+
+    assert data is not None
+    assert data["url"] == "https://example.org"
+
+
+def test_read_forecast_file_requires_url_for_new_non_anonymous_submissions():
+    data = _io.read_forecast_file("forecast.json", f=_forecast_file_handle())
+
+    assert data is None
+
+
+def test_read_forecast_file_allows_missing_url_for_anonymous_submissions():
+    data = _io.read_forecast_file(
+        "forecast.json",
+        f=_forecast_file_handle(organization="Anonymous 12"),
+    )
+
+    assert data is not None
+    assert "url" not in data
+
+
+def test_read_forecast_file_allows_missing_url_for_old_forecast_sets():
+    data = _io.read_forecast_file(
+        "forecast.json",
+        f=_forecast_file_handle(question_set="2026-05-24-llm.json"),
+    )
+
+    assert data is not None
+
+
+def test_read_forecast_file_rejects_non_http_url():
+    data = _io.read_forecast_file(
+        "forecast.json",
+        f=_forecast_file_handle(url="example.org"),
+    )
+
+    assert data is None
+
+
 def test_valid_forecast_files_excludes_nested_test_files(monkeypatch):
     """Do not include date-folder test forecast files in leaderboard inputs."""
     monkeypatch.setattr(