From d7959b2e4ae824ede1dbc73cdc973ea91f658943 Mon Sep 17 00:00:00 2001 From: Houtan Bastani Date: Mon, 3 Aug 2026 12:45:56 +0000 Subject: [PATCH 1/2] fix: remove infer as a source Infer/RFI closed on Aug 1, 2026 and the API is no longer accessible. Introduce `run_fetch` and `run_update` fields in _metadata.py, setting `run_fetch` to `false` for Infer. Leave a stub for updating Infer source in a second step using an LLM to resolve questions that have not yet resolved. This will be done in a second step. No longer sample Infer questions in the question set. Closes #274 --- Makefile | 5 +- src/_schemas.py | 8 - src/helpers/infer.py | 6 - src/helpers/keys.py | 1 - src/helpers/question_curation.py | 12 +- src/leaderboard/main.py | 16 +- src/metadata/validate_questions/main.py | 1 - src/nightly_update_workflow/manager/main.py | 17 +- src/nightly_update_workflow/worker/main.py | 30 +- src/orchestration/_io.py | 6 +- src/orchestration/func_infer_fetch/Makefile | 37 -- src/orchestration/func_infer_fetch/main.py | 34 -- .../func_infer_fetch/requirements.txt | 9 - src/orchestration/func_infer_update/main.py | 24 +- .../func_infer_update/requirements.txt | 4 - src/sources/_metadata.py | 65 +- src/sources/infer.py | 479 +-------------- src/tests/_module_stubs.py | 95 +++ src/tests/conftest.py | 106 ---- src/tests/leaderboard/_leaderboard_import.py | 50 ++ src/tests/leaderboard/conftest.py | 14 + src/tests/leaderboard/test_llm_identities.py | 107 +--- .../test_llm_identity_release_dates.py | 12 +- src/tests/leaderboard/test_source_masks.py | 17 + .../test_load_question_bank_staleness.py | 39 ++ src/tests/test_infer.py | 563 +----------------- src/tests/test_nightly_worker_fetch_list.py | 95 +++ src/tests/test_question_curation_sources.py | 13 + src/tests/test_source_scheduling.py | 51 ++ src/www.forecastbench.org/about/index.md | 2 +- 30 files changed, 554 insertions(+), 1364 deletions(-) delete mode 100644 src/helpers/infer.py delete mode 100644 src/orchestration/func_infer_fetch/Makefile delete mode 100644 src/orchestration/func_infer_fetch/main.py delete mode 100644 src/orchestration/func_infer_fetch/requirements.txt create mode 100644 src/tests/_module_stubs.py create mode 100644 src/tests/leaderboard/_leaderboard_import.py create mode 100644 src/tests/leaderboard/conftest.py create mode 100644 src/tests/leaderboard/test_source_masks.py create mode 100644 src/tests/orchestration/test_load_question_bank_staleness.py create mode 100644 src/tests/test_nightly_worker_fetch_list.py create mode 100644 src/tests/test_question_curation_sources.py create mode 100644 src/tests/test_source_scheduling.py diff --git a/Makefile b/Makefile index c29963fb..f4dc7601 100644 --- a/Makefile +++ b/Makefile @@ -157,10 +157,7 @@ metaculus-fetch: metaculus-update-questions: $(MAKE) -C src/orchestration/func_metaculus_update || echo "* $@" >> $(MAKE_FAILURE_LOG) -infer: infer-fetch infer-update-questions - -infer-fetch: - $(MAKE) -C src/orchestration/func_infer_fetch || echo "* $@" >> $(MAKE_FAILURE_LOG) +infer: infer-update-questions infer-update-questions: $(MAKE) -C src/orchestration/func_infer_update || echo "* $@" >> $(MAKE_FAILURE_LOG) diff --git a/src/_schemas.py b/src/_schemas.py index 8fdc161b..e380d597 100644 --- a/src/_schemas.py +++ b/src/_schemas.py @@ -100,14 +100,6 @@ class Config: coerce = False -class InferFetchFrame(QuestionFrame): - """Output of InferSource.fetch(). QuestionFrame plus transient fields for update().""" - - fetch_datetime: Series[str] - probability: Series[object] = pa.Field(nullable=True) - nullify_question: Series[bool] - - class PolymarketFetchFrame(QuestionFrame): """Output of PolymarketSource.fetch(). QuestionFrame plus transient fields for update().""" diff --git a/src/helpers/infer.py b/src/helpers/infer.py deleted file mode 100644 index cc536d17..00000000 --- a/src/helpers/infer.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Infer-specific variables. Delegates to sources._metadata.""" - -from sources._metadata import SOURCE_METADATA - -SOURCE_INTRO = SOURCE_METADATA["infer"]["source_intro"] -RESOLUTION_CRITERIA = SOURCE_METADATA["infer"]["resolution_criteria"] diff --git a/src/helpers/keys.py b/src/helpers/keys.py index 0e9c162a..95a9b652 100644 --- a/src/helpers/keys.py +++ b/src/helpers/keys.py @@ -35,7 +35,6 @@ def get_secret_that_may_not_exist(secret_name, version_id="latest"): # QUESTION MARKET SOURCES API_KEY_METACULUS = get_secret(secret_name="API_KEY_METACULUS") API_KEY_POLYMARKET = get_secret("API_KEY_POLYMARKET") -API_KEY_INFER = get_secret("API_KEY_INFER") # WORKFLOW BOT API_SLACK_BOT_NOTIFICATION = get_secret(secret_name="API_SLACK_BOT_NOTIFICATION") diff --git a/src/helpers/question_curation.py b/src/helpers/question_curation.py index acf6367f..93fedd88 100644 --- a/src/helpers/question_curation.py +++ b/src/helpers/question_curation.py @@ -9,7 +9,6 @@ dates, dbnomics, fred, - infer, manifold, metaculus, polymarket, @@ -28,8 +27,9 @@ FREEZE_QUESTION_MARKET_SOURCES = { - # If market sources are ever removed, the key must be added to MARKET_SOURCES in - # `helpers/resolution.py` as the resolution code needs all old market sources. + # The market sources we sample questions from. Dropping a source here stops sampling it + # without affecting resolution: `helpers/resolution.py` takes its source lists from + # `sources/_metadata.py`, which holds every market source we've ever published questions for. "manifold": { "name": "Manifold", "source_intro": manifold.SOURCE_INTRO, @@ -40,11 +40,6 @@ "source_intro": metaculus.SOURCE_INTRO, "resolution_criteria": metaculus.RESOLUTION_CRITERIA, }, - "infer": { - "name": "INFER", - "source_intro": infer.SOURCE_INTRO, - "resolution_criteria": infer.RESOLUTION_CRITERIA, - }, "polymarket": { "name": "Polymarket", "source_intro": polymarket.SOURCE_INTRO, @@ -84,7 +79,6 @@ DATA_SOURCES = list(FREEZE_QUESTION_DATA_SOURCES.keys()) MARKET_SOURCES = list(FREEZE_QUESTION_MARKET_SOURCES.keys()) -ALL_SOURCES = DATA_SOURCES + MARKET_SOURCES FREEZE_WINDOW_IN_DAYS = 10 diff --git a/src/leaderboard/main.py b/src/leaderboard/main.py index 69a6c8d9..f7ebc9ac 100644 --- a/src/leaderboard/main.py +++ b/src/leaderboard/main.py @@ -34,7 +34,6 @@ decorator, env, git, - question_curation, resolution, slack, ) @@ -42,6 +41,7 @@ ALL_FORECAST_VARIANT_KEYS_WITH_CONTEXT, ALL_FORECAST_VARIANT_KEYS_WITHOUT_CONTEXT, ) +from sources import DATASET_SOURCE_NAMES, MARKET_SOURCE_NAMES # noqa: E402 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -382,15 +382,15 @@ def download_question_set_save_in_cache( def get_dataset_mask(df: pd.DataFrame) -> pd.Series: - """Generate boolean masks for market questions. + """Generate boolean masks for dataset questions. Args: df (pd.DataFrame): The forecast set. Returns: - pd.Series: questions from DATA_SOURCES + pd.Series: questions from DATASET_SOURCE_NAMES """ - return df["source"].isin(question_curation.DATA_SOURCES) + return df["source"].isin(DATASET_SOURCE_NAMES) def get_market_mask(df: pd.DataFrame) -> pd.Series: @@ -400,9 +400,9 @@ def get_market_mask(df: pd.DataFrame) -> pd.Series: df (pd.DataFrame): The forecast set. Returns: - pd.Series: all questions from MARKET_SOURCES. + pd.Series: all questions from MARKET_SOURCE_NAMES. """ - return df["source"].isin(question_curation.MARKET_SOURCES) + return df["source"].isin(MARKET_SOURCE_NAMES) def get_masks(df: pd.DataFrame) -> Dict[str, pd.Series]: @@ -413,8 +413,8 @@ def get_masks(df: pd.DataFrame) -> Dict[str, pd.Series]: Returns: Dict[str, pd.Series]: Mapping of mask names to boolean Series: - - "dataset": questions from DATA_SOURCES that are resolved. - - "market": all questions from MARKET_SOURCES. + - "dataset": questions from DATASET_SOURCE_NAMES that are resolved. + - "market": all questions from MARKET_SOURCE_NAMES. - "market_resolved": market questions that are resolved. - "market_unresolved": market questions that are unresolved. """ diff --git a/src/metadata/validate_questions/main.py b/src/metadata/validate_questions/main.py index c1035828..1da8b739 100644 --- a/src/metadata/validate_questions/main.py +++ b/src/metadata/validate_questions/main.py @@ -117,7 +117,6 @@ def driver(_): dfmeta = dfmeta[dfmeta["source"] != source] if source in question_curation.DATA_SOURCES + [ - "infer", "metaculus", ]: dfq["valid_question"] = True diff --git a/src/nightly_update_workflow/manager/main.py b/src/nightly_update_workflow/manager/main.py index c6835a6f..72a21cb3 100644 --- a/src/nightly_update_workflow/manager/main.py +++ b/src/nightly_update_workflow/manager/main.py @@ -9,11 +9,22 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) from helpers import cloud_run, constants, env, question_curation, slack # noqa: E402 +from sources import ALL_SOURCE_NAMES # noqa: E402 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +def get_fetch_and_update_task_count() -> int: + """Return the number of worker tasks needed to fetch and update every source. + + The worker runs one task per source, so this must cover the longest job list the worker can + produce on any day. Sized from the source metadata rather than the question-curation lists, + which only cover the sources we still sample questions from. + """ + return len(ALL_SOURCE_NAMES) + + def call_worker(dict_to_use, task_count, timeout=cloud_run.timeout_1h): """Make main() easier to read.""" return cloud_run.call_worker( @@ -33,7 +44,7 @@ def summarize_question_bank(): lines=True, )[["id", "source", "valid_question"]] df = pd.DataFrame() - for source in sorted(question_curation.ALL_SOURCES): + for source in ALL_SOURCE_NAMES: logger.info(f"downloading {source} question file.") dfq = pd.read_json( f"gs://{env.QUESTION_BANK_BUCKET}/{source}_questions.jsonl", @@ -105,9 +116,7 @@ def main(): dict_to_use = "fetch_and_update" timeout_fetch_and_update = cloud_run.timeout_1h * 6 - task_count = len(question_curation.FREEZE_QUESTION_DATA_SOURCES) + len( - question_curation.FREEZE_QUESTION_MARKET_SOURCES - ) + task_count = get_fetch_and_update_task_count() operation = call_worker( dict_to_use=dict_to_use, task_count=task_count, diff --git a/src/nightly_update_workflow/worker/main.py b/src/nightly_update_workflow/worker/main.py index a0746b4f..030291fc 100644 --- a/src/nightly_update_workflow/worker/main.py +++ b/src/nightly_update_workflow/worker/main.py @@ -5,6 +5,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) from helpers import cloud_run, dates, question_curation # noqa: E402 +from sources import ALL_SOURCE_NAMES, SOURCE_METADATA # noqa: E402 metadata = [ [ @@ -88,29 +89,24 @@ def get_publish_question_set_make_llm_baseline(): def get_fetch_and_update(): """Dynamically add acled to list of functions to call dending on the day of the week.""" - sources = [ - "dbnomics", - "fred", - "infer", - "manifold", - "metaculus", - "polymarket", - "wikipedia", - "yfinance", - ] + sources = [source for source in ALL_SOURCE_NAMES if source != "acled"] day_of_week = dates.get_datetime_today().strftime("%A") if day_of_week in ["Wednesday"]: # Fetch ACLED data on Wednesdays. See Issue #115. sources += [ "acled", ] - return [ - [ - (f"func-data-{source}-fetch", True, cloud_run.timeout_1h * 3, 1), - (f"func-data-{source}-update-questions", True, cloud_run.timeout_1h * 3, 1), - ] - for source in sources - ] + jobs = [] + for source in sources: + group = [] + if SOURCE_METADATA[source]["run_fetch"]: + group.append((f"func-data-{source}-fetch", True, cloud_run.timeout_1h * 3, 1)) + if SOURCE_METADATA[source]["run_update"]: + group.append( + (f"func-data-{source}-update-questions", True, cloud_run.timeout_1h * 3, 1) + ) + jobs.append(group) + return jobs def sequential_cloud_run_jobs(functions_to_call): diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index c7d6cc8a..44ba17f8 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -25,7 +25,7 @@ from _schemas import AcledResolutionFrame, QuestionFrame, ResolutionFrame from helpers import data_utils, dates, env from helpers.run_mode import RunMode -from sources import ALL_SOURCE_NAMES, MARKET_SOURCE_NAMES +from sources import ALL_SOURCE_NAMES, MARKET_SOURCE_NAMES, SOURCE_METADATA from sources._base import BaseSource logging.basicConfig(level=logging.INFO) @@ -110,6 +110,10 @@ def load_question_bank(sources_to_get: list[str] | None = None) -> QuestionBank: # Check market dfq files are up-to-date any_out_of_date_dfq = False for source in MARKET_SOURCE_NAMES: + if not SOURCE_METADATA[source]["run_fetch"]: + # Sources we no longer fetch have an intentionally frozen dfq and hence will be + # out of date. + continue last_updated_dfq = data_utils.get_last_modified_time_of_dfq_from_cloud_storage(source) any_out_of_date_dfq |= last_updated_dfq is None or last_updated_dfq.date() < today if last_updated_dfq is None or last_updated_dfq.date() < today: diff --git a/src/orchestration/func_infer_fetch/Makefile b/src/orchestration/func_infer_fetch/Makefile deleted file mode 100644 index 7f6eb853..00000000 --- a/src/orchestration/func_infer_fetch/Makefile +++ /dev/null @@ -1,37 +0,0 @@ -all : - $(MAKE) clean - $(MAKE) deploy - -.PHONY : all clean deploy - -UPLOAD_DIR = upload -ROOT_DIR ?= $(abspath ../../..)/ -include $(ROOT_DIR)orchestration_upload.mk - -.gcloudignore: - cp -r $(ROOT_DIR)src/helpers/.gcloudignore . - -Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template - sed \ - -e 's/REGION/$(CLOUD_DEPLOY_REGION)/g' \ - -e 's/STACK/google-22-full/g' \ - -e 's/PYTHON_VERSION/python312/g' \ - $< > Dockerfile - -deploy : main.py .gcloudignore requirements.txt Dockerfile - $(stage-orchestration-upload) - gcloud run jobs deploy \ - func-data-infer-fetch \ - --project $(CLOUD_PROJECT) \ - --region $(CLOUD_DEPLOY_REGION) \ - --tasks 1 \ - --parallelism 1 \ - --task-timeout 540s \ - --memory 512Mi \ - --max-retries 0 \ - --service-account $(QUESTION_BANK_BUCKET_SERVICE_ACCOUNT) \ - --set-env-vars $(DEFAULT_CLOUD_FUNCTION_ENV_VARS) \ - --source $(UPLOAD_DIR) - -clean : - rm -rf $(UPLOAD_DIR) .gcloudignore Dockerfile diff --git a/src/orchestration/func_infer_fetch/main.py b/src/orchestration/func_infer_fetch/main.py deleted file mode 100644 index 968feb87..00000000 --- a/src/orchestration/func_infer_fetch/main.py +++ /dev/null @@ -1,34 +0,0 @@ -"""INFER fetch entry point.""" - -from __future__ import annotations - -import logging -from typing import Any - -from helpers import data_utils, decorator, keys -from orchestration import _source_io -from sources.infer import InferSource - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -SOURCE = "infer" - - -@decorator.log_runtime -def driver(_: Any) -> None: - """Fetch INFER questions and upload to question bank.""" - source = InferSource() - source.api_key = keys.API_KEY_INFER - - dfq = data_utils.get_data_from_cloud_storage(SOURCE, return_question_data=True) - existing_resolution_ids = _source_io.list_existing_resolution_ids(SOURCE) - - dff = source.fetch(dfq=dfq, existing_resolution_ids=existing_resolution_ids) - - _source_io.write_fetch_output(SOURCE, dff) - logger.info("Done.") - - -if __name__ == "__main__": - driver(None) diff --git a/src/orchestration/func_infer_fetch/requirements.txt b/src/orchestration/func_infer_fetch/requirements.txt deleted file mode 100644 index 37337e71..00000000 --- a/src/orchestration/func_infer_fetch/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -google-cloud-storage -google-cloud-secret-manager -pandas>=2.2.2,<3.0 -pandera -termcolor -requests -certifi -backoff -numpy diff --git a/src/orchestration/func_infer_update/main.py b/src/orchestration/func_infer_update/main.py index 0189a413..eb60732b 100644 --- a/src/orchestration/func_infer_update/main.py +++ b/src/orchestration/func_infer_update/main.py @@ -1,11 +1,9 @@ """INFER update entry point.""" -from __future__ import annotations - import logging from typing import Any -from helpers import data_utils, decorator, keys +from helpers import data_utils, decorator from orchestration import _source_io from sources.infer import InferSource @@ -17,22 +15,18 @@ @decorator.log_runtime def driver(_: Any) -> None: - """Update INFER questions and resolution files.""" - source = InferSource() - source.api_key = keys.API_KEY_INFER + """Run the (currently no-op) update for INFER. - dfq, dff = data_utils.get_data_from_cloud_storage( - SOURCE, return_question_data=True, return_fetch_data=True - ) - existing_resolution_files = _source_io.load_existing_resolution_files( - SOURCE, ids=dff["id"].astype(str).tolist() - ) + INFER is no longer fetched, so this does not read a fetch file. Until INFER questions + are resolved by LLM or by hand, ``update`` returns no changes and nothing is uploaded. + """ + source = InferSource() + dfq = data_utils.get_data_from_cloud_storage(SOURCE, return_question_data=True) - result = source.update(dfq, dff, existing_resolution_files=existing_resolution_files) + result = source.update(dfq) - logger.info("Uploading to GCP...") - data_utils.upload_questions(result.dfq, SOURCE) if result.resolution_files: + logger.info("Uploading resolution files to GCP...") _source_io.upload_resolution_files(SOURCE, result.resolution_files) logger.info("Done.") diff --git a/src/orchestration/func_infer_update/requirements.txt b/src/orchestration/func_infer_update/requirements.txt index 37337e71..69b45e86 100644 --- a/src/orchestration/func_infer_update/requirements.txt +++ b/src/orchestration/func_infer_update/requirements.txt @@ -2,8 +2,4 @@ google-cloud-storage google-cloud-secret-manager pandas>=2.2.2,<3.0 pandera -termcolor -requests -certifi -backoff numpy diff --git a/src/sources/_metadata.py b/src/sources/_metadata.py index 91ca9336..0a8bfc92 100644 --- a/src/sources/_metadata.py +++ b/src/sources/_metadata.py @@ -63,7 +63,62 @@ "resolve as 'Yes'." ), "resolution_criteria": "Resolves to the outcome of the question found at {url}.", - "nullified_questions": [], + # INFER became unavailable on Aug 1, 2026. It shut down before these questions resolved, so + # their resolution files are frozen at the last values we fetched. Nullifying them makes + # explicit that these questions are not currently resolvable and that their market values + # are no longer updated. Delete a line once its question has been resolved, by hand or by + # LLM. + "nullified_questions": [ + NullifiedQuestion(id=nid, nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE) + for nid in sorted( + { + "1385", + "1401", + "1432", + "1451", + "1461", + "1473", + "1474", + "1509", + "1510", + "1511", + "1513", + "1515", + "1519", + "1525", + "1557", + "1611", + "1612", + "1613", + "1614", + "1615", + "1617", + "1644", + "1645", + "1659", + "1661", + "1677", + "1678", + "1679", + "1700", + "1701", + "1702", + "1705", + "1706", + "1728", + "1761", + "1765", + "1766", + "1773", + "1774", + "1775", + } + ) + ], + # INFER / The RAND Forecasting Initiative shut down, so there is nothing left to fetch. + # `run_update` is still True, allowing for LLM resolution of unresolved questions to be + # implemented in a second pass. + "run_fetch": False, }, "manifold": { "source_type": SourceType.MARKET, @@ -382,3 +437,11 @@ MARKET_SOURCE_NAMES = sorted( name for name, m in SOURCE_METADATA.items() if m["source_type"] == SourceType.MARKET ) + +# Which nightly jobs a source is scheduled for. Sources opt out by setting these to False, so an +# entry that says nothing about them is fetched and updated, and consumers can index them +# directly rather than guessing a fallback. +_SCHEDULING_DEFAULTS = {"run_fetch": True, "run_update": True} +for _meta in SOURCE_METADATA.values(): + for _key, _default in _SCHEDULING_DEFAULTS.items(): + _meta.setdefault(_key, _default) diff --git a/src/sources/infer.py b/src/sources/infer.py index 1a8826e9..4eda894e 100644 --- a/src/sources/infer.py +++ b/src/sources/infer.py @@ -1,485 +1,40 @@ -"""INFER question source.""" +"""INFER question source (upstream no longer available).""" -from __future__ import annotations - -import logging -import time -from datetime import timedelta, timezone from typing import Any, ClassVar -import backoff -import certifi -import numpy as np import pandas as pd -import pandera.pandas as pa -import requests from pandera.typing import DataFrame from _fb_types import UpdateResult -from _schemas import InferFetchFrame, QuestionFrame, ResolutionFrame -from helpers import constants, data_utils, dates +from _schemas import QuestionFrame from ._market import MarketSource -logger = logging.getLogger(__name__) - -_INFER_URL = "https://www.randforecastinginitiative.org" - class InferSource(MarketSource): - """INFER Public prediction market source.""" - - name: ClassVar[str] = "infer" - - # ------------------------------------------------------------------ - # Public: fetch - # ------------------------------------------------------------------ - - @pa.check_types - def fetch( - self, - *, - dfq: DataFrame[QuestionFrame] | None = None, - existing_resolution_ids: set[str] | None = None, - ) -> DataFrame[InferFetchFrame]: - """Fetch questions from the INFER API. - - Args: - dfq (DataFrame[QuestionFrame] | None): Existing question bank. - existing_resolution_ids (set[str] | None): Bare IDs that already have a resolution - file in storage. - """ - self._require_api_key() - existing_resolution_ids = existing_resolution_ids or set() - - # Determine which existing questions need re-fetching - resolved_ids: list[str] = [] - unresolved_ids: list[str] = [] - if dfq is not None and not dfq.empty: - resolved_ids = dfq[dfq["resolved"]]["id"].tolist() - unresolved_ids = dfq[~dfq["resolved"]]["id"].tolist() - - logger.info(f"Number resolved_ids: {len(resolved_ids)}") - logger.info(f"Number unresolved_ids: {len(unresolved_ids)}") - - resolved_ids_without_files = [ - id for id in resolved_ids if str(id) not in existing_resolution_ids - ] - logger.info(f"resolved_ids_without_resolution_files: {resolved_ids_without_files}") - - all_existing_ids_to_fetch = unresolved_ids + resolved_ids_without_files - - # Fetch existing (potentially closed) questions - all_existing_questions = ( - self._fetch_questions_from_api(status="all", question_ids=all_existing_ids_to_fetch) - if all_existing_ids_to_fetch - else [] - ) - - # Fetch all active questions - all_active_questions = self._fetch_questions_from_api() - - # Filter active to binary questions with predictions - all_active_binary_questions = [ - q - for q in all_active_questions - if q["state"] == "active" - and q["type"] == "Forecast::YesNoQuestion" - and q["answers"][0]["predictions_count"] > 0 - ] - - # Deduplicate: active takes precedence - active_ids = {q["id"] for q in all_active_binary_questions} - all_existing_questions = [q for q in all_existing_questions if q["id"] not in active_ids] - - all_questions = all_active_binary_questions + all_existing_questions - logger.info(f"Number of questions fetched: {len(all_questions)}") + """INFER prediction market source. - # Transform to InferFetchFrame schema - current_time = dates.get_datetime_now() - rows = [self._transform_question(q, current_time) for q in all_questions] + The RAND Forecasting Initiative shut down, so the source can no longer be fetched. + Already-published questions still resolve from their previously-stored resolution files. + ``update`` is a no-op until INFER questions are resolved by LLM or by hand. + """ - return pd.DataFrame(rows) + name: ClassVar[str] = "infer" - # ------------------------------------------------------------------ - # Public: update - # ------------------------------------------------------------------ + def fetch(self, **kwargs: Any) -> pd.DataFrame: + """Unavailable. INFER shut down, so there is nothing left to fetch.""" + raise RuntimeError(f"{self.name} can no longer be fetched.") - @pa.check_types def update( self, dfq: DataFrame[QuestionFrame], - dff: DataFrame[InferFetchFrame], - *, - existing_resolution_files: dict[str, DataFrame[ResolutionFrame]] | None = None, + dff: pd.DataFrame | None = None, + **kwargs: Any, ) -> UpdateResult: - """Process fetched data into updated questions and resolution files. + """No-op. In second pass, either use LLM to resolve or resolve by hand periodically. Args: - dfq (DataFrame[QuestionFrame]): Existing questions. - dff (DataFrame[InferFetchFrame]): Freshly fetched data. - existing_resolution_files (dict | None): Per-question existing resolution data. + dfq (DataFrame[QuestionFrame]): Existing questions, returned unchanged. + dff (pd.DataFrame | None): Always None. INFER is no longer fetched. """ - self._require_api_key() - existing_resolution_files = existing_resolution_files or {} - resolution_files: dict[str, pd.DataFrame] = {} - - for question in dff.to_dict("records"): - question_id = str(question["id"]) - - # Build/update resolution file - existing_df = existing_resolution_files.get(question_id) - df_res = self._build_resolution_df( - question=question, - resolved=question["resolved"], - existing_df=existing_df, - ) - resolution_files[question_id] = df_res - - # Mark nullified questions as resolved - if question["nullify_question"]: - question["resolved"] = True - - # Strip transient fields (not part of QuestionFrame) - del question["fetch_datetime"] - del question["probability"] - del question["nullify_question"] - - # Upsert into dfq - if question["id"] in dfq["id"].values: - dfq_index = dfq.index[dfq["id"] == question["id"]].tolist()[0] - for key, value in question.items(): - dfq.at[dfq_index, key] = value - else: - dfq = pd.concat([dfq, pd.DataFrame([question])], ignore_index=True) - - return UpdateResult( - dfq=dfq, - resolution_files=resolution_files, - ) - - # ------------------------------------------------------------------ - # Private: API calls - # ------------------------------------------------------------------ - - @backoff.on_exception( - backoff.expo, - requests.exceptions.RequestException, - max_time=300, - on_backoff=data_utils.print_error_info_handler, - ) - def _fetch_questions_from_api( - self, - *, - status: str = "active", - question_ids: list[str] | None = None, - ) -> list[dict]: - """Fetch paginated questions from the INFER API. - - Args: - status (str): "active" or "all". - question_ids (list[str] | None): If provided, fetch these specific IDs. - """ - api_key = self._require_api_key() - endpoint = _INFER_URL + "/api/v1/questions" - headers = {"Authorization": f"Bearer {api_key}"} - params: dict[str, Any] = {"page": 0, "status": status} - if question_ids is not None: - params.update({"status": "all", "ids": ",".join(sorted(question_ids))}) - - questions: list[dict] = [] - seen_ids: set = set() - while True: - response = requests.get( - endpoint, params=params, headers=headers, verify=certifi.where() - ) - if not response.ok: - logger.error(f"Request to Infer questions endpoint failed with params: {params}") - response.raise_for_status() - - new_questions = response.json().get("questions", []) - if not new_questions: - break - - for q in new_questions: - if q["id"] not in seen_ids: - questions.append(q) - seen_ids.add(q["id"]) - - params["page"] += 1 - - return questions - - def _get_historical_forecasts( - self, - current_df: DataFrame[ResolutionFrame] | None, - question_id: str, - ) -> DataFrame[ResolutionFrame]: - """Fetch historical prediction time series for a question. - - Args: - current_df (DataFrame[ResolutionFrame] | None): Existing resolution data. - question_id (str): INFER question ID. - """ - api_key = self._require_api_key() - endpoint = _INFER_URL + "/api/v1/prediction_sets" - params = {"question_id": question_id, "page": 0} - headers = {"Authorization": f"Bearer {api_key}"} - all_responses: list[dict] = [] - current_time = dates.get_datetime_today_midnight() - - # Determine cutoff: only fetch predictions newer than what we have - has_existing = current_df is not None and not current_df.empty - last_date = ( - pd.to_datetime(current_df["date"].iloc[-1]).tz_localize("UTC") - if has_existing - else constants.BENCHMARK_START_DATE_DATETIME.replace(tzinfo=timezone.utc) - ) - - while True: - try: - logger.info(f"Fetched page: {params['page']}, for question ID: {question_id}") - response = requests.get( - endpoint, params=params, headers=headers, verify=certifi.where() - ) - response.raise_for_status() - new_responses = response.json().get("prediction_sets", []) - all_responses.extend(new_responses) - if ( - not new_responses - or pd.to_datetime(new_responses[-1]["created_at"], utc=True) <= last_date - ): - break - params["page"] += 1 - except requests.exceptions.HTTPError as e: - if e.response.status_code != 429: - raise - logger.error("Rate limit reached, waiting 10s before retrying...") - time.sleep(10) - - # Extract (date, probability) from each prediction set - all_forecasts: list[tuple] = [] - for forecast in all_responses: - if not has_existing or pd.to_datetime(forecast["created_at"], utc=True) > last_date: - if len(forecast["predictions"]) == 2: - forecast_yes = forecast["predictions"][0] - if forecast_yes["answer_name"] == "No": - forecast_yes = forecast["predictions"][1] - elif len(forecast["predictions"]) == 1: - forecast_yes = forecast["predictions"][0] - - all_forecasts.append( - ( - dates.convert_zulu_to_iso(forecast["created_at"]), - forecast_yes["final_probability"], - ) - ) - - df = pd.DataFrame(all_forecasts, columns=["date", "value"]) - df["date"] = pd.to_datetime(df["date"]) - df = df[df["date"].dt.date < current_time.date()] - df["value"] = df["value"].astype(float) - df["id"] = question_id - - # Sort and convert to date-only - df_sorted = df.sort_values("date").reset_index(drop=True) - df_sorted["date"] = df_sorted["date"].dt.date - df_final = df_sorted[["id", "date", "value"]] - - # Merge with existing data - if not has_existing: - result_df = df_final.drop_duplicates(subset=["id", "date"], keep="last") - else: - current_df = current_df.copy() - current_df["date"] = pd.to_datetime(current_df["date"]).dt.date - current_df_final = current_df[["id", "date", "value"]] - result_df = ( - pd.concat([current_df_final, df_final], axis=0) - .sort_values(by=["date"], ascending=True) - .drop_duplicates(subset=["id", "date"], keep="last") - .reset_index(drop=True) - ) - - # Forward-fill missing dates - result_df.loc[:, "date"] = pd.to_datetime(result_df["date"]).dt.tz_localize("UTC") - result_df = result_df.infer_objects() - result_df = result_df.sort_values(by="date") - all_dates = pd.date_range( - start=result_df["date"].min(), - end=current_time - timedelta(days=1), - freq="D", - ) - result_df = result_df.set_index("date").reindex(all_dates, method="ffill").reset_index() - result_df["id"] = question_id - result_df.reset_index(inplace=True) - result_df.rename(columns={"index": "date"}, inplace=True) - - return result_df[["id", "date", "value"]] - - # ------------------------------------------------------------------ - # Private: resolution file building - # ------------------------------------------------------------------ - - def _build_resolution_df( - self, - question: dict, - resolved: bool, - existing_df: DataFrame[ResolutionFrame] | None = None, - ) -> DataFrame[ResolutionFrame]: - """Build or update a resolution file for a single question. - - Args: - question (dict): Must have 'id', 'nullify_question'. If resolved, must also - have 'market_info_resolution_datetime' and 'probability'. - resolved (bool): Whether the question has resolved. - existing_df (DataFrame[ResolutionFrame] | None): Existing resolution data. - """ - yesterday = dates.get_datetime_today_midnight() - timedelta(days=1) - - # --- Nullification --- - if question["nullify_question"]: - logger.warning( - f"Nullifying question {question['id']}. " - "Pushing np.nan values to resolution file." - ) - if existing_df is None or existing_df.empty: - return pd.DataFrame( - { - "id": [question["id"]], - "date": [str(yesterday.date())], - "value": [np.nan], - } - ) - else: - df = existing_df.copy() - df["value"] = np.nan - return self._finalize_resolution_df(df) - - # --- Already up-to-date check --- - if ( - existing_df is not None - and not existing_df.empty - and pd.to_datetime(existing_df["date"].iloc[-1]).tz_localize("UTC") >= yesterday - ): - logger.info(f"{question['id']} is skipped because it's already up-to-date!") - return existing_df - - # --- Fetch historical forecasts --- - df = self._get_historical_forecasts(existing_df, question["id"]) - df["date"] = df["date"].dt.date if hasattr(df["date"].dtype, "tz") else df["date"] - - # --- Handle resolved questions --- - if resolved: - resolution_date_str = question["market_info_resolution_datetime"][:10] - resolution_date = pd.to_datetime(resolution_date_str) - df["date"] = pd.to_datetime(df["date"]) - df = df[df["date"] < resolution_date] - resolution_row = pd.DataFrame( - { - "id": [question["id"]], - "date": [resolution_date_str], - "value": [question["probability"]], - } - ) - df = pd.concat([df, resolution_row], ignore_index=True) - - return self._finalize_resolution_df(df) - - @staticmethod - def _finalize_resolution_df(df: pd.DataFrame) -> DataFrame[ResolutionFrame]: - """Apply date filtering and select resolution columns. - - Args: - df (pd.DataFrame): Raw resolution data with id, date, value columns. - """ - df["date"] = pd.to_datetime(df["date"]) - df = df[df["date"].dt.date >= constants.BENCHMARK_START_DATE_DATETIME_DATE] - return df[["id", "date", "value"]].astype(dtype=constants.RESOLUTION_FILE_COLUMN_DTYPE) - - # ------------------------------------------------------------------ - # Private: question transformation - # ------------------------------------------------------------------ - - @staticmethod - def _transform_question(q: dict, current_time: str) -> dict: - """Transform a single INFER API response to InferFetchFrame row. - - Args: - q (dict): Raw question dict from the INFER API. - current_time (str): ISO timestamp for fetch_datetime. - """ - nullify_question = q["type"] != "Forecast::YesNoQuestion" - - # --- Close datetime: min(scoring_end_time, ends_at) --- - scoring_end_time_str = ( - dates.convert_datetime_str_to_iso_utc(q["scoring_end_time"]) - if q["scoring_end_time"] - else "N/A" - ) - ended_at_str = dates.convert_zulu_to_iso(q["ends_at"]) if q["ends_at"] else "N/A" - final_closed_at_str = ( - "N/A" - if scoring_end_time_str == "N/A" and ended_at_str == "N/A" - else ( - ended_at_str - if scoring_end_time_str == "N/A" - else ( - scoring_end_time_str - if ended_at_str == "N/A" - else min(scoring_end_time_str, ended_at_str) - ) - ) - ) - - # --- Open datetime --- - scoring_start_time_str = ( - dates.convert_datetime_str_to_iso_utc(q["scoring_start_time"]) - if q["scoring_start_time"] - else "N/A" - ) - - # --- Resolution datetime: min(resolved_at, close_datetime) --- - resolved_at_str = dates.convert_zulu_to_iso(q["resolved_at"]) if q["resolved_at"] else "N/A" - final_resolved_str = ( - "N/A" - if resolved_at_str == "N/A" and final_closed_at_str == "N/A" - else ( - final_closed_at_str - if resolved_at_str == "N/A" - else ( - resolved_at_str - if final_closed_at_str == "N/A" - else min(resolved_at_str, final_closed_at_str) - ) - ) - ) - - # --- Probability --- - forecast_yes: Any = "N/A" - if len(q["answers"]) == 2 and not nullify_question: - yes_index = 0 if q["answers"][0]["name"].lower() == "yes" else 1 - forecast_yes = q["answers"][yes_index]["probability"] - - return { - "id": str(q["id"]), - "question": q["name"], - "background": q["description"], - "market_info_resolution_criteria": ( - " ".join([content["content"] for content in q["clarifications"]]) - if q["clarifications"] - else "N/A" - ), - "market_info_open_datetime": scoring_start_time_str, - "market_info_close_datetime": final_closed_at_str, - "url": f"{_INFER_URL}/questions/{q['id']}", - "resolved": q.get("resolved?", False), - "market_info_resolution_datetime": ( - "N/A" if not q.get("resolved?", False) else final_resolved_str - ), - "fetch_datetime": current_time, - "probability": forecast_yes, - "forecast_horizons": "N/A", - "freeze_datetime_value": forecast_yes, - "freeze_datetime_value_explanation": "The crowd forecast.", - "nullify_question": nullify_question, - } + return UpdateResult(dfq=dfq, resolution_files={}) diff --git a/src/tests/_module_stubs.py b/src/tests/_module_stubs.py new file mode 100644 index 00000000..10d3532d --- /dev/null +++ b/src/tests/_module_stubs.py @@ -0,0 +1,95 @@ +"""Import modules under stand-in dependencies so tests can run credential-free.""" + +import importlib +import sys +from contextlib import contextmanager + +_MISSING = object() + + +@contextmanager +def stubbed_modules(stubs): + """Install `stubs` in `sys.modules` for the duration of the block. + + Each stub is also set on its parent package, because `from pkg import mod` reads the + attribute the parent already holds rather than `sys.modules`. On exit both `sys.modules` + and the parent attributes are restored, including attributes the import machinery sets as + a side effect while the stubs are in place. + + Args: + stubs (dict): Mapping of dotted module path to stand-in module. + """ + previous_modules = {name: sys.modules.get(name, _MISSING) for name in stubs} + previous_attrs = {} + for name in stubs: + parent, attr = _parent_and_attr(name) + if parent is not None: + previous_attrs[(parent, attr)] = getattr(parent, attr, _MISSING) + + try: + for name, stub in stubs.items(): + sys.modules[name] = stub + parent, attr = _parent_and_attr(name) + if parent is not None: + setattr(parent, attr, stub) + yield + finally: + for name, previous in previous_modules.items(): + if previous is _MISSING: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + for (parent, attr), previous in previous_attrs.items(): + if previous is _MISSING: + if hasattr(parent, attr): + delattr(parent, attr) + else: + setattr(parent, attr, previous) + + +@contextmanager +def reset_modules(*names): + """Drop `names` from `sys.modules` for the duration of the block, restoring them on exit. + + Anything imported under those names inside the block is discarded, so each block gets a + fresh import. + + Args: + names (str): Dotted module paths to re-import from scratch. + """ + previous_modules = {name: sys.modules.get(name, _MISSING) for name in names} + for name in names: + sys.modules.pop(name, None) + try: + yield + finally: + for name, previous in previous_modules.items(): + sys.modules.pop(name, None) + if previous is not _MISSING: + sys.modules[name] = previous + + +@contextmanager +def imported_with_stubs(module_name, stubs): + """Import `module_name` with `stubs` standing in for its dependencies. + + Args: + module_name (str): Dotted path of the module under test. + stubs (dict): Mapping of dotted module path to stand-in module. + """ + with stubbed_modules(stubs), reset_modules(module_name): + yield importlib.import_module(module_name) + + +def _parent_and_attr(name): + """Return the parent package of dotted `name` and the attribute it is bound to. + + The parent is None for top-level modules and for packages that are not yet imported. + + Args: + name (str): Dotted module path. + """ + if "." not in name: + return None, None + parent_name, attr = name.rsplit(".", maxsplit=1) + return sys.modules.get(parent_name), attr diff --git a/src/tests/conftest.py b/src/tests/conftest.py index e6d4e9c9..cbc2f584 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -9,7 +9,6 @@ from sources.acled import AcledSource from sources.fred import FredSource -from sources.infer import InferSource from sources.manifold import ManifoldSource from sources.metaculus import MetaculusSource from sources.polymarket import PolymarketSource @@ -72,14 +71,6 @@ def acled_source(): return AcledSource() -@pytest.fixture() -def infer_source(): - """Return an InferSource instance with a fake API key.""" - src = InferSource() - src.api_key = "test-key" - return src - - @pytest.fixture() def manifold_source(): """Return a ManifoldSource instance.""" @@ -186,103 +177,6 @@ def make_question_set_df(rows): return pd.DataFrame(rows) -# --------------------------------------------------------------------------- -# INFER-specific factories -# --------------------------------------------------------------------------- - - -def make_infer_api_question(**overrides): - """Build a realistic INFER API question dict. Override specific fields as needed.""" - base = { - "id": 9999, - "name": "Will X happen by end of 2026?", - "description": "

Background text.

", - "clarifications": [], - "state": "active", - "type": "Forecast::YesNoQuestion", - "active?": True, - "binary?": False, - "resolved?": False, - "resolved_at": None, - "ends_at": "2026-06-01T04:00:00.000Z", - "starts_at": "2026-01-01T20:00:00.000Z", - "scoring_start_time": "2026-01-01T15:00:00.000-05:00", - "scoring_end_time": "2026-06-01T00:00:00.000-05:00", - "created_at": "2026-01-01T18:00:00.000Z", - "closed_at": None, - "voided_at": None, - "answers": [ - { - "id": 9001, - "name": "Yes", - "probability": 0.65, - "display_probability": "65%", - "predictions_count": 50, - "answer_name": "Yes", - }, - { - "id": 9002, - "name": "No", - "probability": 0.35, - "display_probability": "35%", - "predictions_count": 50, - "answer_name": "No", - }, - ], - } - base.update(overrides) - return base - - -def make_infer_prediction_set(created_at, yes_prob): - """Build a realistic INFER prediction set dict.""" - return { - "id": 999999, - "type": "Forecast::OpinionPoolPredictionSet", - "question_id": 9999, - "created_at": created_at, - "predictions": [ - { - "answer_name": "Yes", - "final_probability": yes_prob, - "forecasted_probability": yes_prob, - "starting_probability": yes_prob, - }, - { - "answer_name": "No", - "final_probability": round(1 - yes_prob, 4), - "forecasted_probability": round(1 - yes_prob, 4), - "starting_probability": round(1 - yes_prob, 4), - }, - ], - } - - -def make_infer_fetch_df(rows): - """Build a DataFrame matching InferFetchFrame schema.""" - defaults = { - "question": "N/A", - "background": "N/A", - "url": "N/A", - "resolved": False, - "forecast_horizons": "N/A", - "freeze_datetime_value": "N/A", - "freeze_datetime_value_explanation": "N/A", - "market_info_resolution_criteria": "N/A", - "market_info_open_datetime": "N/A", - "market_info_close_datetime": "N/A", - "market_info_resolution_datetime": "N/A", - "fetch_datetime": "2026-01-15T00:00:00+00:00", - "probability": 0.5, - "nullify_question": False, - } - df = pd.DataFrame(rows) - for col, default in defaults.items(): - if col not in df.columns: - df[col] = default - return df - - # --------------------------------------------------------------------------- # Yfinance-specific factories # --------------------------------------------------------------------------- diff --git a/src/tests/leaderboard/_leaderboard_import.py b/src/tests/leaderboard/_leaderboard_import.py new file mode 100644 index 00000000..0a937e0b --- /dev/null +++ b/src/tests/leaderboard/_leaderboard_import.py @@ -0,0 +1,50 @@ +"""Credential-free import of `leaderboard.main` for leaderboard tests.""" + +import importlib +import types +from contextlib import chdir, contextmanager +from pathlib import Path + +from tests._module_stubs import reset_modules, stubbed_modules + +ROOT = Path(__file__).resolve().parents[3] + +STUBS = { + "pyfixest": types.SimpleNamespace(), + "jinja2": types.SimpleNamespace(Template=object), + "joblib": types.SimpleNamespace(Parallel=object, delayed=lambda fn: fn), + "scipy": types.SimpleNamespace(), + "scipy.stats": types.SimpleNamespace(norm=object()), + "statsmodels": types.SimpleNamespace(), + "statsmodels.stats": types.SimpleNamespace(), + "statsmodels.stats.multitest": types.SimpleNamespace( + multipletests=lambda *args, **kwargs: None + ), + "termcolor": types.SimpleNamespace(colored=lambda text, *args, **kwargs: text), + "git": types.SimpleNamespace( + Actor=object, + Repo=object, + ), + "helpers.git": types.SimpleNamespace(), + "helpers.slack": types.SimpleNamespace(), +} + + +@contextmanager +def patched_import_environment(): + """Stand in for the GCP credentials and heavy stats dependencies `leaderboard.main` needs. + + `leaderboard.main` reads files relative to its own directory at import time, hence the chdir. + """ + with ( + stubbed_modules(STUBS), + reset_modules("leaderboard.main", "llm_identities"), + chdir(ROOT / "src" / "leaderboard"), + ): + yield + + +def import_leaderboard_main(): + """Import `leaderboard.main` without GCP credentials or the heavy stats dependencies.""" + with patched_import_environment(): + return importlib.import_module("leaderboard.main") diff --git a/src/tests/leaderboard/conftest.py b/src/tests/leaderboard/conftest.py new file mode 100644 index 00000000..db5e0542 --- /dev/null +++ b/src/tests/leaderboard/conftest.py @@ -0,0 +1,14 @@ +"""Shared fixtures for leaderboard tests.""" + +import importlib + +import pytest + +from tests.leaderboard._leaderboard_import import patched_import_environment + + +@pytest.fixture +def leaderboard_main(): + """Yield `leaderboard.main`, imported credential-free.""" + with patched_import_environment(): + yield importlib.import_module("leaderboard.main") diff --git a/src/tests/leaderboard/test_llm_identities.py b/src/tests/leaderboard/test_llm_identities.py index 4f74857e..e9a81310 100644 --- a/src/tests/leaderboard/test_llm_identities.py +++ b/src/tests/leaderboard/test_llm_identities.py @@ -1,83 +1,14 @@ """Tests for strict ForecastBench LLM identities.""" -import importlib -import os -import sys -import types -from contextlib import contextmanager from datetime import date from pathlib import Path import pandas as pd import pytest -ROOT = Path(__file__).resolve().parents[3] - - -@contextmanager -def _patched_import_environment(): - stubs = { - "pyfixest": types.SimpleNamespace(), - "jinja2": types.SimpleNamespace(Template=object), - "joblib": types.SimpleNamespace(Parallel=object, delayed=lambda fn: fn), - "scipy": types.SimpleNamespace(), - "scipy.stats": types.SimpleNamespace(norm=object()), - "statsmodels": types.SimpleNamespace(), - "statsmodels.stats": types.SimpleNamespace(), - "statsmodels.stats.multitest": types.SimpleNamespace( - multipletests=lambda *args, **kwargs: None - ), - "termcolor": types.SimpleNamespace(colored=lambda text, *args, **kwargs: text), - "git": types.SimpleNamespace( - Actor=object, - Repo=object, - ), - "helpers.git": types.SimpleNamespace(), - "helpers.slack": types.SimpleNamespace(), - } - previous_modules = {name: sys.modules.get(name) for name in stubs} - previous_parent_attrs = {} - for name in stubs: - if "." not in name: - continue - parent_name, attr = name.rsplit(".", maxsplit=1) - parent = sys.modules.get(parent_name) - if parent is not None: - previous_parent_attrs[(parent, attr)] = ( - hasattr(parent, attr), - getattr(parent, attr, None), - ) - previous_leaderboard_main = sys.modules.pop("leaderboard.main", None) - previous_top_level_llm_identities = sys.modules.pop("llm_identities", None) - previous_cwd = Path.cwd() - try: - sys.modules.update(stubs) - os.chdir(ROOT / "src" / "leaderboard") - yield - finally: - os.chdir(previous_cwd) - sys.modules.pop("leaderboard.main", None) - if previous_leaderboard_main is not None: - sys.modules["leaderboard.main"] = previous_leaderboard_main - sys.modules.pop("llm_identities", None) - if previous_top_level_llm_identities is not None: - sys.modules["llm_identities"] = previous_top_level_llm_identities - for name, previous in previous_modules.items(): - if previous is None: - sys.modules.pop(name, None) - else: - sys.modules[name] = previous - for (parent, attr), (had_attr, previous_attr) in previous_parent_attrs.items(): - if had_attr: - setattr(parent, attr, previous_attr) - elif hasattr(parent, attr): - delattr(parent, attr) - - -def _import_leaderboard_main(): - with _patched_import_environment(): - return importlib.import_module("leaderboard.main") +from tests.leaderboard._leaderboard_import import import_leaderboard_main +ROOT = Path(__file__).resolve().parents[3] CANONICAL_FORECASTBENCH_LLM = { "organization": "ForecastBench", @@ -127,7 +58,7 @@ def test_leaderboard_org_logo_lookup_uses_shared_lab_and_provider_names(): from utils.llm.lab_registry import LABS from utils.llm.provider_registry import PROVIDERS - main = _import_leaderboard_main() + main = import_leaderboard_main() assert main.get_org_logo(LABS["MiniMax"].name) == "minimax.svg" assert main.get_org_logo(LABS["Moonshot"].name) == "moonshot.svg" @@ -136,7 +67,7 @@ def test_leaderboard_org_logo_lookup_uses_shared_lab_and_provider_names(): def test_leaderboard_org_logo_lookup_keeps_legacy_and_external_names(): - main = _import_leaderboard_main() + main = import_leaderboard_main() assert main.get_org_logo("Moonshot") == "moonshot.svg" assert main.get_org_logo("Minimax") == "minimax.svg" @@ -486,7 +417,7 @@ def test_new_file_identity_uses_explicit_keys_for_display_and_semantics(): def test_leaderboard_filters_use_precomputed_selection_flags(): - main = _import_leaderboard_main() + main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -513,7 +444,7 @@ def test_leaderboard_filters_use_precomputed_selection_flags(): def test_baseline_filter_keeps_baseline_llm_variants_and_drops_tournament_variants(): - main = _import_leaderboard_main() + main = import_leaderboard_main() normalize = main.llm_identities.normalize_llm_identity df = pd.DataFrame( [ @@ -605,7 +536,7 @@ def test_baseline_filter_keeps_baseline_llm_variants_and_drops_tournament_varian def test_tournament_filter_keeps_tournament_llm_variants_and_drops_baseline_variants(): - main = _import_leaderboard_main() + main = import_leaderboard_main() normalize = main.llm_identities.normalize_llm_identity df = pd.DataFrame( [ @@ -681,7 +612,7 @@ def test_tournament_filter_keeps_tournament_llm_variants_and_drops_baseline_vari def test_leaderboard_filters_require_selection_flag_columns(): - main = _import_leaderboard_main() + main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -697,7 +628,7 @@ def test_leaderboard_filters_require_selection_flag_columns(): def test_preliminary_leaderboard_filters_to_tournament_models_before_scoring(monkeypatch): - main = _import_leaderboard_main() + main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -758,7 +689,7 @@ def fake_generate_simulated_leaderboards(df, **kwargs): def test_two_way_fixed_effects_excludes_external_submission_flag_from_fit(monkeypatch): - main = _import_leaderboard_main() + main = import_leaderboard_main() fitted_model_pks = [] df = pd.DataFrame( [ @@ -806,7 +737,7 @@ def fake_feols(_formula, data): def test_two_way_fixed_effects_excludes_llm_crowd_comparison_models_from_fit(monkeypatch): - main = _import_leaderboard_main() + main = import_leaderboard_main() fitted_model_pks = [] crowd_models = [ "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) " @@ -984,7 +915,7 @@ def test_explicit_new_identity_sets_uses_tools_from_model_run_key(): def test_leaderboard_integration_uses_model_run_and_forecast_variant_for_model_pk(): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -1025,7 +956,7 @@ def test_leaderboard_integration_uses_model_run_and_forecast_variant_for_model_p def test_set_model_pk_errors_when_forecastbench_llm_identity_columns_are_missing(): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -1042,7 +973,7 @@ def test_set_model_pk_errors_when_forecastbench_llm_identity_columns_are_missing def test_set_model_pk_uses_forecastbench_llm_flag(): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -1060,7 +991,7 @@ def test_set_model_pk_uses_forecastbench_llm_flag(): def test_get_df_info_handles_forecastbench_comparison_model_without_llm_identity(): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -1092,7 +1023,7 @@ def test_get_df_info_handles_forecastbench_comparison_model_without_llm_identity def test_get_df_info_handles_llm_crowd_comparison_models_without_llm_identity(monkeypatch): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() messages = [] monkeypatch.setattr(leaderboard_main.slack, "send_message", messages.append, raising=False) comparison_models = [ @@ -1135,7 +1066,7 @@ def test_get_df_info_handles_llm_crowd_comparison_models_without_llm_identity(mo def test_get_df_info_raises_and_sends_slack_for_unclassified_forecastbench_model(monkeypatch): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() messages = [] monkeypatch.setattr(leaderboard_main.slack, "send_message", messages.append, raising=False) df = pd.DataFrame( @@ -1166,7 +1097,7 @@ def test_get_df_info_raises_and_sends_slack_for_unclassified_forecastbench_model def test_get_df_info_uses_pre_normalized_llm_identity(monkeypatch): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() org_and_model = leaderboard_main.llm_identities.normalize_llm_identity( { "organization": "ForecastBench", @@ -1209,7 +1140,7 @@ def fail_if_called(*args, **kwargs): def test_legacy_and_new_model_run_identity_share_model_pk(): - leaderboard_main = _import_leaderboard_main() + leaderboard_main = import_leaderboard_main() df = pd.DataFrame( [ { diff --git a/src/tests/leaderboard/test_llm_identity_release_dates.py b/src/tests/leaderboard/test_llm_identity_release_dates.py index 126c5d52..76e25427 100644 --- a/src/tests/leaderboard/test_llm_identity_release_dates.py +++ b/src/tests/leaderboard/test_llm_identity_release_dates.py @@ -173,9 +173,9 @@ def test_explicit_model_run_and_forecast_variant_keys_are_validated(): def test_score_models_keeps_reference_models_when_llm_identity_columns_exist(): - from tests.leaderboard.test_llm_identities import _import_leaderboard_main + from tests.leaderboard._leaderboard_import import import_leaderboard_main - main = _import_leaderboard_main() + main = import_leaderboard_main() df = pd.DataFrame( [ { @@ -263,9 +263,9 @@ def test_score_models_keeps_reference_models_when_llm_identity_columns_exist(): def test_release_date_info_preserves_forecastbench_created_reference_models(monkeypatch): - from tests.leaderboard.test_llm_identities import _import_leaderboard_main + from tests.leaderboard._leaderboard_import import import_leaderboard_main - main = _import_leaderboard_main() + main = import_leaderboard_main() messages = [] monkeypatch.setattr(main.slack, "send_message", messages.append, raising=False) df = pd.DataFrame( @@ -313,9 +313,9 @@ def test_release_date_info_preserves_forecastbench_created_reference_models(monk def test_release_date_info_errors_when_forecastbench_llm_model_run_key_is_unknown(monkeypatch): - from tests.leaderboard.test_llm_identities import _import_leaderboard_main + from tests.leaderboard._leaderboard_import import import_leaderboard_main - main = _import_leaderboard_main() + main = import_leaderboard_main() messages = [] monkeypatch.setattr(main.slack, "send_message", messages.append, raising=False) df = pd.DataFrame( diff --git a/src/tests/leaderboard/test_source_masks.py b/src/tests/leaderboard/test_source_masks.py new file mode 100644 index 00000000..afb5ebf4 --- /dev/null +++ b/src/tests/leaderboard/test_source_masks.py @@ -0,0 +1,17 @@ +"""Leaderboard question categorization by source, including legacy market sources.""" + +import pandas as pd + + +def test_dataset_sources_counted(leaderboard_main): + df = pd.DataFrame({"source": ["acled", "yfinance", "manifold", "infer"]}) + mask = leaderboard_main.get_dataset_mask(df) + # dataset sources True; market sources (incl. legacy infer) False + assert mask.tolist() == [True, True, False, False] + + +def test_infer_counts_as_market(leaderboard_main): + # INFER questions must still be categorized as market on the leaderboard. + df = pd.DataFrame({"source": ["infer", "manifold", "acled"]}) + mask = leaderboard_main.get_market_mask(df) + assert mask.tolist() == [True, True, False] diff --git a/src/tests/orchestration/test_load_question_bank_staleness.py b/src/tests/orchestration/test_load_question_bank_staleness.py new file mode 100644 index 00000000..3d6c4650 --- /dev/null +++ b/src/tests/orchestration/test_load_question_bank_staleness.py @@ -0,0 +1,39 @@ +"""The staleness guard must skip sources we no longer fetch but still catch the ones we do.""" + +from datetime import datetime +from unittest.mock import patch + +import pytest + +from orchestration import _io +from sources import MARKET_SOURCE_NAMES + +TODAY = datetime(2026, 8, 3) +STALE = datetime(2026, 7, 1) + + +def _modified_times(active_time, infer_time): + def _fn(source): + return infer_time if source == "infer" else active_time + + return _fn + + +@patch.object(_io, "_build_question_bank", return_value={}) +@patch.object(_io.gcp.storage, "get_last_modified_time", return_value=TODAY) +@patch.object(_io.dates, "get_date_today", return_value=TODAY.date()) +@patch.object(_io.data_utils, "get_last_modified_time_of_dfq_from_cloud_storage") +def test_stale_unfetched_source_does_not_raise(mock_mtime, *_): + assert "infer" in MARKET_SOURCE_NAMES + mock_mtime.side_effect = _modified_times(active_time=TODAY, infer_time=STALE) + _io.load_question_bank(sources_to_get=[]) # should not raise + + +@patch.object(_io, "_build_question_bank", return_value={}) +@patch.object(_io.gcp.storage, "get_last_modified_time", return_value=TODAY) +@patch.object(_io.dates, "get_date_today", return_value=TODAY.date()) +@patch.object(_io.data_utils, "get_last_modified_time_of_dfq_from_cloud_storage") +def test_stale_active_source_still_raises(mock_mtime, *_): + mock_mtime.side_effect = _modified_times(active_time=STALE, infer_time=TODAY) + with pytest.raises(ValueError, match="Market-based dfq files need updating"): + _io.load_question_bank(sources_to_get=[]) diff --git a/src/tests/test_infer.py b/src/tests/test_infer.py index c562a526..d5355786 100644 --- a/src/tests/test_infer.py +++ b/src/tests/test_infer.py @@ -1,556 +1,35 @@ -"""Tests for InferSource fetch/update logic.""" +"""Tests for InferSource, whose upstream shut down so it can no longer be fetched.""" -from datetime import date -from unittest.mock import Mock, patch - -import numpy as np import pandas as pd import pytest -from _schemas import InferFetchFrame, QuestionFrame, ResolutionFrame +from sources import MARKET_SOURCE_NAMES from sources.infer import InferSource -from .conftest import ( - make_infer_api_question, - make_infer_fetch_df, - make_infer_prediction_set, - make_question_df, - make_resolution_df, -) - -# --------------------------------------------------------------------------- -# _transform_question (pure, no mocking) -# --------------------------------------------------------------------------- - - -class TestTransformQuestion: - """Tests for InferSource._transform_question static method.""" - - CURRENT_TIME = "2026-01-15T00:00:00+00:00" - - def test_standard_active_question(self): - """All fields populated, output matches InferFetchFrame schema.""" - q = make_infer_api_question() - row = InferSource._transform_question(q, self.CURRENT_TIME) - - assert row["id"] == "9999" - assert row["question"] == q["name"] - assert row["probability"] == 0.65 - assert row["nullify_question"] is False - assert row["resolved"] is False - assert row["market_info_resolution_datetime"] == "N/A" - assert row["fetch_datetime"] == self.CURRENT_TIME - # Verify schema compliance - df = pd.DataFrame([row]) - InferFetchFrame.validate(df) - - def test_resolved_question(self): - """Resolved question has resolution datetime set.""" - q = make_infer_api_question( - **{ - "resolved?": True, - "resolved_at": "2026-01-10T12:00:00.000Z", - "scoring_end_time": "2026-02-01T00:00:00.000-05:00", - } - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - - assert bool(row["resolved"]) is True - assert row["market_info_resolution_datetime"] != "N/A" - assert "2026-01-10" in row["market_info_resolution_datetime"] - - def test_non_binary_question_nullified(self): - """Non-YesNo question types get nullified.""" - q = make_infer_api_question(type="Forecast::MultipleChoiceQuestion") - row = InferSource._transform_question(q, self.CURRENT_TIME) - - assert row["nullify_question"] is True - assert row["probability"] == "N/A" - - def test_missing_datetime_fields(self): - """None datetimes produce N/A strings.""" - q = make_infer_api_question( - scoring_start_time=None, - scoring_end_time=None, - ends_at=None, - resolved_at=None, - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - - assert row["market_info_open_datetime"] == "N/A" - assert row["market_info_close_datetime"] == "N/A" - - def test_close_datetime_picks_earlier(self): - """Close datetime is min(scoring_end_time, ends_at).""" - q = make_infer_api_question( - scoring_end_time="2026-03-01T00:00:00.000-05:00", - ends_at="2026-06-01T04:00:00.000Z", - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - assert "2026-03" in row["market_info_close_datetime"] - - # Reverse: ends_at is earlier - q2 = make_infer_api_question( - scoring_end_time="2026-09-01T00:00:00.000-05:00", - ends_at="2026-06-01T04:00:00.000Z", - ) - row2 = InferSource._transform_question(q2, self.CURRENT_TIME) - assert "2026-06" in row2["market_info_close_datetime"] - - def test_resolution_datetime_picks_earlier(self): - """Resolution datetime is min(resolved_at, close_datetime).""" - q = make_infer_api_question( - **{ - "resolved?": True, - "resolved_at": "2026-02-01T00:00:00.000Z", - "scoring_end_time": "2026-06-01T00:00:00.000-05:00", - } - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - assert "2026-02-01" in row["market_info_resolution_datetime"] - - def test_answers_swapped_order(self): - """Extracts Yes probability even when No is first.""" - q = make_infer_api_question( - answers=[ - {"name": "No", "probability": 0.3, "predictions_count": 10}, - {"name": "Yes", "probability": 0.7, "predictions_count": 10}, - ] - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - assert row["probability"] == 0.7 - - def test_single_answer(self): - """Single-answer question still extracts probability.""" - q = make_infer_api_question( - answers=[{"name": "Yes", "probability": 0.8, "predictions_count": 5}] - ) - # Single answer → len != 2, so probability is N/A (binary check fails) - row = InferSource._transform_question(q, self.CURRENT_TIME) - assert row["probability"] == "N/A" - - def test_clarifications_joined(self): - """Multiple clarifications are joined into one string.""" - q = make_infer_api_question( - clarifications=[ - {"content": "Clarification 1."}, - {"content": "Clarification 2."}, - ] - ) - row = InferSource._transform_question(q, self.CURRENT_TIME) - assert "Clarification 1." in row["market_info_resolution_criteria"] - assert "Clarification 2." in row["market_info_resolution_criteria"] - - -# --------------------------------------------------------------------------- -# _finalize_resolution_df (pure, no mocking) -# --------------------------------------------------------------------------- - - -class TestFinalizeResolutionDf: - """Tests for InferSource._finalize_resolution_df static method.""" - - def test_filters_before_benchmark_start(self): - """Rows before BENCHMARK_START_DATE are dropped.""" - df = pd.DataFrame( - { - "id": ["A", "A", "A"], - "date": pd.to_datetime(["2020-01-01", "2024-06-01", "2024-07-01"]), - "value": [0.1, 0.2, 0.3], - } - ) - result = InferSource._finalize_resolution_df(df) - assert len(result) == 2 - assert result["value"].tolist() == [0.2, 0.3] - - def test_validates_schema(self): - """Output is a valid ResolutionFrame.""" - df = pd.DataFrame( - { - "id": ["A"], - "date": pd.to_datetime(["2024-06-01"]), - "value": [0.5], - } - ) - result = InferSource._finalize_resolution_df(df) - ResolutionFrame.validate(result) - - def test_only_keeps_id_date_value(self): - """Extra columns are stripped.""" - df = pd.DataFrame( - { - "id": ["A"], - "date": pd.to_datetime(["2024-06-01"]), - "value": [0.5], - "extra": ["junk"], - } - ) - result = InferSource._finalize_resolution_df(df) - assert list(result.columns) == ["id", "date", "value"] - - -# --------------------------------------------------------------------------- -# _build_resolution_df (mock _get_historical_forecasts) -# --------------------------------------------------------------------------- - - -class TestBuildResolutionDf: - """Tests for InferSource._build_resolution_df.""" - - def _question(self, **overrides): - base = { - "id": "200", - "nullify_question": False, - "market_info_resolution_datetime": "N/A", - "probability": 0.6, - } - base.update(overrides) - return base - - @patch.object(InferSource, "_get_historical_forecasts") - def test_nullified_no_existing(self, mock_hist, infer_source, freeze_today): - """Nullified question with no existing data returns single NaN row.""" - freeze_today(date(2026, 1, 15)) - q = self._question(nullify_question=True) - df = infer_source._build_resolution_df(q, resolved=False, existing_df=None) - - assert len(df) == 1 - assert np.isnan(df["value"].iloc[0]) - mock_hist.assert_not_called() - - @patch.object(InferSource, "_get_historical_forecasts") - def test_nullified_with_existing(self, mock_hist, infer_source, freeze_today): - """Nullified question with existing data sets all values to NaN.""" - freeze_today(date(2026, 1, 15)) - existing = make_resolution_df( - [ - {"id": "200", "date": "2024-06-01", "value": 0.5}, - {"id": "200", "date": "2024-06-02", "value": 0.6}, - ] - ) - q = self._question(nullify_question=True) - df = infer_source._build_resolution_df(q, resolved=False, existing_df=existing) - - assert df["value"].isna().all() - mock_hist.assert_not_called() - - @patch.object(InferSource, "_get_historical_forecasts") - def test_already_up_to_date(self, mock_hist, infer_source, freeze_today): - """Skips API call if existing data covers through yesterday.""" - freeze_today(date(2026, 1, 15)) - existing = make_resolution_df( - [ - {"id": "200", "date": "2024-06-01", "value": 0.5}, - {"id": "200", "date": "2026-01-14", "value": 0.6}, - ] - ) - q = self._question() - df = infer_source._build_resolution_df(q, resolved=False, existing_df=existing) - - assert df.equals(existing) - mock_hist.assert_not_called() - - @patch.object(InferSource, "_get_historical_forecasts") - def test_fetches_when_stale(self, mock_hist, infer_source, freeze_today): - """Calls _get_historical_forecasts when existing data is stale.""" - freeze_today(date(2026, 1, 15)) - mock_hist.return_value = make_resolution_df( - [ - {"id": "200", "date": "2024-06-01", "value": 0.5}, - {"id": "200", "date": "2026-01-14", "value": 0.65}, - ] - ) - existing = make_resolution_df([{"id": "200", "date": "2024-06-01", "value": 0.5}]) - q = self._question() - df = infer_source._build_resolution_df(q, resolved=False, existing_df=existing) - - assert not df.empty - mock_hist.assert_called_once() - - @patch.object(InferSource, "_get_historical_forecasts") - def test_resolved_truncates_and_appends(self, mock_hist, infer_source, freeze_today): - """Resolved question truncates at resolution date and appends final row.""" - freeze_today(date(2026, 1, 15)) - mock_hist.return_value = make_resolution_df( - [ - {"id": "200", "date": "2024-06-01", "value": 0.4}, - {"id": "200", "date": "2026-01-10", "value": 0.6}, - {"id": "200", "date": "2026-01-12", "value": 0.7}, - ] - ) - q = self._question( - market_info_resolution_datetime="2026-01-11T00:00:00+00:00", - probability=1.0, - ) - df = infer_source._build_resolution_df(q, resolved=True, existing_df=None) - - # Should have rows up to resolution date - assert not df.empty - # Last row should be the resolution value - assert float(df.iloc[-1]["value"]) == 1.0 - - -# --------------------------------------------------------------------------- -# fetch() (mock _fetch_questions_from_api) -# --------------------------------------------------------------------------- - - -class TestFetch: - """Tests for InferSource.fetch.""" - - @patch.object(InferSource, "_fetch_questions_from_api") - def test_basic_fetch(self, mock_api, infer_source): - """Returns InferFetchFrame with correct rows.""" - mock_api.return_value = [ - make_infer_api_question(id=200), - make_infer_api_question(id=201), - ] - dff = infer_source.fetch() - - assert len(dff) == 2 - InferFetchFrame.validate(dff) - - @patch.object(InferSource, "_fetch_questions_from_api") - def test_active_filter(self, mock_api, infer_source): - """Only active binary questions with predictions pass the filter.""" - mock_api.return_value = [ - make_infer_api_question(id=1, state="active"), - make_infer_api_question(id=2, state="closed"), # filtered out - make_infer_api_question(id=3, type="Forecast::MultipleChoiceQuestion"), # filtered out - make_infer_api_question( - id=4, - answers=[ - {"name": "Yes", "probability": 0.5, "predictions_count": 0}, - {"name": "No", "probability": 0.5, "predictions_count": 0}, - ], - ), # filtered out (no predictions) - ] - dff = infer_source.fetch() - assert len(dff) == 1 - assert dff.iloc[0]["id"] == "1" - - @patch.object(InferSource, "_fetch_questions_from_api") - def test_deduplication_active_wins(self, mock_api, infer_source): - """When same ID appears in both active and existing, active version wins.""" - mock_api.side_effect = [ - [make_infer_api_question(id=100, state="closed")], # existing re-fetch - [make_infer_api_question(id=100, state="active")], # active fetch - ] - dfq = make_question_df([{"id": "100", "resolved": False}]) - dff = infer_source.fetch(dfq=dfq, existing_resolution_ids=set()) - - assert len(dff) == 1 - - @patch.object(InferSource, "_fetch_questions_from_api") - def test_resolved_without_files_refetched(self, mock_api, infer_source): - """Resolved questions missing resolution files are re-fetched.""" - mock_api.side_effect = [ - [make_infer_api_question(id=100, state="resolved", **{"resolved?": True})], - [], # no active - ] - dfq = make_question_df([{"id": "100", "resolved": True}]) - # No resolution file in storage → should re-fetch - dff = infer_source.fetch(dfq=dfq, existing_resolution_ids=set()) - - assert len(dff) == 1 - mock_api.assert_any_call(status="all", question_ids=["100"]) - - @patch.object(InferSource, "_fetch_questions_from_api") - def test_empty_dfq(self, mock_api, infer_source): - """Works with no existing questions.""" - mock_api.side_effect = [ - [make_infer_api_question(id=300)], - ] - dff = infer_source.fetch(dfq=None, existing_resolution_ids=set()) - assert len(dff) == 1 - - def test_api_key_required(self): - """Raises RuntimeError if api_key not set.""" - src = InferSource() # no api_key - with pytest.raises(RuntimeError, match="api_key must be set"): - src.fetch() - - -# --------------------------------------------------------------------------- -# update() (mock _build_resolution_df) -# --------------------------------------------------------------------------- - - -class TestUpdate: - """Tests for InferSource.update.""" - - @patch.object(InferSource, "_build_resolution_df") - def test_basic_update(self, mock_build, infer_source): - """Returns UpdateResult with valid dfq and resolution files.""" - mock_build.return_value = make_resolution_df( - [{"id": "200", "date": "2024-06-01", "value": 0.65}] - ) - dfq = make_question_df([{"id": "100"}]) - dff = make_infer_fetch_df([{"id": "200"}]) - - result = infer_source.update(dfq, dff) - - assert "200" in result.dfq["id"].values - assert "200" in result.resolution_files - QuestionFrame.validate(result.dfq) - - @patch.object(InferSource, "_build_resolution_df") - def test_new_question_inserted(self, mock_build, infer_source): - """Question not in dfq gets appended.""" - mock_build.return_value = make_resolution_df( - [{"id": "300", "date": "2024-06-01", "value": 0.5}] - ) - dfq = make_question_df([{"id": "100"}]) - dff = make_infer_fetch_df([{"id": "300"}]) - - result = infer_source.update(dfq, dff) - assert len(result.dfq) == 2 - assert set(result.dfq["id"].tolist()) == {"100", "300"} - - @patch.object(InferSource, "_build_resolution_df") - def test_existing_question_updated(self, mock_build, infer_source): - """Existing question fields are updated in place.""" - mock_build.return_value = make_resolution_df( - [{"id": "100", "date": "2024-06-01", "value": 0.5}] - ) - dfq = make_question_df([{"id": "100", "question": "Old text"}]) - dff = make_infer_fetch_df([{"id": "100", "question": "New text"}]) - - result = infer_source.update(dfq, dff) - assert len(result.dfq) == 1 - assert result.dfq.iloc[0]["question"] == "New text" - - @patch.object(InferSource, "_build_resolution_df") - def test_nullified_marked_resolved(self, mock_build, infer_source): - """Nullified questions are marked as resolved in dfq.""" - mock_build.return_value = make_resolution_df( - [{"id": "200", "date": "2024-06-01", "value": np.nan}] - ) - dfq = make_question_df([{"id": "100"}]) - dff = make_infer_fetch_df([{"id": "200", "nullify_question": True}]) - - result = infer_source.update(dfq, dff) - row = result.dfq[result.dfq["id"] == "200"].iloc[0] - assert bool(row["resolved"]) is True - - @patch.object(InferSource, "_build_resolution_df") - def test_transient_fields_stripped(self, mock_build, infer_source): - """fetch_datetime, probability, nullify_question not in output dfq.""" - mock_build.return_value = make_resolution_df( - [{"id": "200", "date": "2024-06-01", "value": 0.5}] - ) - dfq = make_question_df([{"id": "placeholder"}]).iloc[:0] - dff = make_infer_fetch_df([{"id": "200"}]) - - result = infer_source.update(dfq, dff) - for col in ["fetch_datetime", "probability", "nullify_question"]: - assert col not in result.dfq.columns - - def test_api_key_required(self): - """Raises RuntimeError if api_key not set.""" - src = InferSource() - dfq = make_question_df([{"id": "100"}]) - dff = make_infer_fetch_df([{"id": "200"}]) - with pytest.raises(RuntimeError, match="api_key must be set"): - src.update(dfq, dff) - - -# --------------------------------------------------------------------------- -# _get_historical_forecasts (mock requests.get) -# --------------------------------------------------------------------------- - - -class TestGetHistoricalForecasts: - """Tests for InferSource._get_historical_forecasts.""" - - def _mock_response(self, prediction_sets): - resp = Mock() - resp.ok = True - resp.json.return_value = {"prediction_sets": prediction_sets} - resp.raise_for_status = Mock() - return resp - - @patch("sources.infer.requests.get") - def test_basic_fetch_no_existing(self, mock_get, infer_source, freeze_today): - """Builds time series from scratch.""" - freeze_today(date(2026, 1, 15)) - mock_get.side_effect = [ - self._mock_response( - [ - make_infer_prediction_set("2026-01-10T12:00:00.000Z", 0.4), - make_infer_prediction_set("2026-01-12T14:00:00.000Z", 0.6), - ] - ), - self._mock_response([]), # empty page stops pagination - ] - - df = infer_source._get_historical_forecasts(None, "200") - - assert not df.empty - assert list(df.columns) == ["id", "date", "value"] - assert (df["id"] == "200").all() - # Should have forward-filled dates between 10th and 14th - assert len(df) >= 4 - - @patch("sources.infer.requests.get") - def test_incremental_with_existing(self, mock_get, infer_source, freeze_today): - """Only fetches newer predictions when existing data provided.""" - freeze_today(date(2026, 1, 15)) - existing = make_resolution_df( - [ - {"id": "200", "date": "2026-01-10", "value": 0.4}, - {"id": "200", "date": "2026-01-11", "value": 0.4}, - ] - ) - mock_get.side_effect = [ - self._mock_response([make_infer_prediction_set("2026-01-13T12:00:00.000Z", 0.7)]), - self._mock_response([]), - ] - - df = infer_source._get_historical_forecasts(existing, "200") - - assert not df.empty - # Should contain both old and new data, forward-filled - assert len(df) >= 4 - - @patch("sources.infer.requests.get") - def test_forward_fill_gaps(self, mock_get, infer_source, freeze_today): - """Missing dates between predictions are forward-filled.""" - freeze_today(date(2026, 1, 15)) - mock_get.side_effect = [ - self._mock_response( - [ - make_infer_prediction_set("2026-01-10T12:00:00.000Z", 0.3), - make_infer_prediction_set("2026-01-13T12:00:00.000Z", 0.8), - ] - ), - self._mock_response([]), - ] +from .conftest import make_question_df - df = infer_source._get_historical_forecasts(None, "200") - # Dates 10, 11, 12, 13, 14 should exist (14 = today-1) - dates_in_df = pd.to_datetime(df["date"]).dt.date.tolist() - assert date(2026, 1, 11) in dates_in_df # forward-filled - assert date(2026, 1, 12) in dates_in_df # forward-filled +def test_infer_still_a_market_source(): + assert "infer" in MARKET_SOURCE_NAMES - @patch("sources.infer.requests.get") - @patch("sources.infer.time.sleep") - def test_rate_limit_retry(self, mock_sleep, mock_get, infer_source, freeze_today): - """429 response triggers retry after sleep.""" - freeze_today(date(2026, 1, 15)) - rate_limit_resp = Mock() - rate_limit_resp.raise_for_status.side_effect = __import__("requests").exceptions.HTTPError( - response=Mock(status_code=429) - ) +def test_infer_fetch_raises(): + with pytest.raises(RuntimeError, match="can no longer be fetched"): + InferSource().fetch() - ok_resp = self._mock_response([make_infer_prediction_set("2026-01-10T12:00:00.000Z", 0.5)]) - empty_resp = self._mock_response([]) - mock_get.side_effect = [rate_limit_resp, ok_resp, empty_resp] +def test_infer_update_is_noop(): + dfq = make_question_df([{"id": "100"}]) + before = dfq.copy() + result = InferSource().update(dfq) + # No-op writes no resolution files and leaves the caller's questions untouched, so the + # update job has nothing to upload. + assert not result.resolution_files + pd.testing.assert_frame_equal(result.dfq, before, check_dtype=False) + pd.testing.assert_frame_equal(dfq, before, check_dtype=False) - df = infer_source._get_historical_forecasts(None, "200") - assert not df.empty - mock_sleep.assert_called_once_with(10) +def test_infer_nullified_questions_are_registered(): + # The questions INFER left unresolved must be nullified so they drop out of scoring + # instead of silently resolving to NaN against a frozen resolution file. + assert InferSource().nullified_questions diff --git a/src/tests/test_nightly_worker_fetch_list.py b/src/tests/test_nightly_worker_fetch_list.py new file mode 100644 index 00000000..bde83787 --- /dev/null +++ b/src/tests/test_nightly_worker_fetch_list.py @@ -0,0 +1,95 @@ +"""The nightly manager must launch enough worker tasks to cover every source it fetches.""" + +import types +from unittest.mock import patch + +import pytest + +from tests._module_stubs import imported_with_stubs + +WEEKDAYS = [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", + "Sunday", +] + + +def _stub_cloud_run(): + """Build a stand-in for `helpers.cloud_run`. + + The real module imports `google.cloud.run_v2`, which is unavailable in credential-free test + runs. + """ + cloud_run = types.ModuleType("helpers.cloud_run") + cloud_run.timeout_1h = 3600 + cloud_run.run_job = None + cloud_run.block_and_check_job_result = None + cloud_run.call_worker = None + return cloud_run + + +@pytest.fixture() +def worker(): + """Import the nightly worker credential-free.""" + with imported_with_stubs( + "nightly_update_workflow.worker.main", + {"helpers.cloud_run": _stub_cloud_run()}, + ) as module: + yield module + + +@pytest.fixture() +def manager(): + """Import the nightly manager credential-free. + + `helpers.slack` pulls a bot token out of Secret Manager at import time. + """ + with imported_with_stubs( + "nightly_update_workflow.manager.main", + { + "helpers.cloud_run": _stub_cloud_run(), + "helpers.slack": types.ModuleType("helpers.slack"), + }, + ) as module: + yield module + + +def _jobs_on(worker, day_of_week): + with patch.object(worker.dates, "get_datetime_today") as mock_today: + mock_today.return_value.strftime.return_value = day_of_week + return worker.get_fetch_and_update() + + +def _job_names(jobs): + return {job[0] for group in jobs for job in group} + + +def test_infer_has_update_but_no_fetch(worker): + names = _job_names(worker.get_fetch_and_update()) + assert "func-data-infer-update-questions" in names + assert "func-data-infer-fetch" not in names + + +def test_active_source_has_both(worker): + names = _job_names(worker.get_fetch_and_update()) + assert "func-data-manifold-fetch" in names + assert "func-data-manifold-update-questions" in names + + +@pytest.mark.parametrize("day_of_week", WEEKDAYS) +def test_manager_launches_enough_tasks_for_every_source(manager, worker, day_of_week): + # The worker exits any task index >= len(jobs), so an undersized task_count silently drops + # whole sources -- no error, no log. ACLED is only in the list on Wednesdays, which makes an + # off-by-one here invisible six days a week. + jobs = _jobs_on(worker, day_of_week) + assert manager.get_fetch_and_update_task_count() >= len(jobs) + + +def test_acled_is_fetched_on_wednesdays(worker): + names = _job_names(_jobs_on(worker, "Wednesday")) + assert "func-data-acled-fetch" in names + assert "func-data-acled-update-questions" in names diff --git a/src/tests/test_question_curation_sources.py b/src/tests/test_question_curation_sources.py new file mode 100644 index 00000000..3e9cbb93 --- /dev/null +++ b/src/tests/test_question_curation_sources.py @@ -0,0 +1,13 @@ +"""Sampling should exclude INFER; resolution/categorization should still include it.""" + +from helpers import question_curation +from sources import MARKET_SOURCE_NAMES + + +def test_infer_not_sampled(): + assert "infer" not in question_curation.MARKET_SOURCES + assert "infer" not in question_curation.FREEZE_QUESTION_MARKET_SOURCES + + +def test_infer_still_a_market_source_for_resolution(): + assert "infer" in MARKET_SOURCE_NAMES diff --git a/src/tests/test_source_scheduling.py b/src/tests/test_source_scheduling.py new file mode 100644 index 00000000..e7abbf3b --- /dev/null +++ b/src/tests/test_source_scheduling.py @@ -0,0 +1,51 @@ +"""Which nightly jobs each source is scheduled for, per `run_fetch` / `run_update`.""" + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +from sources import ALL_SOURCE_NAMES, SOURCE_METADATA + + +def test_every_source_has_scheduling_flags(): + # Both flags default to True, so a source that never mentions them still reads as scheduled + # and no consumer needs a fallback. + for source in ALL_SOURCE_NAMES: + assert SOURCE_METADATA[source]["run_fetch"] in (True, False) + assert SOURCE_METADATA[source]["run_update"] in (True, False) + + +def test_infer_is_updated_but_never_fetched(): + # INFER's upstream shut down. Its questions still need updating so they can be resolved. + assert SOURCE_METADATA["infer"]["run_fetch"] is False + assert SOURCE_METADATA["infer"]["run_update"] is True + + +def test_every_other_source_is_fetched_and_updated(): + for source in ALL_SOURCE_NAMES: + if source == "infer": + continue + assert SOURCE_METADATA[source]["run_fetch"] is True, source + assert SOURCE_METADATA[source]["run_update"] is True, source + + +def test_sources_import_stays_lightweight(): + # Importing the lightweight `sources` surface must NOT pull the heavy registry + # or any concrete-source dependency (this is the worker deploy-safety property). + code = textwrap.dedent(""" + import sys + import sources + assert sources.SOURCE_METADATA["infer"]["run_fetch"] is False + heavy = [m for m in ("sources.registry", "yfinance", "backoff") if m in sys.modules] + assert not heavy, f"lightweight import pulled heavy modules: {heavy}" + """) + src_dir = Path(__file__).resolve().parents[1] + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": str(src_dir)}, + ) + assert result.returncode == 0, result.stderr diff --git a/src/www.forecastbench.org/about/index.md b/src/www.forecastbench.org/about/index.md index 57838ed9..1a2858e3 100644 --- a/src/www.forecastbench.org/about/index.md +++ b/src/www.forecastbench.org/about/index.md @@ -16,7 +16,7 @@ footer_scripts:

We use two types of binary prediction questions:

ForecastBench operates as a fully automated, dynamic system. New forecasting rounds occur every two weeks, with each round generating 500 questions split evenly between market and dataset questions. The leaderboard is updated nightly as new data becomes available and market questions resolve over time, allowing us to continuously track forecasting performance.

From e64237c789916b6e4c57fae1851a0462536aac6c Mon Sep 17 00:00:00 2001 From: Houtan Bastani Date: Wed, 5 Aug 2026 13:38:46 +0200 Subject: [PATCH 2/2] feat: chart number of questions sampled per source Add a `source_counts` argument to `plot_sampling_distribution()` that charts sampled vs target counts for every source as an extra row on the ALL SOURCES figure. Report each figure's sampled, target, and available counts in its title. --- .../create_question_set/main.py | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/src/curate_questions/create_question_set/main.py b/src/curate_questions/create_question_set/main.py index 746d6a7b..eb2b9f2c 100644 --- a/src/curate_questions/create_question_set/main.py +++ b/src/curate_questions/create_question_set/main.py @@ -163,6 +163,7 @@ def process_questions( df_all_available, total_market_requested, source_name="ALL SOURCES", + source_counts=[(source, got, want) for source, got, want, _ in source_summaries], ) log_sampling_summary( @@ -561,6 +562,7 @@ def plot_sampling_distribution( df_available: pd.DataFrame, n_target: int, source_name: str | None = None, + source_counts: list[tuple[str, int, int]] | None = None, ) -> None: """Plot the realized vs expected distribution for market question sampling. @@ -572,6 +574,8 @@ def plot_sampling_distribution( df_available (pd.DataFrame): All available questions (must include bin columns) n_target (int): Number of questions requested source_name (str | None): Name for the source (used in chart title) + source_counts (list[tuple[str, int, int]] | None): (source, sampled, target) per source, + charted as an extra row when given """ if not env.RUNNING_LOCALLY: return @@ -589,11 +593,19 @@ def plot_sampling_distribution( title = "Sampling Distribution" if source_name: title = f"Sampling Distribution: {source_name}" + title += ( + f"
{len(df_sampled):,}/{n_target:,} sampled " + f"from {len(df_available):,} available" + ) + + subplot_titles = ["Market Value", "Time Horizon", "Category"] + if source_counts: + subplot_titles.append("Questions by Source") fig = make_subplots( - rows=3, + rows=len(subplot_titles), cols=1, - subplot_titles=("Market Value", "Time Horizon", "Category"), + subplot_titles=tuple(subplot_titles), vertical_spacing=0.1, ) @@ -733,9 +745,43 @@ def add_line_chart( ) fig.update_xaxes(tickangle=45, row=3, col=1) + # Number of questions sampled from each source + if source_counts: + source_row = len(subplot_titles) + source_labels = [source for source, _, _ in source_counts] + # The target bar is never shorter than the selected bar, so labeling it keeps the + # sampled/target counts clear of both bars. + fig.add_trace( + go.Bar( + name="Target", + x=source_labels, + y=[target for _, _, target in source_counts], + text=[f"{sampled}/{target}" for _, sampled, target in source_counts], + textposition="outside", + marker=dict(color="coral", opacity=0.5), + legendgroup="target", + showlegend=False, + ), + row=source_row, + col=1, + ) + fig.add_trace( + go.Bar( + name="Selected", + x=source_labels, + y=[sampled for _, sampled, _ in source_counts], + marker=dict(color="steelblue"), + legendgroup="selected", + showlegend=False, + ), + row=source_row, + col=1, + ) + fig.update_xaxes(tickangle=45, row=source_row, col=1) + fig.update_layout( title_text=title, - height=1000, + height=250 + 250 * len(subplot_titles), barmode="overlay", legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), )