From b1b24946066909ab96e42fdafd04a684762504c7 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:09:38 +0200 Subject: [PATCH 001/105] copy private.py from api/ooniapi/private.py --- .../src/ooniprobe/routers/private.py | 1246 +++++++++++++++++ 1 file changed, 1246 insertions(+) create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py new file mode 100644 index 000000000..6d6a63756 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -0,0 +1,1246 @@ +""" +prefix: /api/_ + +In here live private API endpoints for use only by OONI services. You should +not rely on these as they are likely to change, break in unexpected ways. Also +there is no versioning on them. +""" +from datetime import date, datetime, timedelta +from itertools import product + +from urllib.parse import urljoin, urlencode +from typing import Dict, Any + +import logging +import math + +from flask import Blueprint, current_app, request, Response + +from sqlalchemy import sql + +from werkzeug.exceptions import BadRequest + +from ooniapi.auth import role_required +from ooniapi.config import metrics +from ooniapi.countries import lookup_country +from ooniapi.data import dnscheck_inputs, stunreachability_inputs +from ooniapi.database import query_click, query_click_one_row +from ooniapi.models import TEST_GROUPS +from ooniapi.prio import generate_test_list +from ooniapi.utils import cachedjson, nocachejson, jerror, req_json + +from ooniapi.probe_services import ( + probe_geoip, + extract_probe_ipaddr_octect, + generate_test_helpers_conf, + generate_report_id, +) + +# The private API is exposed under the prefix /api/_ +# e.g. https://api.ooni.io/api/_/test_names + +api_private_blueprint = Blueprint("api_private", "measurements") + +# TODO: configure tags for HTTP caching across where useful + +log = logging.getLogger() + + +def daterange(start_date, end_date): + for n in range(int((end_date - start_date).days)): + yield start_date + timedelta(n) + + +def validate_probe_cc_query_param(): + probe_cc = request.args.get("probe_cc") + if probe_cc is None or len(probe_cc) != 2: + raise BadRequest("missing or incorrect probe_cc") + return probe_cc + + +def validate_probe_asn_query_param(): + probe_asn = request.args.get("probe_asn") + if probe_asn is None: + # TODO: validate and return int? + raise BadRequest("missing or incorrect probe_asn") + return probe_asn + + +def expand_dates(li): + """Replaces 'date' key in a list of dict""" + for i in li: + i["date"] = i["date"].strftime("%Y-%m-%dT00:00:00+00:00") + + +@api_private_blueprint.route("/asn_by_month") +def api_private_asn_by_month() -> Response: + """Network count by month + --- + responses: + '200': + description: [{"date":"2018-08-31","value":4411}, ... ] + """ + q = """SELECT + COUNT(DISTINCT(probe_asn)) AS value, + toStartOfMonth(measurement_start_time) AS date + FROM fastpath + WHERE measurement_start_time < toStartOfMonth(addMonths(now(), 1)) + AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) + GROUP BY date ORDER BY date + """ + li = list(query_click(q, {})) + expand_dates(li) + return cachedjson("1d", li) + + +@api_private_blueprint.route("/countries_by_month") +def api_private_countries_by_month() -> Response: + """Countries count by month + --- + responses: + '200': + description: TODO + """ + q = """SELECT + COUNT(DISTINCT(probe_cc)) AS value, + toStartOfMonth(measurement_start_time) AS date + FROM fastpath + WHERE measurement_start_time < toStartOfMonth(addMonths(now(), 1)) + AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) + GROUP BY date ORDER BY date + """ + li = list(query_click(q, {})) + expand_dates(li) + return cachedjson("1d", li) + + +@api_private_blueprint.route("/test_names", methods=["GET"]) +def api_private_test_names() -> Response: + """Provides test names and descriptions to Explorer + --- + responses: + '200': + description: TODO + """ + # TODO: eventually drop this, once we see nobody is using it + TEST_NAMES = { + "bridge_reachability": "Bridge Reachability", + "dash": "DASH", + "dns_consistency": "DNS Consistency", + "dnscheck": "DNS Check", + "facebook_messenger": "Facebook Messenger", + "http_header_field_manipulation": "HTTP Header Field Manipulation", + "http_host": "HTTP Host", + "http_invalid_request_line": "HTTP Invalid Request Line", + "http_requests": "HTTP Requests", + "meek_fronted_requests_test": "Meek Fronted Requests", + "multi_protocol_traceroute": "Multi Protocol Traceroute", + "ndt": "NDT", + "psiphon": "Psiphon", + "riseupvpn": "RiseupVPN", + "signal": "Signal", + "stunreachability": "STUN Reachability", + "tcp_connect": "TCP Connect", + "telegram": "Telegram", + "tor": "Tor", + "torsf": "Tor Snowflake", + "urlgetter": "URL Getter", + "vanilla_tor": "Vanilla Tor", + "web_connectivity": "Web Connectivity", + "whatsapp": "WhatsApp", + } + test_names = [{"id": k, "name": v} for k, v in TEST_NAMES.items()] + return cachedjson("1h", test_names=test_names) + + +@api_private_blueprint.route("/countries", methods=["GET"]) +def api_private_countries() -> Response: + """Summary of countries + --- + responses: + '200': + description: {"countries": [{"alpha_2": x, "count": y, "name": z}, ... ]} + """ + q = """ + SELECT probe_cc, COUNT() AS measurement_count + FROM fastpath + GROUP BY probe_cc ORDER BY probe_cc + """ + c = [] + rows = query_click(q, {}) + for r in rows: + try: + name = lookup_country(r["probe_cc"]) + c.append( + dict(alpha_2=r["probe_cc"], name=name, count=r["measurement_count"]) + ) + except KeyError: + pass + + return cachedjson("1d", countries=c) + + +@api_private_blueprint.route("/quotas_summary", methods=["GET"]) +@role_required(["admin"]) +def api_private_quotas_summary() -> Response: + """Summary on rate-limiting quotas. + [(first ipaddr octet, remaining daily quota), ... ] + """ + return nocachejson(current_app.limiter.get_lowest_daily_quotas_summary()) + + +@api_private_blueprint.route("/check_report_id", methods=["GET"]) +def check_report_id() -> Response: + """Legacy. Used to check if a report_id existed in the fastpath table. + Used by https://github.com/ooni/probe/issues/1034 + --- + produces: + - application/json + parameters: + - name: report_id + in: query + type: string + responses: + 200: + description: Always returns True. + schema: + type: object + properties: + v: + type: integer + description: version number of this response + found: + type: boolean + description: True + example: { "found": true, "v": 0 } + + """ + return cachedjson("1s", v=0, found=True) + + +def last_30days(begin=31, end=1): + first_day = datetime.now() - timedelta(begin) + first_day = datetime(first_day.year, first_day.month, first_day.day) + + last_day = datetime.now() - timedelta(end) + last_day = datetime(last_day.year, last_day.month, last_day.day) + + for d in daterange(first_day, last_day): + yield d.strftime("%Y-%m-%d") + + +def pivot_test_coverage(rows, test_group_names, days): + # "pivot": create a datapoint for each test group, for each day + # lookup map (tg, day) -> cnt + tmp = {} + for r in rows: + day = r["measurement_start_day"].strftime("%Y-%m-%d") + k = (r["test_group"], day) + tmp[k] = r["msmt_cnt"] + + test_coverage = [ + dict( + count=tmp.get((tg, day), 0), + test_day=day, + test_group=tg, + ) + for tg in test_group_names + for day in days + ] + return test_coverage + + +def get_recent_test_coverage_ch(probe_cc): + """Returns + [{"count": 4888, "test_day": "2021-10-16", "test_group": "websites"}, ... ] + """ + q = "SELECT DISTINCT(test_group) FROM test_groups ORDER BY test_group" + rows = query_click(sql.text(q), {}) + test_group_names = [r["test_group"] for r in rows] + + q = """SELECT + toDate(measurement_start_time) as measurement_start_day, + test_group, + COUNT() as msmt_cnt + FROM fastpath + ANY LEFT JOIN test_groups USING (test_name) + WHERE measurement_start_day >= today() - interval 32 day + AND measurement_start_day < today() - interval 2 day + AND probe_cc = :probe_cc + GROUP BY measurement_start_day, test_group + """ + rows = query_click(sql.text(q), dict(probe_cc=probe_cc)) + rows = tuple(rows) + l30d = tuple(last_30days(32, 2)) + return pivot_test_coverage(rows, test_group_names, l30d) + + +def get_recent_network_coverage_ch(probe_cc, test_groups): + """Count ASNs with at least one measurements, grouped by day, + for a given CC, and filtered by test groups + Return [{"count": 58, "test_day": "2021-10-16" }, ... ]""" + s = """SELECT + toDate(measurement_start_time) AS test_day, + COUNT(DISTINCT probe_asn) as count + FROM fastpath + WHERE test_day >= today() - interval 31 day + AND test_day < today() - interval 1 day + AND probe_cc = :probe_cc + --mark-- + GROUP BY test_day ORDER BY test_day + WITH FILL + FROM today() - interval 31 day + TO today() - interval 1 day + """ + if test_groups: + assert isinstance(test_groups, list) + test_names = set() + for tg in test_groups: + tnames = TEST_GROUPS.get(tg, []) + test_names.update(tnames) + test_names = sorted(test_names) + s = s.replace("--mark--", "AND test_name IN :test_names") + d = {"probe_cc": probe_cc, "test_names": test_names} + + else: + s = s.replace("--mark--", "") + d = {"probe_cc": probe_cc} + + return query_click(sql.text(s), d) + + +@api_private_blueprint.route("/test_coverage", methods=["GET"]) +def api_private_test_coverage() -> Response: + """Return number of measurements per day across test categories + --- + parameters: + - name: probe_cc + in: query + type: string + minLength: 2 + required: true + responses: + '200': + description: '{"network_coverage: [...], "test_coverage": [...]}' + """ + # TODO: merge the two queries into one? + # TODO: remove test categories or move aggregation to the front-end? + probe_cc = validate_probe_cc_query_param() + test_groups = request.args.get("test_groups") + if test_groups is not None: + test_groups = test_groups.split(",") + + tc = get_recent_test_coverage_ch(probe_cc) + nc = get_recent_network_coverage_ch(probe_cc, test_groups) + return cachedjson("1h", network_coverage=nc, test_coverage=tc) + + +@api_private_blueprint.route("/website_networks", methods=["GET"]) +def api_private_website_network_tests() -> Response: + """TODO + --- + parameters: + - name: probe_cc + in: query + type: string + description: The two letter country code + responses: + '200': + description: TODO + """ + probe_cc = validate_probe_cc_query_param() + s = """SELECT + COUNT() AS count, + probe_asn + FROM fastpath + WHERE + measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND probe_cc = :probe_cc + GROUP BY probe_asn + ORDER BY count DESC + """ + results = query_click(sql.text(s), {"probe_cc": probe_cc}) + return cachedjson("0s", results=results) + + +@api_private_blueprint.route("/website_stats", methods=["GET"]) +def api_private_website_stats() -> Response: + """TODO + --- + responses: + '200': + description: TODO + """ + # uses_pg_index counters_day_cc_asn_input_idx a BRIN index was not used at + # all, but BTREE on (measurement_start_day, probe_cc, probe_asn, input) + # made queries go from full scan to 50ms + url = request.args.get("input") + probe_cc = validate_probe_cc_query_param() + probe_asn = validate_probe_asn_query_param() + + s = """SELECT + toDate(measurement_start_time) AS test_day, + countIf(anomaly = 't') as anomaly_count, + countIf(confirmed = 't') as confirmed_count, + countIf(msm_failure = 't') as failure_count, + COUNT() AS total_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + AND input = :input + GROUP BY test_day ORDER BY test_day + """ + d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} + results = query_click(sql.text(s), d) + return cachedjson("0s", results=results) + + +@api_private_blueprint.route("/website_urls", methods=["GET"]) +def api_private_website_test_urls() -> Response: + """TODO + --- + responses: + '200': + description: TODO + """ + # TODO optimize or remove + limit = int(request.args.get("limit", 10)) + if limit <= 0: + limit = 10 + offset = int(request.args.get("offset", 0)) + + probe_cc = validate_probe_cc_query_param() + probe_asn = validate_probe_asn_query_param() + probe_asn = int(probe_asn.replace("AS", "")) + + # Count how many distinct inputs we have in this CC / ASN / period + s = """ + SELECT COUNT(DISTINCT(input)) as input_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND test_name = 'web_connectivity' + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + """ + q = query_click_one_row(sql.text(s), dict(probe_cc=probe_cc, probe_asn=probe_asn)) + total_count = q["input_count"] if q else 0 + + # Group msmts by CC / ASN / period with LIMIT and OFFSET + s = """SELECT input, + countIf(anomaly = 't') as anomaly_count, + countIf(confirmed = 't') as confirmed_count, + countIf(msm_failure = 't') as failure_count, + COUNT() AS total_count + FROM fastpath + WHERE measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + AND test_name = 'web_connectivity' + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + GROUP BY input + ORDER BY confirmed_count DESC, total_count DESC, + anomaly_count DESC, input ASC + LIMIT :limit + OFFSET :offset + """ + d = { + "probe_cc": probe_cc, + "probe_asn": probe_asn, + "limit": limit, + "offset": offset, + } + results = query_click(sql.text(s), d) + current_page = math.ceil(offset / limit) + 1 + metadata = { + "offset": offset, + "limit": limit, + "current_page": current_page, + "total_count": total_count, + "next_url": None, + } + + # Create next_url + if len(results) >= limit: + args = dict( + limit=limit, + offset=offset + limit, + probe_asn=probe_asn, + probe_cc=probe_cc, + ) + # TODO: remove BASE_URL? + next_url = urljoin( + current_app.config["BASE_URL"], + "/api/_/website_urls?%s" % urlencode(args), + ) + metadata["next_url"] = next_url + + return cachedjson("1h", metadata=metadata, results=results) # TODO caching + + +@api_private_blueprint.route("/vanilla_tor_stats", methods=["GET"]) +def api_private_vanilla_tor_stats() -> Response: + """Tor statistics over ASN for a given CC + --- + parameters: + - name: probe_cc + in: query + type: string + minLength: 2 + required: true + responses: + '200': + description: TODO + """ + probe_cc = validate_probe_cc_query_param() + blocked = 0 + nets = [] + s = """SELECT + countIf(msm_failure = 't') as failure_count, + toDate(MAX(measurement_start_time)) AS last_tested, + probe_asn, + COUNT() as total_count, + total_count - countIf(anomaly = 't') AS success_count + FROM fastpath + WHERE measurement_start_time > today() - INTERVAL 6 MONTH + AND probe_cc = :probe_cc + GROUP BY probe_asn + """ + q = query_click(sql.text(s), {"probe_cc": probe_cc}) + extras = { + "test_runtime_avg": None, + "test_runtime_max": None, + "test_runtime_min": None, + } + for n in q: + n.update(extras) + nets.append(n) + if n["total_count"] > 5: + if float(n["success_count"]) / float(n["total_count"]) < 0.6: + blocked += 1 + + if not nets: + return cachedjson("0s", networks=[], notok_networks=0, last_tested=None) + + lt = max(n["last_tested"] for n in nets) + return cachedjson("0s", networks=nets, notok_networks=blocked, last_tested=lt) + + +@api_private_blueprint.route("/im_networks", methods=["GET"]) +def api_private_im_networks() -> Response: + """Instant messaging networks statistics + --- + responses: + '200': + description: TODO + """ + probe_cc = validate_probe_cc_query_param() + s = """SELECT + COUNT() AS total_count, + '' AS name, + toDate(MAX(measurement_start_time)) AS last_tested, + probe_asn, + test_name + FROM fastpath + WHERE measurement_start_time >= today() - interval 31 day + AND measurement_start_time < today() + AND probe_cc = :probe_cc + AND test_name IN :test_names + GROUP BY test_name, probe_asn + ORDER BY test_name ASC, total_count DESC + """ + results: Dict[str, Dict] = {} + test_names = sorted(TEST_GROUPS["im"]) + q = query_click(sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) + for r in q: + e = results.get( + r["test_name"], + { + "anomaly_networks": [], + "ok_networks": [], + "last_tested": r["last_tested"], + }, + ) + e["ok_networks"].append( + { + "asn": r["probe_asn"], + "name": "", + "total_count": r["total_count"], + "last_tested": r["last_tested"], + } + ) + if e["last_tested"] < r["last_tested"]: + e["last_tested"] = r["last_tested"] + results[r["test_name"]] = e + + return cachedjson("1h", **results) # TODO caching + + +def isomid(d) -> str: + """Returns 2020-08-01T00:00:00+00:00""" + return f"{d}T00:00:00+00:00" + + +@api_private_blueprint.route("/im_stats", methods=["GET"]) +def api_private_im_stats() -> Response: + """Instant messaging statistics + --- + responses: + '200': + description: TODO + """ + test_name = request.args.get("test_name") + if not test_name or test_name not in TEST_GROUPS["im"]: + raise BadRequest("invalid test_name") + + probe_cc = validate_probe_cc_query_param() + probe_asn = validate_probe_asn_query_param() + probe_asn = int(probe_asn.replace("AS", "")) + + s = """SELECT + COUNT() as total_count, + toDate(measurement_start_time) AS test_day + FROM fastpath + WHERE probe_cc = :probe_cc + AND test_name = :test_name + AND probe_asn = :probe_asn + AND measurement_start_time >= today() - interval '31 day' + AND measurement_start_time < today() + GROUP BY test_day + ORDER BY test_day + """ + query_params = { + "probe_cc": probe_cc, + "probe_asn": probe_asn, + "test_name": test_name, + } + q = query_click(sql.text(s), query_params) + tmp = {r["test_day"]: r for r in q} + results = [] + days = [date.today() + timedelta(days=(d - 31)) for d in range(32)] + for d in days: + if d in tmp: + test_day = isomid(tmp[d]["test_day"]) + total_count = tmp[d]["total_count"] + else: + test_day = isomid(d) + total_count = 0 + e = dict(anomaly_count=None, test_day=test_day, total_count=total_count) + results.append(e) + + return cachedjson("1h", results=results) # TODO caching + + +@api_private_blueprint.route("/network_stats", methods=["GET"]) +def api_private_network_stats() -> Response: + """Network speed statistics - not implemented + --- + parameters: + - name: probe_cc + in: query + type: string + description: The two letter country code + responses: + '200': + description: TODO + """ + # TODO: implement the stats from NDT in fastpath and then here + probe_cc = validate_probe_cc_query_param() + + return cachedjson( + "1d", + { + "metadata": { + "current_page": 1, + "limit": 10, + "next_url": None, + "offset": 0, + "total_count": 0, + }, + "results": [], + }, + ) + + # Sample: + # { + # "metadata": { + # "current_page": 1, + # "limit": 10, + # "next_url": "https://api.ooni.io/api/_/network_stats?probe_cc=IT&offset=10&limit=10", + # "offset": 0, + # "total_count": 238, + # }, + # "results": [ + # { + # "asn": 3269, + # "asn_name": "ASN-IBSNAZ", + # "download_speed_mbps_median": 12.154, + # "middlebox_detected": null, + # "msm_count": 24596.0, + # "rtt_avg": 87.826, + # "upload_speed_mbps_median": 2.279, + # }, + # ... + # ], + # } + + +@api_private_blueprint.route("/country_overview", methods=["GET"]) +def api_private_country_overview() -> Response: + """Country-specific overview + --- + responses: + '200': + description: { + "first_bucket_date":"2012-12-01", + "measurement_count":6659891, + "network_count":333} + """ + # TODO: add circumvention_tools_blocked im_apps_blocked + # middlebox_detected_networks websites_confirmed_blocked + probe_cc = validate_probe_cc_query_param() + s = """SELECT + toDate(MIN(measurement_start_time)) AS first_bucket_date, + COUNT() AS measurement_count, + COUNT(DISTINCT probe_asn) AS network_count + FROM fastpath + WHERE probe_cc = :probe_cc + AND measurement_start_time > '2012-12-01' + """ + r = query_click_one_row(sql.text(s), {"probe_cc": probe_cc}) + assert r + # FIXME: websites_confirmed_blocked + return cachedjson("1d", **r) + + +@api_private_blueprint.route("/global_overview", methods=["GET"]) +def api_private_global_overview() -> Response: + """Provide global summary of measurements + Sources: global_stats db table + --- + responses: + '200': + description: JSON struct TODO + """ + q = """SELECT + COUNT(DISTINCT(probe_asn)) AS network_count, + COUNT(DISTINCT probe_cc) AS country_count, + COUNT(*) AS measurement_count + FROM fastpath + """ + r = query_click_one_row(q, {}) + assert r + return cachedjson("1d", **r) + + +@api_private_blueprint.route("/global_overview_by_month", methods=["GET"]) +def api_private_global_by_month() -> Response: + """Provide global summary of measurements + Sources: global_by_month db table + --- + responses: + '200': + description: JSON struct TODO + """ + q = """SELECT + COUNT(DISTINCT probe_asn) AS networks_by_month, + COUNT(DISTINCT probe_cc) AS countries_by_month, + COUNT() AS measurements_by_month, + toStartOfMonth(measurement_start_time) AS month + FROM fastpath + WHERE measurement_start_time > toStartOfMonth(today() - interval 2 year) + AND measurement_start_time < toStartOfMonth(today() + interval 1 month) + GROUP BY month ORDER BY month + """ + rows = query_click(sql.text(q), {}) + rows = list(rows) + + n = [{"date": r["month"], "value": r["networks_by_month"]} for r in rows] + c = [{"date": r["month"], "value": r["countries_by_month"]} for r in rows] + m = [{"date": r["month"], "value": r["measurements_by_month"]} for r in rows] + expand_dates(n) + expand_dates(c) + expand_dates(m) + return cachedjson( + "1d", networks_by_month=n, countries_by_month=c, measurements_by_month=m + ) + + +@api_private_blueprint.route("/circumvention_stats_by_country") +def api_private_circumvention_stats_by_country() -> Response: + """Aggregated statistics on protocols used for circumvention, + grouped by country. + --- + responses: + 200: + description: List of dicts with keys probe_cc and cnt + """ + q = """SELECT probe_cc, COUNT(*) as cnt + FROM fastpath + WHERE measurement_start_time > today() - interval 6 month + AND measurement_start_time < today() - interval 1 day + AND test_name IN ['torsf', 'tor', 'stunreachability', 'psiphon','riseupvpn'] + GROUP BY probe_cc ORDER BY probe_cc + """ + try: + result = query_click(sql.text(q), {}) + return cachedjson("1d", v=0, results=result) + + except Exception as e: + return cachedjson("0d", v=0, error=str(e)) + + +def pivot_circumvention_runtime_stats(rows): + # "pivot": create a datapoint for each probe_cc/test_name/date + tmp = {} + test_names = set() + ccs = set() + dates = set() + for r in rows: + k = (r["test_name"], r["probe_cc"], r["date"]) + tmp[k] = (r["p50"], r["p90"], r["cnt"]) + test_names.add(k[0]) + ccs.add(k[1]) + dates.add(k[2]) + + dates = sorted(dates) + ccs = sorted(ccs) + test_names = sorted(test_names) + no_data = () + result = [ + dict( + test_name=k[0], + probe_cc=k[1], + date=k[2], + v=tmp.get(k, no_data), + ) + for k in product(test_names, ccs, dates) + ] + return result + + +@api_private_blueprint.route("/circumvention_runtime_stats") +def api_private_circumvention_runtime_stats() -> Response: + """Runtime statistics on protocols used for circumvention, + grouped by date, country, test_name. + --- + responses: + 200: + description: List of dicts with keys probe_cc and cnt + """ + q = """SELECT + toDate(measurement_start_time) AS date, + test_name, + probe_cc, + quantile(.5)(JSONExtractFloat(scores, 'extra', 'test_runtime')) AS p50, + quantile(.9)(JSONExtractFloat(scores, 'extra', 'test_runtime')) AS p90, + count() as cnt + FROM fastpath + WHERE test_name IN ['torsf', 'tor', 'stunreachability', 'psiphon','riseupvpn'] + AND measurement_start_time > today() - interval 6 month + AND measurement_start_time < today() - interval 1 day + AND JSONHas(scores, 'extra', 'test_runtime') + GROUP BY date, probe_cc, test_name + """ + try: + r = query_click(sql.text(q), {}) + result = pivot_circumvention_runtime_stats(r) + return cachedjson("1d", v=0, results=result) + + except Exception as e: + return jerror(str(e), v=0) + + +@api_private_blueprint.route("/domain_metadata") +def api_private_domain_metadata() -> Response: + """Return the primary category code of a certain domain_name and its + canonical representation. + We consider the primary category code to be the category code of whatever is + the shortest URL in the test lists giving higher priority to what is in the + global list (e.g. if we have https://twitter.com/amnesty categorised as HUMR + and https://twitter.com/ as GRP, we will be picking GRP as the canonical + category code). + The canonical representation of a domain is whatever is present in the + global list. If it's not present, then the shortest possible + representation in the other test lists. + Some research related to this problem was done here: + https://gist.github.com/hellais/fab319ae20b0ccca7b548a060ed66e14, where some + notes were taken on what fixes need to be done in the test-lists to ensure + all of this works as expected (ex. moving shortest URL representations from + the country lists into the global list). + + Returns: + { + "category_code": "CITIZENLAB_CATEGORY_CODE", + "canonical_domain": "canonical.tld" + } + """ + category_code = "MISC" + domain = request.args.get("domain") + if domain is None: + raise BadRequest("missing domain") + + if domain.startswith("www."): + canonical_domain = domain[4:] + domains = [canonical_domain, domain] + else: + canonical_domain = domain + domains = [canonical_domain, "www." + domain] + + # case 1: domain with or without www is in the global list (cc = 'ZZ') + q = """ + SELECT category_code, domain + FROM citizenlab + WHERE domain IN :domains + AND cc = 'ZZ' + ORDER BY length(url) + LIMIT 1 + """ + res = query_click_one_row(sql.text(q), dict(domains=domains)) + if not res: + # case 2: domain only inside a country list, so we just select the + # shortest domain among the one with and without www + q = """ + SELECT category_code, domain FROM citizenlab + WHERE domain IN :domains + ORDER BY length(url) + LIMIT 1 + """ + res = query_click_one_row(sql.text(q), dict(domains=domains)) + + if res: + category_code = res["category_code"] + canonical_domain = res["domain"] + + j = dict(category_code=category_code, canonical_domain=canonical_domain) + return cachedjson("2h", **j) + + +@api_private_blueprint.route("/asnmeta") +def api_private_asnmeta() -> Response: + """Look up organization name by ASN + Sources: ansmeta db table + --- + parameters: + - name: asn + in: query + type: string + description: ASN + responses: + 200: + description: JSON object + """ + asn = request.args.get("asn") + if asn is None: + raise BadRequest("missing asn") + + q = """SELECT org_name + FROM asnmeta + WHERE asn = :asn + ORDER BY changed DESC + LIMIT 1 + """ + res = query_click_one_row(sql.text(q), dict(asn=asn)) + org_name = res["org_name"] if res else "Unknown" + return cachedjson("2h", org_name=org_name) + + +@api_private_blueprint.route("/networks") +def api_private_networks() -> Response: + """List all networks that have measurements + --- + responses: + 200: + description: JSON object + """ + q = """ + SELECT probe_asn, cnt, org_name FROM ( + SELECT + COUNT() as cnt, + probe_asn + FROM fastpath + GROUP BY probe_asn + ) as cnts + + LEFT JOIN ( + SELECT + any(org_name) as org_name, + asn + FROM ( + SELECT org_name, asn + FROM asnmeta + ORDER BY changed DESC + ) + GROUP BY asn + ) as asorgs + ON (asorgs.asn = cnts.probe_asn) + """ + try: + results = query_click(sql.text(q), {}) + return cachedjson("2h", v=0, results=results) + except Exception as e: + return jerror(str(e), v=0) + + +@api_private_blueprint.route("/domains") +def api_private_domains() -> Response: + """List all the domains in the test-lists with their measurement count + --- + responses: + 200: + description: JSON object + """ + # The nested ORDER BY lower(cc) puts global entries (cc=ZZ) on top so that + # any(category_code) picks it up as the most meaningful category code. + q = """ + SELECT domain AS domain_name, category_code, measurement_count + FROM ( + SELECT + any(category_code) as category_code, + domain FROM ( + SELECT domain, category_code + FROM citizenlab + ORDER BY lower(cc) != 'zz' + ) + GROUP BY domain + ) AS cz + LEFT JOIN ( + SELECT domain, count() AS measurement_count + FROM fastpath + WHERE test_name = 'web_connectivity' + GROUP BY domain + ) AS fp + ON (fp.domain == cz.domain) + ORDER BY domain_name + """ + try: + results = query_click(sql.text(q), {}) + return cachedjson("2h", v=0, results=results) + except Exception as e: + return jerror(str(e), v=0) + + +@api_private_blueprint.route("/check-in", methods=["POST"]) +def private_api_check_in() -> Response: + """Private, experimental check-in + --- + produces: + - application/json + consumes: + - application/json + parameters: + - in: body + name: probe self-description + required: true + schema: + type: object + properties: + probe_cc: + type: string + description: Two letter, uppercase country code + example: IT + probe_asn: + type: string + description: ASN, two uppercase letters followed by number + example: AS1234 + platform: + type: string + example: android + software_name: + type: string + example: ooniprobe + software_version: + type: string + example: 0.0.1 + on_wifi: + type: boolean + charging: + description: set only on devices with battery; true when charging + type: boolean + run_type: + type: string + description: timed or manual + example: timed + web_connectivity: + type: object + properties: + category_codes: + description: List/array of URL categories, all uppercase + type: array + items: + type: string + example: NEWS + description: probe_asn and probe_cc are not provided if unknown + + responses: + '200': + description: Give a URL test list to a probe running web_connectivity + tests; additional data for other tests; + schema: + type: object + properties: + v: + type: integer + description: response format version + probe_cc: + type: string + description: probe CC inferred from GeoIP or ZZ + probe_asn: + type: string + description: probe ASN inferred from GeoIP or AS0 + probe_network_name: + type: string + description: probe network name inferred from GeoIP or None + utc_time: + type: string + description: current UTC time as YYYY-mm-ddTHH:MM:SSZ + conf: + type: object + description: auxiliary configuration parameters + features: + type: object + description: feature flags + nettests: + type: array + items: + type: object + description: test-specific configuration + properties: + test_name: string + report_id: string + targets: + type: array + items: + type: object + properties: + attributes: + type: object + input: string + options: + type: object + + """ + log = current_app.logger + # TODO: Implement throttling + data = req_json() + run_type = data.get("run_type", "timed") + charging = data.get("charging", True) + probe_cc = data.get("probe_cc", "ZZ").upper() + probe_asn = data.get("probe_asn", "AS0") + + resp, probe_cc, asn_i = probe_geoip(probe_cc, probe_asn) + resp["v"] = 2 + + # On run_type=manual preserve the old behavior: test the whole list + # On timed runs test few URLs, especially when on battery + if run_type == "manual": + url_limit = 9999 # same as prio.py + elif charging: + url_limit = 100 + else: + url_limit = 20 + + if "web_connectivity" in data: + catcodes = data["web_connectivity"].get("category_codes", []) + if isinstance(catcodes, str): + category_codes = catcodes.split(",") + else: + category_codes = catcodes + + else: + category_codes = [] + + for c in category_codes: + assert c.isalpha() + + try: + webconn_test_items, _1, _2 = generate_test_list( + probe_cc, category_codes, asn_i, url_limit, False + ) + except Exception as e: + log.error(e, exc_info=True) + # TODO: use same failover as prio.py:list_test_urls + # failover_generate_test_list runs without any database interaction + # test_items = failover_generate_test_list(country_code, category_codes, limit) + webconn_test_items = [] + + metrics.gauge("check-in-test-list-count", len(webconn_test_items)) + conf: Dict[str, Any] = dict(features={}) + + # set webconnectivity_0.5 feature flag for some probes + octect = extract_probe_ipaddr_octect(1, 0) + if octect in (34, 239): + conf["features"]["webconnectivity_0.5"] = True + + conf["test_helpers"] = generate_test_helpers_conf() + + resp["conf"] = conf + resp["utc_time"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + + test_names = ( + "bridge_reachability", + "dash", + "dns_consistency", + "dnscheck", + "dnsping", + "echcheck", + "facebook_messenger", + "http_header_field_manipulation", + "http_host", + "http_invalid_request_line", + "http_requests", + "meek_fronted_requests_test", + "multi_protocol_traceroute", + "ndt", + "portfiltering", + "psiphon", + "quicping", + "riseupvpn", + "signal", + "simplequicping", + "sniblocking", + "stunreachability", + "tcp_connect", + "tcpping", + "telegram", + "tlsmiddlebox", + "tlsping", + "tor", + "torsf", + "urlgetter", + "vanilla_tor", + "web_connectivity", + "whatsapp", + ) + resp["nettests"] = [] + # Add one example for each nettest + for tn in test_names: + rid = generate_report_id(tn, probe_cc, asn_i) + targets = [] + if tn == "web_connectivity": + for d in webconn_test_items: + tgt = { + "attributes": { + "category_code": d["category_code"], + "country_code": d["country_code"], + }, + "input": d["url"], + "options": {}, + } + targets.append(tgt) + elif tn == "dnscheck": + for url in dnscheck_inputs: + tgt = {"attributes": {}, "input": url, "options": {}} + targets.append(tgt) + elif tn == "stunreachability": + for url in stunreachability_inputs: + tgt = {"attributes": {}, "input": url, "options": {}} + targets.append(tgt) + + block = {"test_name": tn, "report_id": rid, "targets": targets} + resp["nettests"].append(block) + + return nocachejson(**resp) From 284d89f8ce0b33160bda83e88ddb99ab5c8a5888 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:10:45 +0200 Subject: [PATCH 002/105] add private api router, logger --- ooniapi/services/ooniprobe/src/ooniprobe/main.py | 3 ++- .../services/ooniprobe/src/ooniprobe/routers/private.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/main.py b/ooniapi/services/ooniprobe/src/ooniprobe/main.py index c44041e8a..753b99703 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/main.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/main.py @@ -22,7 +22,7 @@ from .common.version import get_build_label from .dependencies import PostgresSessionDep, S3ClientDep, get_manifest from .download_geoip import try_update -from .routers import bouncer, prio_crud, reports +from .routers import bouncer, prio_crud, reports, private from .routers.v1 import probe_services from .routers.v2 import vpn @@ -92,6 +92,7 @@ def update_geoip_task(): app.include_router(reports.router) app.include_router(bouncer.router) app.include_router(prio_crud.router, prefix="/api") +app.include_router(private.router, prefix="/api") @app.get("/version") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 6d6a63756..c1b274f80 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -36,14 +36,19 @@ generate_report_id, ) + +from ..common.routers import BaseModel +from fastapi import APIRouter, Header, Request, Response + + # The private API is exposed under the prefix /api/_ # e.g. https://api.ooni.io/api/_/test_names +router = APIRouter(prefix="/_") -api_private_blueprint = Blueprint("api_private", "measurements") +log = logging.getLogger(__name__) # TODO: configure tags for HTTP caching across where useful -log = logging.getLogger() def daterange(start_date, end_date): From 0fa6b0de97a2483e186f3941bb42834880baa544 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:13:08 +0200 Subject: [PATCH 003/105] remove current_app.logger --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index c1b274f80..de1211c6d 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1128,7 +1128,6 @@ def private_api_check_in() -> Response: type: object """ - log = current_app.logger # TODO: Implement throttling data = req_json() run_type = data.get("run_type", "timed") From b53646924872c4bab08de9a7f97ae278d5d3bd7f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:16:43 +0200 Subject: [PATCH 004/105] migrate api/_/asn_by_month --- .../ooniprobe/src/ooniprobe/routers/private.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index de1211c6d..bee84cf41 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -77,8 +77,16 @@ def expand_dates(li): i["date"] = i["date"].strftime("%Y-%m-%dT00:00:00+00:00") -@api_private_blueprint.route("/asn_by_month") -def api_private_asn_by_month() -> Response: +class ASNCount(BaseModel): + date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") + value: int = Field(..., description="Count of unique ASN seen.") + + +ASNByMonthResponse = List[ASNCount] + + +@router.get("/asn_by_month", tags=["private_api"], response_model=ASNByMonthResponse) +def api_private_asn_by_month() -> ASNByMonthResponse: """Network count by month --- responses: @@ -95,7 +103,8 @@ def api_private_asn_by_month() -> Response: """ li = list(query_click(q, {})) expand_dates(li) - return cachedjson("1d", li) + validated: ASNByMonthResponse = [ASNCount(**item) for item in li] + return validated @api_private_blueprint.route("/countries_by_month") From db3ff78484941a0efc7de24a69686e475eaf93b5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:22:23 +0200 Subject: [PATCH 005/105] migrate api/_/countries_by_month --- .../ooniprobe/src/ooniprobe/routers/private.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index bee84cf41..44a7bdc4f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -107,7 +107,15 @@ def api_private_asn_by_month() -> ASNByMonthResponse: return validated -@api_private_blueprint.route("/countries_by_month") +class CountryCount(BaseModel): + date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") + value: int = Field(..., description="Count of unique countries seen.") + + +CountryByMonthResponse = List[ASNCount] + + +@router.get("/countries_by_month", tags=["private_api"], response_model=CountryByMonthResponse) def api_private_countries_by_month() -> Response: """Countries count by month --- @@ -125,7 +133,8 @@ def api_private_countries_by_month() -> Response: """ li = list(query_click(q, {})) expand_dates(li) - return cachedjson("1d", li) + validated: CountryByMonthResponse = [CountryCount(**item) for item in li] + return validated @api_private_blueprint.route("/test_names", methods=["GET"]) From 2948eb70fb58cdaba83e93ff7e6e54debb09fe1e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:29:08 +0200 Subject: [PATCH 006/105] migrate api/_/test_names --- .../ooniprobe/src/ooniprobe/routers/private.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 44a7bdc4f..e68be0cc7 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -136,8 +136,13 @@ def api_private_countries_by_month() -> Response: validated: CountryByMonthResponse = [CountryCount(**item) for item in li] return validated +class TestName(BaseModel): + id: str = Field(..., description="test id") + name: str = Field(..., description="test name") -@api_private_blueprint.route("/test_names", methods=["GET"]) +TestNamesResponse: List[TestName] + +@router.get("/test_names", tags=["private_api"], response_model=TestNamesResponse) def api_private_test_names() -> Response: """Provides test names and descriptions to Explorer --- @@ -172,8 +177,8 @@ def api_private_test_names() -> Response: "web_connectivity": "Web Connectivity", "whatsapp": "WhatsApp", } - test_names = [{"id": k, "name": v} for k, v in TEST_NAMES.items()] - return cachedjson("1h", test_names=test_names) + validated: TestNamesResponse = [TestName(id=k, name=v) for k, v in TEST_NAMES.items()] + return validated @api_private_blueprint.route("/countries", methods=["GET"]) From 1c63ab438b58563e4921ac301e20c0f9b74e0149 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:36:18 +0200 Subject: [PATCH 007/105] migrate api/_/countries --- .../src/ooniprobe/routers/private.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index e68be0cc7..e55902ab5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -39,6 +39,7 @@ from ..common.routers import BaseModel from fastapi import APIRouter, Header, Request, Response +from pydantic_extra_types.country import CountryAlpha2 # The private API is exposed under the prefix /api/_ @@ -181,8 +182,17 @@ def api_private_test_names() -> Response: return validated -@api_private_blueprint.route("/countries", methods=["GET"]) -def api_private_countries() -> Response: +class CountryStat(BaseModel): + alpha_2: CountryAlpha2 = Field(..., description="Country Code") + count: int = Field(..., description="Measurement count") + name: str = Field(..., description="Country Name") + + +AllCountryStats : List[CountryStat] + + +@router.get("/countries", tags=["private_api"], response_model=AllCountryStats) +def api_private_countries() -> AllCountryStats: """Summary of countries --- responses: @@ -200,12 +210,13 @@ def api_private_countries() -> Response: try: name = lookup_country(r["probe_cc"]) c.append( - dict(alpha_2=r["probe_cc"], name=name, count=r["measurement_count"]) + CountryStat(alpha_2=r["probe_cc"], name=name, count=r["measurement_count"]) ) except KeyError: pass - return cachedjson("1d", countries=c) + validated: AllCountryStats = c + return validated @api_private_blueprint.route("/quotas_summary", methods=["GET"]) From 4b629c9b7e6ef46f61e6132b6871df3bcd92d291 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:39:32 +0200 Subject: [PATCH 008/105] import from clickhouse_utils --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index e55902ab5..8b756620e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -24,7 +24,6 @@ from ooniapi.config import metrics from ooniapi.countries import lookup_country from ooniapi.data import dnscheck_inputs, stunreachability_inputs -from ooniapi.database import query_click, query_click_one_row from ooniapi.models import TEST_GROUPS from ooniapi.prio import generate_test_list from ooniapi.utils import cachedjson, nocachejson, jerror, req_json @@ -38,6 +37,7 @@ from ..common.routers import BaseModel +from ..common.clickhouse_utils import query_click, query_click_one_row from fastapi import APIRouter, Header, Request, Response from pydantic_extra_types.country import CountryAlpha2 From 430b5cd225f550b8d060c6bc4828d7066c8ac9aa Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:47:43 +0200 Subject: [PATCH 009/105] TODO: compete. start migration of quotas_summary --- .../src/ooniprobe/routers/private.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 8b756620e..0c89e0a6c 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -36,11 +36,13 @@ ) -from ..common.routers import BaseModel -from ..common.clickhouse_utils import query_click, query_click_one_row -from fastapi import APIRouter, Header, Request, Response +from fastapi import APIRouter, Depends, Header, Request, Response from pydantic_extra_types.country import CountryAlpha2 +from ..common.clickhouse_utils import query_click, query_click_one_row +from ..common.dependencies import role_required +from ..common.routers import BaseModel + # The private API is exposed under the prefix /api/_ # e.g. https://api.ooni.io/api/_/test_names @@ -219,13 +221,20 @@ def api_private_countries() -> AllCountryStats: return validated -@api_private_blueprint.route("/quotas_summary", methods=["GET"]) -@role_required(["admin"]) +@router.get( + "/quotas_summary", + response_model=AllCountryStats, + tags=["private_api"], + dependencies=[Depends(role_required(["admin"]))], +) def api_private_quotas_summary() -> Response: """Summary on rate-limiting quotas. [(first ipaddr octet, remaining daily quota), ... ] """ - return nocachejson(current_app.limiter.get_lowest_daily_quotas_summary()) + # XXX: add limiter to ooniapi + #return nocachejson(current_app.limiter.get_lowest_daily_quotas_summary()) + raise NotImplemented + @api_private_blueprint.route("/check_report_id", methods=["GET"]) From dcd788e7314af37dd5cf73804921f1a9b5ca7ebd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 12:48:29 +0200 Subject: [PATCH 010/105] migrate api/_/check_report_id --- .../ooniprobe/src/ooniprobe/routers/private.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 0c89e0a6c..7c295a449 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -236,9 +236,16 @@ def api_private_quotas_summary() -> Response: raise NotImplemented +class CheckReportIDResponse(BaseModel): + v: int = Field(..., description="version number of this response") + found: bool = Field(..., description="Report found") -@api_private_blueprint.route("/check_report_id", methods=["GET"]) -def check_report_id() -> Response: + +@router.get("/check_report_id", + response_model=CheckReportIDResponse, + tags=["private_api"], +) +def check_report_id() -> CheckReportIDResponse: """Legacy. Used to check if a report_id existed in the fastpath table. Used by https://github.com/ooni/probe/issues/1034 --- @@ -263,7 +270,7 @@ def check_report_id() -> Response: example: { "found": true, "v": 0 } """ - return cachedjson("1s", v=0, found=True) + return CheckReportIDResponse(v=0, found=true) def last_30days(begin=31, end=1): From 48a6c44354aa21591918a1d03593e1288dc1915b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 13:08:24 +0200 Subject: [PATCH 011/105] TODO: fix: migrate api/_/test_coverage --- .../ooniprobe/src/ooniprobe/routers/private.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7c295a449..10028f269 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -364,8 +364,16 @@ def get_recent_network_coverage_ch(probe_cc, test_groups): return query_click(sql.text(s), d) -@api_private_blueprint.route("/test_coverage", methods=["GET"]) -def api_private_test_coverage() -> Response: +class TestCoverageResponse(BaseModel): + network_coverage: List[Any] + test_coverage: List[Any] + + +@router.get("/test_coverage", response_model=TestCoverageResponse, tags=["private"]) +def api_private_test_coverage( + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + test_groups: str = Field(None, description="XXX: What is this?") +) -> TestCoverageResponse: """Return number of measurements per day across test categories --- parameters: @@ -380,14 +388,13 @@ def api_private_test_coverage() -> Response: """ # TODO: merge the two queries into one? # TODO: remove test categories or move aggregation to the front-end? - probe_cc = validate_probe_cc_query_param() - test_groups = request.args.get("test_groups") if test_groups is not None: test_groups = test_groups.split(",") tc = get_recent_test_coverage_ch(probe_cc) nc = get_recent_network_coverage_ch(probe_cc, test_groups) - return cachedjson("1h", network_coverage=nc, test_coverage=tc) + validated = TestCoverageResponse(network_coverage=nc, test_coverage=tc) + return validated @api_private_blueprint.route("/website_networks", methods=["GET"]) From 20ef3da5f3cbe84a3b4940712faca0284aad0aff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 13:19:45 +0200 Subject: [PATCH 012/105] TODO: fix: migrate api/_/website_networks --- .../ooniprobe/src/ooniprobe/routers/private.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 10028f269..55db98e4f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -397,8 +397,18 @@ def api_private_test_coverage( return validated -@api_private_blueprint.route("/website_networks", methods=["GET"]) -def api_private_website_network_tests() -> Response: +class MeasurementsByASN(BaseModel): + count: int = Field(..., description="Number of measurments for eachnetworks in country") + probe_asn: str = Field(..., description="Autonomous System Number") + + +WebsiteNetworksResponse: List[MeasurementsByASN] + + +@router.get("/website_networks", response_model=WebsiteNetworksResponse, tags=["private"]) +def api_private_website_network_tests( + probe_cc: CountryAlpha2 = Query(..., description="Country Code") + ) -> WebsiteNetworksResponse: """TODO --- parameters: @@ -410,7 +420,6 @@ def api_private_website_network_tests() -> Response: '200': description: TODO """ - probe_cc = validate_probe_cc_query_param() s = """SELECT COUNT() AS count, probe_asn @@ -423,7 +432,8 @@ def api_private_website_network_tests() -> Response: ORDER BY count DESC """ results = query_click(sql.text(s), {"probe_cc": probe_cc}) - return cachedjson("0s", results=results) + validated: WebsiteNetworksResponse = [MeasurementsByASN(**x) for x in results] + return validated @api_private_blueprint.route("/website_stats", methods=["GET"]) From a570f0834287d1320aa5d3495699badc940087d5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 13:31:28 +0200 Subject: [PATCH 013/105] import fastapi Query, pydantic AnyUrl --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 55db98e4f..a988a503c 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -36,8 +36,9 @@ ) -from fastapi import APIRouter, Depends, Header, Request, Response +from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 +from pydantic import AnyUrl from ..common.clickhouse_utils import query_click, query_click_one_row from ..common.dependencies import role_required From 00b2ba3cdcd531228e17fa805294d20685cfc4b1 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 22 Apr 2026 13:31:51 +0200 Subject: [PATCH 014/105] squash: test_coverage fix query --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a988a503c..1dda392cf 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -373,7 +373,7 @@ class TestCoverageResponse(BaseModel): @router.get("/test_coverage", response_model=TestCoverageResponse, tags=["private"]) def api_private_test_coverage( probe_cc: CountryAlpha2 = Query(..., description="Country Code"), - test_groups: str = Field(None, description="XXX: What is this?") + test_groups: str = Query(None, description="XXX: What is this?") ) -> TestCoverageResponse: """Return number of measurements per day across test categories --- From 262800f21810d4d4f549c3d342343c16e1528229 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 09:23:39 +0200 Subject: [PATCH 015/105] migrate api/_/website_stats --- .../src/ooniprobe/routers/private.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1dda392cf..897996898 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -38,7 +38,7 @@ from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 -from pydantic import AnyUrl +from pydantic import AnyUrl, conint from ..common.clickhouse_utils import query_click, query_click_one_row from ..common.dependencies import role_required @@ -436,21 +436,32 @@ def api_private_website_network_tests( validated: WebsiteNetworksResponse = [MeasurementsByASN(**x) for x in results] return validated +class DayStats(BaseModel): + test_day: date + anomaly_count: conint(ge=0) + confirmed_count: conint(ge=0) + failure_count: conint(ge=0) + total_count: conint(ge=0) -@api_private_blueprint.route("/website_stats", methods=["GET"]) -def api_private_website_stats() -> Response: - """TODO +class WebsiteStatsResponse(BaseModel): + results: List[DayStats] + +@router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) +def api_private_website_stats( + input: AnyUrl = Query(..., "Website to query stats"), + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + probe_asn: int = Query(..., description="ASN (integer)"), +) -> Response: + """Returns daily aggregated stats for the given website, country and ASN --- responses: '200': - description: TODO + description: daily website stats """ # uses_pg_index counters_day_cc_asn_input_idx a BRIN index was not used at # all, but BTREE on (measurement_start_day, probe_cc, probe_asn, input) # made queries go from full scan to 50ms - url = request.args.get("input") - probe_cc = validate_probe_cc_query_param() - probe_asn = validate_probe_asn_query_param() + url = input s = """SELECT toDate(measurement_start_time) AS test_day, @@ -468,7 +479,7 @@ def api_private_website_stats() -> Response: """ d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} results = query_click(sql.text(s), d) - return cachedjson("0s", results=results) + return WebsiteStatsResponse(result=results) @api_private_blueprint.route("/website_urls", methods=["GET"]) From 9e547140848c9ee3b48a0e4cb1d80cdd3b9d002a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 09:36:53 +0200 Subject: [PATCH 016/105] migrate api/_/website_urls --- .../src/ooniprobe/routers/private.py | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 897996898..adc984c65 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -482,8 +482,32 @@ def api_private_website_stats( return WebsiteStatsResponse(result=results) -@api_private_blueprint.route("/website_urls", methods=["GET"]) -def api_private_website_test_urls() -> Response: +class WebsiteURLItem(BaseModel): + input: AnyUrl + anomaly_count: conint(ge=0) + confirmed_count: conint(ge=0) + failure_count: conint(ge=0) + total_count: conint(ge=0) + +class PaginationMetadata(BaseModel): + offset: int + limit: int + current_page: int + total_count: int + next_url: Optional[AnyUrl] = None + +class WebsiteURLsResponse(BaseModel): + metadata: PaginationMetadata + results: List[WebsiteURLItem] + + +@router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) +def api_private_website_test_urls( + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + probe_asn: str = Query(..., description="ASN, e.g. AS1234") + limit: int = Query(10, description="Limit results") + offset: int = Query(0, description="Offset results") +) -> WebsiteURLsResponse: """TODO --- responses: @@ -491,13 +515,9 @@ def api_private_website_test_urls() -> Response: description: TODO """ # TODO optimize or remove - limit = int(request.args.get("limit", 10)) if limit <= 0: limit = 10 - offset = int(request.args.get("offset", 0)) - probe_cc = validate_probe_cc_query_param() - probe_asn = validate_probe_asn_query_param() probe_asn = int(probe_asn.replace("AS", "")) # Count how many distinct inputs we have in this CC / ASN / period @@ -562,7 +582,8 @@ def api_private_website_test_urls() -> Response: ) metadata["next_url"] = next_url - return cachedjson("1h", metadata=metadata, results=results) # TODO caching + # validate and return + return WebsiteURLsResponse(metadata=metadata, results=results) @api_private_blueprint.route("/vanilla_tor_stats", methods=["GET"]) From 5c50c6761b756706763a451d5846ec67df7d3f0f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 09:58:07 +0200 Subject: [PATCH 017/105] migrate api/_/vanilla_tor_stats --- .../src/ooniprobe/routers/private.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index adc984c65..b3e345477 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -586,8 +586,27 @@ def api_private_website_test_urls( return WebsiteURLsResponse(metadata=metadata, results=results) -@api_private_blueprint.route("/vanilla_tor_stats", methods=["GET"]) -def api_private_vanilla_tor_stats() -> Response: +class NetworkStat(BaseModel): + failure_count: int + last_tested: Optional[date] # SQL toDate() returns a date + probe_asn: int + total_count: int + success_count: int + test_runtime_avg: Optional[float] = None + test_runtime_max: Optional[float] = None + test_runtime_min: Optional[float] = None + + +class TorStatsResponse(BaseModel): + last_tested: Optional[date] + networks: List[NetworkStat] + notok_networks: int + + +@router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) +def api_private_vanilla_tor_stats( + probe_cc: CountryAlpha2 = Query(..., description="Country Code") +) -> TorStatsResponse: """Tor statistics over ASN for a given CC --- parameters: @@ -600,7 +619,6 @@ def api_private_vanilla_tor_stats() -> Response: '200': description: TODO """ - probe_cc = validate_probe_cc_query_param() blocked = 0 nets = [] s = """SELECT @@ -628,10 +646,18 @@ def api_private_vanilla_tor_stats() -> Response: blocked += 1 if not nets: - return cachedjson("0s", networks=[], notok_networks=0, last_tested=None) + return TorStatsResponse( + last_tested=None, + networks=[], + notok_networks=0, + ) lt = max(n["last_tested"] for n in nets) - return cachedjson("0s", networks=nets, notok_networks=blocked, last_tested=lt) + return TorStatsResponse( + last_tested=lt, + networks=nets, + notok_networks=blocked, + ) @api_private_blueprint.route("/im_networks", methods=["GET"]) From 1d631b860d70cbebaa16c81ae3dc5b18d3ce5877 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 11:34:46 +0200 Subject: [PATCH 018/105] migrate api/_/im_networks --- .../src/ooniprobe/routers/private.py | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index b3e345477..0b6cc81dd 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -659,16 +659,32 @@ def api_private_vanilla_tor_stats( notok_networks=blocked, ) +class NetworkEntry(BaseModel): + asn: int + name: str + total_count: int + last_tested: date + + +class IMNetworkStats(BaseModel): + anomaly_networks: List[NetworkEntry] + ok_networks: List[NetworkEntry] + last_tested: date + -@api_private_blueprint.route("/im_networks", methods=["GET"]) -def api_private_im_networks() -> Response: +IMNetworksResponse: Dict[str, IMNetworksStats] + + +@router.get("/im_networks", response_model=IMNetworksResponse, tags=["private"]) +def api_private_im_networks( + probe_cc: CountryAlpha2 = Query(..., description="Country Code") +) -> IMNetworksResponse: """Instant messaging networks statistics --- responses: '200': description: TODO """ - probe_cc = validate_probe_cc_query_param() s = """SELECT COUNT() AS total_count, '' AS name, @@ -683,31 +699,27 @@ def api_private_im_networks() -> Response: GROUP BY test_name, probe_asn ORDER BY test_name ASC, total_count DESC """ - results: Dict[str, Dict] = {} - test_names = sorted(TEST_GROUPS["im"]) + test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] q = query_click(sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) + results = IMNetworksResponse() for r in q: - e = results.get( - r["test_name"], - { - "anomaly_networks": [], - "ok_networks": [], - "last_tested": r["last_tested"], - }, - ) - e["ok_networks"].append( - { - "asn": r["probe_asn"], - "name": "", - "total_count": r["total_count"], - "last_tested": r["last_tested"], - } - ) - if e["last_tested"] < r["last_tested"]: - e["last_tested"] = r["last_tested"] - results[r["test_name"]] = e + # get stats for test_name or create a new IMNetworksStats + stats = results.get(r["test_name"], IMNetworkStats(anomaly_networks=[], ok_networks=[], last_tested)) + + # create and add a new entry + entry = NetworkEntry(asn=r["probe_asn"], name="", total_count=r["total_count"], last_tested=r["last_tested"]) + stats.ok_networks.append(entry) + + # XXX: anomaly_networks appears unused + + # update last_tested if it is the latest measurement + if stats.last_tested < entry.last_tested: + stats.last_tested = entry.last_tested + + # save the object in results + results[stats.test_name] = stats - return cachedjson("1h", **results) # TODO caching + return results def isomid(d) -> str: From c87143fc8ba9ddca5271e763af08bfed730ab7d5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 12:12:03 +0200 Subject: [PATCH 019/105] migrate api/_/im_stats --- .../src/ooniprobe/routers/private.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 0b6cc81dd..7b6af977e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -727,21 +727,33 @@ def isomid(d) -> str: return f"{d}T00:00:00+00:00" -@api_private_blueprint.route("/im_stats", methods=["GET"]) -def api_private_im_stats() -> Response: +class IMStatsItem(BaseModel): + anomaly_count: Optional[int] = None + test_day: datetime + total_count: int + + +class IMStatsResponse(BaseModel): + results: List[IMStatsItem] + + +@router.get("/im_stats", response_model=IMStatsResponse, tags=["private"]) +def api_private_im_stats( + probe_asn: str = Query(..., description="ASN, e.g. AS1234"), + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + test_name: str = Query(..., description="Test name") +) -> IMStatsResponse: """Instant messaging statistics --- responses: '200': description: TODO """ - test_name = request.args.get("test_name") - if not test_name or test_name not in TEST_GROUPS["im"]: - raise BadRequest("invalid test_name") + test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] + if test_name not in test_names: + raise HTTPException(status_code=400, detail="Invalid test_name") - probe_cc = validate_probe_cc_query_param() - probe_asn = validate_probe_asn_query_param() - probe_asn = int(probe_asn.replace("AS", "")) + probe_asn = int(probe_asn.upper().replace("AS", "")) s = """SELECT COUNT() as total_count, @@ -762,7 +774,7 @@ def api_private_im_stats() -> Response: } q = query_click(sql.text(s), query_params) tmp = {r["test_day"]: r for r in q} - results = [] + items: List[IMStatsItem] = [] days = [date.today() + timedelta(days=(d - 31)) for d in range(32)] for d in days: if d in tmp: @@ -771,10 +783,10 @@ def api_private_im_stats() -> Response: else: test_day = isomid(d) total_count = 0 - e = dict(anomaly_count=None, test_day=test_day, total_count=total_count) - results.append(e) + items.append(IMStatsItem(anomaly_count=None, test_day=test_day, total_count=total_count)) - return cachedjson("1h", results=results) # TODO caching + response = IMStatsResponse(results=items) + return response @api_private_blueprint.route("/network_stats", methods=["GET"]) From 75b8d71bce7e1189cfbac87ca28211ca64ba5c05 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 12:27:08 +0200 Subject: [PATCH 020/105] migrate api/_/network_stats --- .../src/ooniprobe/routers/private.py | 67 ++++++++----------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7b6af977e..1bb31b40b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -789,8 +789,33 @@ def api_private_im_stats( return response -@api_private_blueprint.route("/network_stats", methods=["GET"]) -def api_private_network_stats() -> Response: +class NetworkStats(BaseModel): + asn: int = Field(..., description="ASN (int)") + asn_name: str = Field(..., description="Autonomous System Name") + download_speed_mbps_median: float = Field(description="Median download speed in megabits") + upload_speed_mbps_median: float = Field(description="Median upload speed in megabites") + middlebox_detected: bool = Field(..., description="Middlebox was detected") + msm_count: float = Field(..., description="FIXME: example: 24596.0") + rtt_avg: float = Field(..., description="Round Trip Time Average") + + +class NetworkMetadata(BaseModel): + current_page: int = Field(1, description="Current page") + limit: int = Field(10, description="Result limit") + next_url: Optional[AnyUrl] = Field(None, description="Next URL") + offset: int = Field(0, description="Result offset") + total_count: int = Field(0, description="Total results") + + +class NetworkStatsResponse(BaseModel): + metadata: NetworkMetadata = Field(..., description="Networks metadata") + results: List[NetworkStats] = Field(..., description="List of network stats") + + +@router.get("/network_stats", response_model=NetworkStatsResponse, tags=["private"]) +def api_private_network_stats( + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), +) -> NetworkStatsResponse: """Network speed statistics - not implemented --- parameters: @@ -803,44 +828,8 @@ def api_private_network_stats() -> Response: description: TODO """ # TODO: implement the stats from NDT in fastpath and then here - probe_cc = validate_probe_cc_query_param() - - return cachedjson( - "1d", - { - "metadata": { - "current_page": 1, - "limit": 10, - "next_url": None, - "offset": 0, - "total_count": 0, - }, - "results": [], - }, - ) - # Sample: - # { - # "metadata": { - # "current_page": 1, - # "limit": 10, - # "next_url": "https://api.ooni.io/api/_/network_stats?probe_cc=IT&offset=10&limit=10", - # "offset": 0, - # "total_count": 238, - # }, - # "results": [ - # { - # "asn": 3269, - # "asn_name": "ASN-IBSNAZ", - # "download_speed_mbps_median": 12.154, - # "middlebox_detected": null, - # "msm_count": 24596.0, - # "rtt_avg": 87.826, - # "upload_speed_mbps_median": 2.279, - # }, - # ... - # ], - # } + return NetworkStatsResponse(metadata=NetworkMetadata(), results=[]) @api_private_blueprint.route("/country_overview", methods=["GET"]) From 216f1f9d33cdf631a6d02d4c7ded147dc66476b0 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 12:33:27 +0200 Subject: [PATCH 021/105] migrate api/_/country_overview --- .../src/ooniprobe/routers/private.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1bb31b40b..7ddf6dd11 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -832,8 +832,15 @@ def api_private_network_stats( return NetworkStatsResponse(metadata=NetworkMetadata(), results=[]) -@api_private_blueprint.route("/country_overview", methods=["GET"]) -def api_private_country_overview() -> Response: +class CountryOverviewResponse(BaseModel): + first_bucket_date: date = Field(..., description="First bucket date YYYY-MM-DD") + measurement_count: int = Field(..., description="Number of measurements") + network_count: int = Field(..., description="Number of networks measured") + +@router.get("/country_overview", response_model=CountryOverviewResponse, tags=["private"]) +def api_private_country_overview( + probe_cc: CountryAlpha2 = Query(..., description="Country Code"), +) -> CountryOverviewResponse: """Country-specific overview --- responses: @@ -845,7 +852,6 @@ def api_private_country_overview() -> Response: """ # TODO: add circumvention_tools_blocked im_apps_blocked # middlebox_detected_networks websites_confirmed_blocked - probe_cc = validate_probe_cc_query_param() s = """SELECT toDate(MIN(measurement_start_time)) AS first_bucket_date, COUNT() AS measurement_count, @@ -856,8 +862,11 @@ def api_private_country_overview() -> Response: """ r = query_click_one_row(sql.text(s), {"probe_cc": probe_cc}) assert r - # FIXME: websites_confirmed_blocked - return cachedjson("1d", **r) + result = CountryOverviewResponse( + first_bucket_date=r["first_bucket_date"], + measurement_count=r["measurement_count"], + network_count=r["network_count"]) + return result @api_private_blueprint.route("/global_overview", methods=["GET"]) From da1cd6ef8c54eaf39cede427ec9456527e36f771 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 23 Apr 2026 12:36:57 +0200 Subject: [PATCH 022/105] migrate api/_/global_overview --- .../ooniprobe/src/ooniprobe/routers/private.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7ddf6dd11..1a72857fb 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -869,8 +869,14 @@ def api_private_country_overview( return result -@api_private_blueprint.route("/global_overview", methods=["GET"]) -def api_private_global_overview() -> Response: +class GlobalOverviewResponse(BaseModel): + network_count: int = Field(..., "Number of networks measured") + country_count: int = Field(..., "Nubmer of countries measured") + measurement_count: int = Field(..., "Number of total measurements") + + +@router.get("/global_overview", response_model=GlobalOverviewResponse, tags=["private"]) +def api_private_global_overview() -> GlobalOverviewResponse: """Provide global summary of measurements Sources: global_stats db table --- @@ -886,7 +892,11 @@ def api_private_global_overview() -> Response: """ r = query_click_one_row(q, {}) assert r - return cachedjson("1d", **r) + result = GlobalOverViewResponse( + network_count=r["network_count"], + country_count=r["country_count"], + measurement_count=r["measurement_count"]) + return result @api_private_blueprint.route("/global_overview_by_month", methods=["GET"]) From a893f4c9bca15641b8d5301c77947082a3198e51 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 07:19:36 +0200 Subject: [PATCH 023/105] migrate api/_/global_overview_by_month --- .../ooniprobe/src/ooniprobe/routers/private.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1a72857fb..cf5b716c3 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -899,7 +899,16 @@ def api_private_global_overview() -> GlobalOverviewResponse: return result -@api_private_blueprint.route("/global_overview_by_month", methods=["GET"]) +class GlobalOverviewStat(BaseModel): + date: datetime + value: int + +class GlobalOverviewMonthResponse(BaseModel): + networks_by_month: List[GlobalOverviewStat] + countries_by_month: List[GlobalOverviewStat] + measurement_by_month: List[GlobalOverviewStat] + +@router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) def api_private_global_by_month() -> Response: """Provide global summary of measurements Sources: global_by_month db table @@ -927,9 +936,8 @@ def api_private_global_by_month() -> Response: expand_dates(n) expand_dates(c) expand_dates(m) - return cachedjson( - "1d", networks_by_month=n, countries_by_month=c, measurements_by_month=m - ) + validated = GlobalOverviewMonthResponse(networks_by_month=n, countries_by_month=c, measurements_by_month=m) + return validated @api_private_blueprint.route("/circumvention_stats_by_country") From eb5303f430b29c3957dd16f5d712a93bde420618 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 07:56:59 +0200 Subject: [PATCH 024/105] migrate api/_/circumvention_stats_by_country --- .../ooniprobe/src/ooniprobe/routers/private.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index cf5b716c3..7282801d9 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -940,7 +940,17 @@ def api_private_global_by_month() -> Response: return validated -@api_private_blueprint.route("/circumvention_stats_by_country") +class CountryCircumventionStat(BaseModel): + cnt: conint(ge=0) = Field(..., description="Count of measurements") + probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") + + +class CircumventionStatsResponse(BaseModel): + results: Optional[List[CountryCircumventionStat]] = Field(None, description="List of per-country circumvention tool measurement counts over 6 months") + v: int = Field(..., description="API Response version") + + +@router.get("/circumvention_stats_by_country", response_model=CircumventionStatsResponse, tags=["private"]) def api_private_circumvention_stats_by_country() -> Response: """Aggregated statistics on protocols used for circumvention, grouped by country. @@ -958,10 +968,10 @@ def api_private_circumvention_stats_by_country() -> Response: """ try: result = query_click(sql.text(q), {}) - return cachedjson("1d", v=0, results=result) + return CountryCircumVentionStatsResponse(results=result) except Exception as e: - return cachedjson("0d", v=0, error=str(e)) + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) def pivot_circumvention_runtime_stats(rows): From c541aa70b39246d119ae9ee5af60d3efefefbbfd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 07:58:35 +0200 Subject: [PATCH 025/105] migrate api/_/circumvention_runtime_stats --- .../src/ooniprobe/routers/private.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7282801d9..7b119a793 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -9,7 +9,7 @@ from itertools import product from urllib.parse import urljoin, urlencode -from typing import Dict, Any +from typing import Dict, Any, Tuple import logging import math @@ -974,7 +974,14 @@ def api_private_circumvention_stats_by_country() -> Response: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) -def pivot_circumvention_runtime_stats(rows): +class CircumventionRuntimeStat(BaseModel): + test_name: str = Field(..., description="Name of test") + probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") + date: date = Field(..., description="Date of metric") + v: Tuple[float, float, int] = Field((), description="(p50, p90, cnt)") + + +def pivot_circumvention_runtime_stats(rows) -> List[CircumventionRuntimeStat]: # "pivot": create a datapoint for each probe_cc/test_name/date tmp = {} test_names = set() @@ -992,18 +999,18 @@ def pivot_circumvention_runtime_stats(rows): test_names = sorted(test_names) no_data = () result = [ - dict( - test_name=k[0], - probe_cc=k[1], - date=k[2], - v=tmp.get(k, no_data), - ) + RuntimeStat(test_name=k[0], probe_cc=k[1], date=k[2], v=tmp.get(k, no_data)) for k in product(test_names, ccs, dates) ] return result -@api_private_blueprint.route("/circumvention_runtime_stats") +class CircumventionRuntimeStatsResponse(BaseModel): + results: List[CircumventionRuntimeStat] + v: int = Field(..., description="Version of API response") + + +@router.get("/circumvention_runtime_stats", response_model=CircumventionRuntimeStatsResponse, tags=["private"]) def api_private_circumvention_runtime_stats() -> Response: """Runtime statistics on protocols used for circumvention, grouped by date, country, test_name. @@ -1028,11 +1035,10 @@ def api_private_circumvention_runtime_stats() -> Response: """ try: r = query_click(sql.text(q), {}) - result = pivot_circumvention_runtime_stats(r) - return cachedjson("1d", v=0, results=result) + return CircumventionRuntimeStatsResponse(results=pivot_circumvention_runtime_stats(r), v=0) except Exception as e: - return jerror(str(e), v=0) + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) @api_private_blueprint.route("/domain_metadata") From 4d9717d9e9acc26ca7a3426c90dd30a9890eed6e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 08:09:43 +0200 Subject: [PATCH 026/105] migrate api/_/domain_metadata --- .../src/ooniprobe/routers/private.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7b119a793..85d0b57eb 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -38,7 +38,7 @@ from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 -from pydantic import AnyUrl, conint +from pydantic import AnyUrl, conint, constr from ..common.clickhouse_utils import query_click, query_click_one_row from ..common.dependencies import role_required @@ -1041,8 +1041,21 @@ def api_private_circumvention_runtime_stats() -> Response: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) -@api_private_blueprint.route("/domain_metadata") -def api_private_domain_metadata() -> Response: +DomainStr = constr( + strip_whitespace=True, + regex=r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[A-Za-z]{2,}$" +) + + +class DomainMetadataResponse(BaseModel): + canonical_domain: DomainStr = Field(..., description="Domain name") + category_code: str = Field(..., description="Citizenlab Category Code") + + +@router.get("/domain_metadata", response_model=DomainMetadataResponse, tags=["private"]) +def api_private_domain_metadata( + domain: DomainStr = Query(..., description="Domain Name"), +) -> DomainMetadataResponse: """Return the primary category code of a certain domain_name and its canonical representation. We consider the primary category code to be the category code of whatever is @@ -1102,8 +1115,7 @@ def api_private_domain_metadata() -> Response: category_code = res["category_code"] canonical_domain = res["domain"] - j = dict(category_code=category_code, canonical_domain=canonical_domain) - return cachedjson("2h", **j) + return DomainMetadataResponse(category_code=category_code, canonical_domain=canonical_domain) @api_private_blueprint.route("/asnmeta") From ec9c5901f62dcf024ca56a50ab1f4272707a2ba6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 08:13:33 +0200 Subject: [PATCH 027/105] migrate api/_/asnmeta --- .../ooniprobe/src/ooniprobe/routers/private.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 85d0b57eb..401487ed1 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1118,8 +1118,14 @@ def api_private_domain_metadata( return DomainMetadataResponse(category_code=category_code, canonical_domain=canonical_domain) -@api_private_blueprint.route("/asnmeta") -def api_private_asnmeta() -> Response: +class ASNMetadataResponse(BaseModel): + org_name: string = Field("Unknown", description="ORG Name of ASN") + + +@router.get("/asnmeta", response_model=ASNMetadataResponse, tags=["private"]) +def api_private_asnmeta( + asn: int = Query(..., description="Autonomous System Number, e.g. 1234"), +) -> ASNMetadataResponse: """Look up organization name by ASN Sources: ansmeta db table --- @@ -1144,7 +1150,7 @@ def api_private_asnmeta() -> Response: """ res = query_click_one_row(sql.text(q), dict(asn=asn)) org_name = res["org_name"] if res else "Unknown" - return cachedjson("2h", org_name=org_name) + return ASNMetadataResponse(org_name=org_name) @api_private_blueprint.route("/networks") From 65f4ce70f4fcea001b87af5b1ddc5fb30e6ee9cc Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 08:23:20 +0200 Subject: [PATCH 028/105] migrate api/_/networks --- .../ooniprobe/src/ooniprobe/routers/private.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 401487ed1..9dd731e53 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1153,7 +1153,18 @@ def api_private_asnmeta( return ASNMetadataResponse(org_name=org_name) -@api_private_blueprint.route("/networks") +class MeasuredNetworkStat(BaseModel): + cnt: conint(ge=0) = Field(..., description="Number of measurements") + org_name: str = Field("", description="ORG Name of network") + probe_asn: conint(ge=0) = Field(..., description="ASN of network (int)") + + +class MeasuredNetworksResponse(BaseModel): + results: List[MeasuredNetworkStat] = Field(..., description="Networks that have measurements") + v: int = Field(..., description="Version of API response") + + +@router.get("/networks", response_model=MeasuredNetworksResponse, tags=["private"]) def api_private_networks() -> Response: """List all networks that have measurements --- @@ -1185,9 +1196,9 @@ def api_private_networks() -> Response: """ try: results = query_click(sql.text(q), {}) - return cachedjson("2h", v=0, results=results) + return MeasuredNetworksResponse(results=results v=0) except Exception as e: - return jerror(str(e), v=0) + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) @api_private_blueprint.route("/domains") From 1684c0b08308407d9d5d90cf9c485e5976bef46b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 08:24:19 +0200 Subject: [PATCH 029/105] migrate api/_/domains --- .../src/ooniprobe/routers/private.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 9dd731e53..cbee5cc3a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1201,8 +1201,19 @@ def api_private_networks() -> Response: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) -@api_private_blueprint.route("/domains") -def api_private_domains() -> Response: +class MeasuredDomainStat(BaseModel): + category_code: str = Field(..., description="Citizenlab Category Code") + domain_name: DomainStr = Field(..., description="Domain Name") + measurement_count: int = Field(..., description="Number of measurements") + + +class DomainsMeasuredResponse(BaseModel): + results: List[MeasuredDomainStat] = Field(..., description="Domains that have measurements") + v: int = Field(..., description="Version of API response") + + +@router.get("/domains", response_model=DomainsMeasuredResponse, tags=["private"]) +def api_private_domains() -> DomainsMeasuredResponse: """List all the domains in the test-lists with their measurement count --- responses: @@ -1234,9 +1245,9 @@ def api_private_domains() -> Response: """ try: results = query_click(sql.text(q), {}) - return cachedjson("2h", v=0, results=results) + return DomainsMeasuredResponse(v=0, results=results) except Exception as e: - return jerror(str(e), v=0) + raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) @api_private_blueprint.route("/check-in", methods=["POST"]) From 12652eab6fc00272ed62d6f1d382bc92a6812e11 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 09:53:45 +0200 Subject: [PATCH 030/105] remove unused imports --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index cbee5cc3a..99123adaa 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -14,12 +14,8 @@ import logging import math -from flask import Blueprint, current_app, request, Response - from sqlalchemy import sql -from werkzeug.exceptions import BadRequest - from ooniapi.auth import role_required from ooniapi.config import metrics from ooniapi.countries import lookup_country From a96c899ae95c8fa83180c89230acca4acfc7d0d5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 09:53:57 +0200 Subject: [PATCH 031/105] add TEST_GROUPS from old API --- .../src/ooniprobe/routers/private.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 99123adaa..ca48b1146 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -49,6 +49,33 @@ # TODO: configure tags for HTTP caching across where useful +TEST_GROUPS = { + "websites": ["web_connectivity"], + "im": ["facebook_messenger", "signal", "telegram", "whatsapp"], + "middlebox": ["http_invalid_request_line", "http_header_field_manipulation"], + "performance": ["ndt", "dash"], + "circumvention": [ + "bridge_reachability", + "meek_fronted_requests_test", + "vanilla_tor", + "tcp_connect", + "psiphon", + "tor", + "torsf", + "riseupvpn", + ], + "legacy": [ + "http_requests", + "dns_consistency", + "http_host", + "multi_protocol_traceroute", + ], + "experimental": [ + "urlgetter", + "dnscheck", + "stunreachability", + ], +} def daterange(start_date, end_date): From c8a30b8026e2ef0535d72e20092a32dfcd90ce12 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 09:54:29 +0200 Subject: [PATCH 032/105] fix website_urls --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ca48b1146..05c2fc641 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -527,8 +527,8 @@ class WebsiteURLsResponse(BaseModel): @router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) def api_private_website_test_urls( probe_cc: CountryAlpha2 = Query(..., description="Country Code"), - probe_asn: str = Query(..., description="ASN, e.g. AS1234") - limit: int = Query(10, description="Limit results") + probe_asn: str = Query(..., description="ASN, e.g. AS1234"), + limit: int = Query(10, description="Limit results"), offset: int = Query(0, description="Offset results") ) -> WebsiteURLsResponse: """TODO From 6567cf3f1bf6a359696efb8bac30597e0d6b95f0 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:09:02 +0200 Subject: [PATCH 033/105] add data.py from old api --- .../services/ooniprobe/src/ooniprobe/data.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/data.py diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/data.py b/ooniapi/services/ooniprobe/src/ooniprobe/data.py new file mode 100644 index 000000000..93b72e2d7 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/data.py @@ -0,0 +1,56 @@ +dnscheck_inputs = [ + "https://dns.google/dns-query", + "https://8.8.8.8/dns-query", + "dot://8.8.8.8:853/", + "dot://8.8.4.4:853/", + "https://8.8.4.4/dns-query", + "https://cloudflare-dns.com/dns-query", + "https://1.1.1.1/dns-query", + "https://1.0.0.1/dns-query", + "dot://1.1.1.1:853/", + "dot://1.0.0.1:853/", + "https://dns.quad9.net/dns-query", + "https://9.9.9.9/dns-query", + "dot://9.9.9.9:853/", + "dot://dns.quad9.net/", + "https://family.cloudflare-dns.com/dns-query", + "dot://family.cloudflare-dns.com/dns-query", + "https://dns11.quad9.net/dns-query", + "dot://dns11.quad9.net/dns-query", + "https://dns9.quad9.net/dns-query", + "dot://dns9.quad9.net/dns-query", + "https://dns12.quad9.net/dns-query", + "dot://dns12.quad9.net/dns-query", + "https://1dot1dot1dot1.cloudflare-dns.com/dns-query", + "dot://1dot1dot1dot1.cloudflare-dns.com/dns-query", + "https://dns.adguard.com/dns-query", + "dot://dns.adguard.com/dns-query", + "https://dns-family.adguard.com/dns-query", + "dot://dns-family.adguard.com/dns-query", + "https://dns.cloudflare.com/dns-query", + "https://adblock.doh.mullvad.net/dns-query", + "dot://adblock.doh.mullvad.net/dns-query", + "https://dns.alidns.com/dns-query", + "dot://dns.alidns.com/dns-query", + "https://doh.opendns.com/dns-query", + "https://dns.nextdns.io/dns-query", + "dot://dns.nextdns.io/dns-query", + "https://dns10.quad9.net/dns-query", + "dot://dns10.quad9.net/dns-query", + "https://security.cloudflare-dns.com/dns-query", + "dot://security.cloudflare-dns.com/dns-query", + "https://dns.switch.ch/dns-query", + "dot://dns.switch.ch/dns-query", +] +stunreachability_inputs = [ + "stun://stun.voip.blackberry.com:3478", + "stun://stun.antisip.com:3478", + "stun://stun.bluesip.net:3478", + "stun://stun.dus.net:3478", + "stun://stun.epygi.com:3478", + "stun://stun.sonetel.com:3478", + "stun://stun.sonetel.net:3478", + "stun://stun.uls.co.za:3478", + "stun://stun.voipgate.com:3478", + "stun://stun.voys.nl:3478", +] From dc60fe38bcf3955353aca51a06b1638e252c6e08 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:09:20 +0200 Subject: [PATCH 034/105] add countries from old api --- .../src/ooniprobe/countries/__init__.py | 273 ++++++++++++++++++ .../src/ooniprobe/countries/country-list.json | 1 + 2 files changed, 274 insertions(+) create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py create mode 100644 ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py b/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py new file mode 100644 index 000000000..29bd0680c --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/countries/__init__.py @@ -0,0 +1,273 @@ +""" +CC to country name lookup table + +Regenerate the dict with: + + python3 ooniapi/countries/__init__.py + +""" + +_countries = { + "AD": "Andorra", + "AE": "United Arab Emirates", + "AF": "Afghanistan", + "AG": "Antigua & Barbuda", + "AI": "Anguilla", + "AL": "Albania", + "AM": "Armenia", + "AO": "Angola", + "AQ": "Antarctica", + "AR": "Argentina", + "AS": "American Samoa", + "AT": "Austria", + "AU": "Australia", + "AW": "Aruba", + "AX": "Åland Islands", + "AZ": "Azerbaijan", + "BA": "Bosnia", + "BB": "Barbados", + "BD": "Bangladesh", + "BE": "Belgium", + "BF": "Burkina Faso", + "BG": "Bulgaria", + "BH": "Bahrain", + "BI": "Burundi", + "BJ": "Benin", + "BL": "St. Barthélemy", + "BM": "Bermuda", + "BN": "Brunei", + "BO": "Bolivia", + "BQ": "Caribbean Netherlands", + "BR": "Brazil", + "BS": "Bahamas", + "BT": "Bhutan", + "BV": "Bouvet Island", + "BW": "Botswana", + "BY": "Belarus", + "BZ": "Belize", + "CA": "Canada", + "CC": "Cocos (Keeling) Islands", + "CD": "Congo - Kinshasa", + "CF": "Central African Republic", + "CG": "Congo - Brazzaville", + "CH": "Switzerland", + "CI": "Côte d’Ivoire", + "CK": "Cook Islands", + "CL": "Chile", + "CM": "Cameroon", + "CN": "China", + "CO": "Colombia", + "CR": "Costa Rica", + "CU": "Cuba", + "CV": "Cape Verde", + "CW": "Curaçao", + "CX": "Christmas Island", + "CY": "Cyprus", + "CZ": "Czechia", + "DE": "Germany", + "DJ": "Djibouti", + "DK": "Denmark", + "DM": "Dominica", + "DO": "Dominican Republic", + "DZ": "Algeria", + "EC": "Ecuador", + "EE": "Estonia", + "EG": "Egypt", + "EH": "Western Sahara", + "ER": "Eritrea", + "ES": "Spain", + "ET": "Ethiopia", + "FI": "Finland", + "FJ": "Fiji", + "FK": "Falkland Islands", + "FM": "Micronesia", + "FO": "Faroe Islands", + "FR": "France", + "GA": "Gabon", + "GB": "United Kingdom", + "GD": "Grenada", + "GE": "Georgia", + "GF": "French Guiana", + "GG": "Guernsey", + "GH": "Ghana", + "GI": "Gibraltar", + "GL": "Greenland", + "GM": "Gambia", + "GN": "Guinea", + "GP": "Guadeloupe", + "GQ": "Equatorial Guinea", + "GR": "Greece", + "GS": "South Georgia & South Sandwich Islands", + "GT": "Guatemala", + "GU": "Guam", + "GW": "Guinea-Bissau", + "GY": "Guyana", + "HK": "Hong Kong", + "HM": "Heard & McDonald Islands", + "HN": "Honduras", + "HR": "Croatia", + "HT": "Haiti", + "HU": "Hungary", + "ID": "Indonesia", + "IE": "Ireland", + "IL": "Israel", + "IM": "Isle of Man", + "IN": "India", + "IO": "British Indian Ocean Territory", + "IQ": "Iraq", + "IR": "Iran", + "IS": "Iceland", + "IT": "Italy", + "JE": "Jersey", + "JM": "Jamaica", + "JO": "Jordan", + "JP": "Japan", + "KE": "Kenya", + "KG": "Kyrgyzstan", + "KH": "Cambodia", + "KI": "Kiribati", + "KM": "Comoros", + "KN": "St. Kitts & Nevis", + "KP": "North Korea", + "KR": "South Korea", + "KW": "Kuwait", + "KY": "Cayman Islands", + "KZ": "Kazakhstan", + "LA": "Laos", + "LB": "Lebanon", + "LC": "St. Lucia", + "LI": "Liechtenstein", + "LK": "Sri Lanka", + "LR": "Liberia", + "LS": "Lesotho", + "LT": "Lithuania", + "LU": "Luxembourg", + "LV": "Latvia", + "LY": "Libya", + "MA": "Morocco", + "MC": "Monaco", + "MD": "Moldova", + "ME": "Montenegro", + "MF": "St. Martin", + "MG": "Madagascar", + "MH": "Marshall Islands", + "MK": "North Macedonia", + "ML": "Mali", + "MM": "Myanmar", + "MN": "Mongolia", + "MO": "Macao", + "MP": "Northern Mariana Islands", + "MQ": "Martinique", + "MR": "Mauritania", + "MS": "Montserrat", + "MT": "Malta", + "MU": "Mauritius", + "MV": "Maldives", + "MW": "Malawi", + "MX": "Mexico", + "MY": "Malaysia", + "MZ": "Mozambique", + "NA": "Namibia", + "NC": "New Caledonia", + "NE": "Niger", + "NF": "Norfolk Island", + "NG": "Nigeria", + "NI": "Nicaragua", + "NL": "Netherlands", + "NO": "Norway", + "NP": "Nepal", + "NR": "Nauru", + "NU": "Niue", + "NZ": "New Zealand", + "OM": "Oman", + "PA": "Panama", + "PE": "Peru", + "PF": "French Polynesia", + "PG": "Papua New Guinea", + "PH": "Philippines", + "PK": "Pakistan", + "PL": "Poland", + "PM": "St. Pierre & Miquelon", + "PN": "Pitcairn Islands", + "PR": "Puerto Rico", + "PS": "Palestine", + "PT": "Portugal", + "PW": "Palau", + "PY": "Paraguay", + "QA": "Qatar", + "RE": "Réunion", + "RO": "Romania", + "RS": "Serbia", + "RU": "Russia", + "RW": "Rwanda", + "SA": "Saudi Arabia", + "SB": "Solomon Islands", + "SC": "Seychelles", + "SD": "Sudan", + "SE": "Sweden", + "SG": "Singapore", + "SH": "St. Helena", + "SI": "Slovenia", + "SJ": "Svalbard & Jan Mayen", + "SK": "Slovakia", + "SL": "Sierra Leone", + "SM": "San Marino", + "SN": "Senegal", + "SO": "Somalia", + "SR": "Suriname", + "SS": "South Sudan", + "ST": "São Tomé & Príncipe", + "SV": "El Salvador", + "SX": "Sint Maarten", + "SY": "Syria", + "SZ": "Eswatini", + "TC": "Turks & Caicos Islands", + "TD": "Chad", + "TF": "French Southern Territories", + "TG": "Togo", + "TH": "Thailand", + "TJ": "Tajikistan", + "TK": "Tokelau", + "TL": "Timor-Leste", + "TM": "Turkmenistan", + "TN": "Tunisia", + "TO": "Tonga", + "TR": "Turkey", + "TT": "Trinidad & Tobago", + "TV": "Tuvalu", + "TW": "Taiwan", + "TZ": "Tanzania", + "UA": "Ukraine", + "UG": "Uganda", + "UM": "U.S. Outlying Islands", + "US": "United States", + "UY": "Uruguay", + "UZ": "Uzbekistan", + "VA": "Vatican City", + "VC": "St. Vincent & Grenadines", + "VE": "Venezuela", + "VG": "British Virgin Islands", + "VI": "U.S. Virgin Islands", + "VN": "Vietnam", + "VU": "Vanuatu", + "WF": "Wallis & Futuna", + "WS": "Samoa", + "YE": "Yemen", + "YT": "Mayotte", + "ZA": "South Africa", + "ZM": "Zambia", + "ZW": "Zimbabwe", +} + + +def lookup_country(probe_cc: str) -> str: + """Translate 2-char country code into country name""" + return _countries[probe_cc.upper()] + + +if __name__ == "__main__": + import json + + with open("ooniapi/countries/country-list.json") as f: + d = {e["iso3166_alpha2"].upper(): e["name"] for e in json.load(f)} + print(dict(sorted(d.items()))) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json b/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json new file mode 100644 index 000000000..cb30c2550 --- /dev/null +++ b/ooniapi/services/ooniprobe/src/ooniprobe/countries/country-list.json @@ -0,0 +1 @@ +[{"iso3166_alpha2": "TW", "iso3166_alpha3": "TWN", "iso3166_num": "158", "iso3166_name": "Taiwan", "name": "Taiwan", "languages": ["zh-TW", "zh", "nan", "hak"], "tld": ".tw", "capital": "Taipei", "region_code": "142", "sub_region_code": "156"}, {"iso3166_alpha2": "AF", "iso3166_alpha3": "AFG", "iso3166_num": "004", "iso3166_name": "Afghanistan", "name": "Afghanistan", "languages": ["fa-AF", "ps", "uz-AF", "tk"], "tld": ".af", "capital": "Kabul", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "AL", "iso3166_alpha3": "ALB", "iso3166_num": "008", "iso3166_name": "Albania", "name": "Albania", "languages": ["sq", "el"], "tld": ".al", "capital": "Tirana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "DZ", "iso3166_alpha3": "DZA", "iso3166_num": "012", "iso3166_name": "Algeria", "name": "Algeria", "languages": ["ar-DZ"], "tld": ".dz", "capital": "Algiers", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "AS", "iso3166_alpha3": "ASM", "iso3166_num": "016", "iso3166_name": "American Samoa", "name": "American Samoa", "languages": ["en-AS", "sm", "to"], "tld": ".as", "capital": "Pago Pago", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "AD", "iso3166_alpha3": "AND", "iso3166_num": "020", "iso3166_name": "Andorra", "name": "Andorra", "languages": ["ca"], "tld": ".ad", "capital": "Andorra la Vella", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "AO", "iso3166_alpha3": "AGO", "iso3166_num": "024", "iso3166_name": "Angola", "name": "Angola", "languages": ["pt-AO"], "tld": ".ao", "capital": "Luanda", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AI", "iso3166_alpha3": "AIA", "iso3166_num": "660", "iso3166_name": "Anguilla", "name": "Anguilla", "languages": ["en-AI"], "tld": ".ai", "capital": "The Valley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AQ", "iso3166_alpha3": "ATA", "iso3166_num": "010", "iso3166_name": "Antarctica", "name": "Antarctica", "languages": [""], "tld": ".aq", "capital": "", "region_code": "", "sub_region_code": ""}, {"iso3166_alpha2": "AG", "iso3166_alpha3": "ATG", "iso3166_num": "028", "iso3166_name": "Antigua & Barbuda", "name": "Antigua & Barbuda", "languages": ["en-AG"], "tld": ".ag", "capital": "St. John's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AR", "iso3166_alpha3": "ARG", "iso3166_num": "032", "iso3166_name": "Argentina", "name": "Argentina", "languages": ["es-AR", "en", "it", "de", "fr", "gn"], "tld": ".ar", "capital": "Buenos Aires", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AM", "iso3166_alpha3": "ARM", "iso3166_num": "051", "iso3166_name": "Armenia", "name": "Armenia", "languages": ["hy"], "tld": ".am", "capital": "Yerevan", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "AW", "iso3166_alpha3": "ABW", "iso3166_num": "533", "iso3166_name": "Aruba", "name": "Aruba", "languages": ["nl-AW", "pap", "es", "en"], "tld": ".aw", "capital": "Oranjestad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AU", "iso3166_alpha3": "AUS", "iso3166_num": "036", "iso3166_name": "Australia", "name": "Australia", "languages": ["en-AU"], "tld": ".au", "capital": "Canberra", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "AT", "iso3166_alpha3": "AUT", "iso3166_num": "040", "iso3166_name": "Austria", "name": "Austria", "languages": ["de-AT", "hr", "hu", "sl"], "tld": ".at", "capital": "Vienna", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "AZ", "iso3166_alpha3": "AZE", "iso3166_num": "031", "iso3166_name": "Azerbaijan", "name": "Azerbaijan", "languages": ["az", "ru", "hy"], "tld": ".az", "capital": "Baku", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BS", "iso3166_alpha3": "BHS", "iso3166_num": "044", "iso3166_name": "Bahamas", "name": "Bahamas", "languages": ["en-BS"], "tld": ".bs", "capital": "Nassau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BH", "iso3166_alpha3": "BHR", "iso3166_num": "048", "iso3166_name": "Bahrain", "name": "Bahrain", "languages": ["ar-BH", "en", "fa", "ur"], "tld": ".bh", "capital": "Manama", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BD", "iso3166_alpha3": "BGD", "iso3166_num": "050", "iso3166_name": "Bangladesh", "name": "Bangladesh", "languages": ["bn-BD", "en"], "tld": ".bd", "capital": "Dhaka", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BB", "iso3166_alpha3": "BRB", "iso3166_num": "052", "iso3166_name": "Barbados", "name": "Barbados", "languages": ["en-BB"], "tld": ".bb", "capital": "Bridgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BY", "iso3166_alpha3": "BLR", "iso3166_num": "112", "iso3166_name": "Belarus", "name": "Belarus", "languages": ["be", "ru"], "tld": ".by", "capital": "Minsk", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BE", "iso3166_alpha3": "BEL", "iso3166_num": "056", "iso3166_name": "Belgium", "name": "Belgium", "languages": ["nl-BE", "fr-BE", "de-BE"], "tld": ".be", "capital": "Brussels", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "BZ", "iso3166_alpha3": "BLZ", "iso3166_num": "084", "iso3166_name": "Belize", "name": "Belize", "languages": ["en-BZ", "es"], "tld": ".bz", "capital": "Belmopan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BJ", "iso3166_alpha3": "BEN", "iso3166_num": "204", "iso3166_name": "Benin", "name": "Benin", "languages": ["fr-BJ"], "tld": ".bj", "capital": "Porto-Novo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BM", "iso3166_alpha3": "BMU", "iso3166_num": "060", "iso3166_name": "Bermuda", "name": "Bermuda", "languages": ["en-BM", "pt"], "tld": ".bm", "capital": "Hamilton", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "BT", "iso3166_alpha3": "BTN", "iso3166_num": "064", "iso3166_name": "Bhutan", "name": "Bhutan", "languages": ["dz"], "tld": ".bt", "capital": "Thimphu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BO", "iso3166_alpha3": "BOL", "iso3166_num": "068", "iso3166_name": "Bolivia", "name": "Bolivia", "languages": ["es-BO", "qu", "ay"], "tld": ".bo", "capital": "Sucre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BQ", "iso3166_alpha3": "BES", "iso3166_num": "535", "iso3166_name": "Caribbean Netherlands", "name": "Caribbean Netherlands", "languages": ["nl", "pap", "en"], "tld": ".bq", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BA", "iso3166_alpha3": "BIH", "iso3166_num": "070", "iso3166_name": "Bosnia", "name": "Bosnia", "languages": ["bs", "hr-BA", "sr-BA"], "tld": ".ba", "capital": "Sarajevo", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "BW", "iso3166_alpha3": "BWA", "iso3166_num": "072", "iso3166_name": "Botswana", "name": "Botswana", "languages": ["en-BW", "tn-BW"], "tld": ".bw", "capital": "Gaborone", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BV", "iso3166_alpha3": "BVT", "iso3166_num": "074", "iso3166_name": "Bouvet Island", "name": "Bouvet Island", "languages": [""], "tld": ".bv", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BR", "iso3166_alpha3": "BRA", "iso3166_num": "076", "iso3166_name": "Brazil", "name": "Brazil", "languages": ["pt-BR", "es", "en", "fr"], "tld": ".br", "capital": "Brasilia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "IO", "iso3166_alpha3": "IOT", "iso3166_num": "086", "iso3166_name": "British Indian Ocean Territory", "name": "British Indian Ocean Territory", "languages": ["en-IO"], "tld": ".io", "capital": "Diego Garcia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "VG", "iso3166_alpha3": "VGB", "iso3166_num": "092", "iso3166_name": "British Virgin Islands", "name": "British Virgin Islands", "languages": ["en-VG"], "tld": ".vg", "capital": "Road Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BN", "iso3166_alpha3": "BRN", "iso3166_num": "096", "iso3166_name": "Brunei", "name": "Brunei", "languages": ["ms-BN", "en-BN"], "tld": ".bn", "capital": "Bandar Seri Begawan", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "BG", "iso3166_alpha3": "BGR", "iso3166_num": "100", "iso3166_name": "Bulgaria", "name": "Bulgaria", "languages": ["bg", "tr-BG", "rom"], "tld": ".bg", "capital": "Sofia", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BF", "iso3166_alpha3": "BFA", "iso3166_num": "854", "iso3166_name": "Burkina Faso", "name": "Burkina Faso", "languages": ["fr-BF", "mos"], "tld": ".bf", "capital": "Ouagadougou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BI", "iso3166_alpha3": "BDI", "iso3166_num": "108", "iso3166_name": "Burundi", "name": "Burundi", "languages": ["fr-BI", "rn"], "tld": ".bi", "capital": "Bujumbura", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CV", "iso3166_alpha3": "CPV", "iso3166_num": "132", "iso3166_name": "Cape Verde", "name": "Cape Verde", "languages": ["pt-CV"], "tld": ".cv", "capital": "Praia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KH", "iso3166_alpha3": "KHM", "iso3166_num": "116", "iso3166_name": "Cambodia", "name": "Cambodia", "languages": ["km", "fr", "en"], "tld": ".kh", "capital": "Phnom Penh", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "CM", "iso3166_alpha3": "CMR", "iso3166_num": "120", "iso3166_name": "Cameroon", "name": "Cameroon", "languages": ["en-CM", "fr-CM"], "tld": ".cm", "capital": "Yaounde", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CA", "iso3166_alpha3": "CAN", "iso3166_num": "124", "iso3166_name": "Canada", "name": "Canada", "languages": ["en-CA", "fr-CA", "iu"], "tld": ".ca", "capital": "Ottawa", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "KY", "iso3166_alpha3": "CYM", "iso3166_num": "136", "iso3166_name": "Cayman Islands", "name": "Cayman Islands", "languages": ["en-KY"], "tld": ".ky", "capital": "George Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CF", "iso3166_alpha3": "CAF", "iso3166_num": "140", "iso3166_name": "Central African Republic", "name": "Central African Republic", "languages": ["fr-CF", "sg", "ln", "kg"], "tld": ".cf", "capital": "Bangui", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TD", "iso3166_alpha3": "TCD", "iso3166_num": "148", "iso3166_name": "Chad", "name": "Chad", "languages": ["fr-TD", "ar-TD", "sre"], "tld": ".td", "capital": "N'Djamena", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CL", "iso3166_alpha3": "CHL", "iso3166_num": "152", "iso3166_name": "Chile", "name": "Chile", "languages": ["es-CL"], "tld": ".cl", "capital": "Santiago", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CN", "iso3166_alpha3": "CHN", "iso3166_num": "156", "iso3166_name": "China", "name": "China", "languages": ["zh-CN", "yue", "wuu", "dta", "ug", "za"], "tld": ".cn", "capital": "Beijing", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "HK", "iso3166_alpha3": "HKG", "iso3166_num": "344", "iso3166_name": "Hong Kong", "name": "Hong Kong", "languages": ["zh-HK", "yue", "zh", "en"], "tld": ".hk", "capital": "Hong Kong", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MO", "iso3166_alpha3": "MAC", "iso3166_num": "446", "iso3166_name": "Macau", "name": "Macao", "languages": ["zh", "zh-MO", "pt"], "tld": ".mo", "capital": "Macao", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CX", "iso3166_alpha3": "CXR", "iso3166_num": "162", "iso3166_name": "Christmas Island", "name": "Christmas Island", "languages": ["en", "zh", "ms-CC"], "tld": ".cx", "capital": "Flying Fish Cove", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CC", "iso3166_alpha3": "CCK", "iso3166_num": "166", "iso3166_name": "Cocos (Keeling) Islands", "name": "Cocos (Keeling) Islands", "languages": ["ms-CC", "en"], "tld": ".cc", "capital": "West Island", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CO", "iso3166_alpha3": "COL", "iso3166_num": "170", "iso3166_name": "Colombia", "name": "Colombia", "languages": ["es-CO"], "tld": ".co", "capital": "Bogota", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "KM", "iso3166_alpha3": "COM", "iso3166_num": "174", "iso3166_name": "Comoros", "name": "Comoros", "languages": ["ar", "fr-KM"], "tld": ".km", "capital": "Moroni", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CG", "iso3166_alpha3": "COG", "iso3166_num": "178", "iso3166_name": "Congo - Brazzaville", "name": "Congo - Brazzaville", "languages": ["fr-CG", "kg", "ln-CG"], "tld": ".cg", "capital": "Brazzaville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CK", "iso3166_alpha3": "COK", "iso3166_num": "184", "iso3166_name": "Cook Islands", "name": "Cook Islands", "languages": ["en-CK", "mi"], "tld": ".ck", "capital": "Avarua", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "CR", "iso3166_alpha3": "CRI", "iso3166_num": "188", "iso3166_name": "Costa Rica", "name": "Costa Rica", "languages": ["es-CR", "en"], "tld": ".cr", "capital": "San Jose", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HR", "iso3166_alpha3": "HRV", "iso3166_num": "191", "iso3166_name": "Croatia", "name": "Croatia", "languages": ["hr-HR", "sr"], "tld": ".hr", "capital": "Zagreb", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "CU", "iso3166_alpha3": "CUB", "iso3166_num": "192", "iso3166_name": "Cuba", "name": "Cuba", "languages": ["es-CU", "pap"], "tld": ".cu", "capital": "Havana", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CW", "iso3166_alpha3": "CUW", "iso3166_num": "531", "iso3166_name": "Cura\u00e7ao", "name": "Cura\u00e7ao", "languages": ["nl", "pap"], "tld": ".cw", "capital": " Willemstad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CY", "iso3166_alpha3": "CYP", "iso3166_num": "196", "iso3166_name": "Cyprus", "name": "Cyprus", "languages": ["el-CY", "tr-CY", "en"], "tld": ".cy", "capital": "Nicosia", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "CZ", "iso3166_alpha3": "CZE", "iso3166_num": "203", "iso3166_name": "Czechia", "name": "Czechia", "languages": ["cs", "sk"], "tld": ".cz", "capital": "Prague", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "CI", "iso3166_alpha3": "CIV", "iso3166_num": "384", "iso3166_name": "C\u00f4te d\u2019Ivoire", "name": "C\u00f4te d\u2019Ivoire", "languages": ["fr-CI"], "tld": ".ci", "capital": "Yamoussoukro", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KP", "iso3166_alpha3": "PRK", "iso3166_num": "408", "iso3166_name": "North Korea", "name": "North Korea", "languages": ["ko-KP"], "tld": ".kp", "capital": "Pyongyang", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CD", "iso3166_alpha3": "COD", "iso3166_num": "180", "iso3166_name": "Congo - Kinshasa", "name": "Congo - Kinshasa", "languages": ["fr-CD", "ln", "ktu", "kg", "sw", "lua"], "tld": ".cd", "capital": "Kinshasa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DK", "iso3166_alpha3": "DNK", "iso3166_num": "208", "iso3166_name": "Denmark", "name": "Denmark", "languages": ["da-DK", "en", "fo", "de-DK"], "tld": ".dk", "capital": "Copenhagen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "DJ", "iso3166_alpha3": "DJI", "iso3166_num": "262", "iso3166_name": "Djibouti", "name": "Djibouti", "languages": ["fr-DJ", "ar", "so-DJ", "aa"], "tld": ".dj", "capital": "Djibouti", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DM", "iso3166_alpha3": "DMA", "iso3166_num": "212", "iso3166_name": "Dominica", "name": "Dominica", "languages": ["en-DM"], "tld": ".dm", "capital": "Roseau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "DO", "iso3166_alpha3": "DOM", "iso3166_num": "214", "iso3166_name": "Dominican Republic", "name": "Dominican Republic", "languages": ["es-DO"], "tld": ".do", "capital": "Santo Domingo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EC", "iso3166_alpha3": "ECU", "iso3166_num": "218", "iso3166_name": "Ecuador", "name": "Ecuador", "languages": ["es-EC"], "tld": ".ec", "capital": "Quito", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EG", "iso3166_alpha3": "EGY", "iso3166_num": "818", "iso3166_name": "Egypt", "name": "Egypt", "languages": ["ar-EG", "en", "fr"], "tld": ".eg", "capital": "Cairo", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SV", "iso3166_alpha3": "SLV", "iso3166_num": "222", "iso3166_name": "El Salvador", "name": "El Salvador", "languages": ["es-SV"], "tld": ".sv", "capital": "San Salvador", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GQ", "iso3166_alpha3": "GNQ", "iso3166_num": "226", "iso3166_name": "Equatorial Guinea", "name": "Equatorial Guinea", "languages": ["es-GQ", "fr"], "tld": ".gq", "capital": "Malabo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ER", "iso3166_alpha3": "ERI", "iso3166_num": "232", "iso3166_name": "Eritrea", "name": "Eritrea", "languages": ["aa-ER", "ar", "tig", "kun", "ti-ER"], "tld": ".er", "capital": "Asmara", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "EE", "iso3166_alpha3": "EST", "iso3166_num": "233", "iso3166_name": "Estonia", "name": "Estonia", "languages": ["et", "ru"], "tld": ".ee", "capital": "Tallinn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "ET", "iso3166_alpha3": "ETH", "iso3166_num": "231", "iso3166_name": "Ethiopia", "name": "Ethiopia", "languages": ["am", "en-ET", "om-ET", "ti-ET", "so-ET", "sid"], "tld": ".et", "capital": "Addis Ababa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "FK", "iso3166_alpha3": "FLK", "iso3166_num": "238", "iso3166_name": "Falkland Islands", "name": "Falkland Islands", "languages": ["en-FK"], "tld": ".fk", "capital": "Stanley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FO", "iso3166_alpha3": "FRO", "iso3166_num": "234", "iso3166_name": "Faroe Islands", "name": "Faroe Islands", "languages": ["fo", "da-FO"], "tld": ".fo", "capital": "Torshavn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FJ", "iso3166_alpha3": "FJI", "iso3166_num": "242", "iso3166_name": "Fiji", "name": "Fiji", "languages": ["en-FJ", "fj"], "tld": ".fj", "capital": "Suva", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "FI", "iso3166_alpha3": "FIN", "iso3166_num": "246", "iso3166_name": "Finland", "name": "Finland", "languages": ["fi-FI", "sv-FI", "smn"], "tld": ".fi", "capital": "Helsinki", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FR", "iso3166_alpha3": "FRA", "iso3166_num": "250", "iso3166_name": "France", "name": "France", "languages": ["fr-FR", "frp", "br", "co", "ca", "eu", "oc"], "tld": ".fr", "capital": "Paris", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GF", "iso3166_alpha3": "GUF", "iso3166_num": "254", "iso3166_name": "French Guiana", "name": "French Guiana", "languages": ["fr-GF"], "tld": ".gf", "capital": "Cayenne", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PF", "iso3166_alpha3": "PYF", "iso3166_num": "258", "iso3166_name": "French Polynesia", "name": "French Polynesia", "languages": ["fr-PF", "ty"], "tld": ".pf", "capital": "Papeete", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TF", "iso3166_alpha3": "ATF", "iso3166_num": "260", "iso3166_name": "French Southern Territories", "name": "French Southern Territories", "languages": ["fr"], "tld": ".tf", "capital": "Port-aux-Francais", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GA", "iso3166_alpha3": "GAB", "iso3166_num": "266", "iso3166_name": "Gabon", "name": "Gabon", "languages": ["fr-GA"], "tld": ".ga", "capital": "Libreville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GM", "iso3166_alpha3": "GMB", "iso3166_num": "270", "iso3166_name": "Gambia", "name": "Gambia", "languages": ["en-GM", "mnk", "wof", "wo", "ff"], "tld": ".gm", "capital": "Banjul", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GE", "iso3166_alpha3": "GEO", "iso3166_num": "268", "iso3166_name": "Georgia", "name": "Georgia", "languages": ["ka", "ru", "hy", "az"], "tld": ".ge", "capital": "Tbilisi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "DE", "iso3166_alpha3": "DEU", "iso3166_num": "276", "iso3166_name": "Germany", "name": "Germany", "languages": ["de"], "tld": ".de", "capital": "Berlin", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GH", "iso3166_alpha3": "GHA", "iso3166_num": "288", "iso3166_name": "Ghana", "name": "Ghana", "languages": ["en-GH", "ak", "ee", "tw"], "tld": ".gh", "capital": "Accra", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GI", "iso3166_alpha3": "GIB", "iso3166_num": "292", "iso3166_name": "Gibraltar", "name": "Gibraltar", "languages": ["en-GI", "es", "it", "pt"], "tld": ".gi", "capital": "Gibraltar", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GR", "iso3166_alpha3": "GRC", "iso3166_num": "300", "iso3166_name": "Greece", "name": "Greece", "languages": ["el-GR", "en", "fr"], "tld": ".gr", "capital": "Athens", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GL", "iso3166_alpha3": "GRL", "iso3166_num": "304", "iso3166_name": "Greenland", "name": "Greenland", "languages": ["kl", "da-GL", "en"], "tld": ".gl", "capital": "Nuuk", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "GD", "iso3166_alpha3": "GRD", "iso3166_num": "308", "iso3166_name": "Grenada", "name": "Grenada", "languages": ["en-GD"], "tld": ".gd", "capital": "St. George's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GP", "iso3166_alpha3": "GLP", "iso3166_num": "312", "iso3166_name": "Guadeloupe", "name": "Guadeloupe", "languages": ["fr-GP"], "tld": ".gp", "capital": "Basse-Terre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GU", "iso3166_alpha3": "GUM", "iso3166_num": "316", "iso3166_name": "Guam", "name": "Guam", "languages": ["en-GU", "ch-GU"], "tld": ".gu", "capital": "Hagatna", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "GT", "iso3166_alpha3": "GTM", "iso3166_num": "320", "iso3166_name": "Guatemala", "name": "Guatemala", "languages": ["es-GT"], "tld": ".gt", "capital": "Guatemala City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GG", "iso3166_alpha3": "GGY", "iso3166_num": "831", "iso3166_name": "Guernsey", "name": "Guernsey", "languages": ["en", "nrf"], "tld": ".gg", "capital": "St Peter Port", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "GN", "iso3166_alpha3": "GIN", "iso3166_num": "324", "iso3166_name": "Guinea", "name": "Guinea", "languages": ["fr-GN"], "tld": ".gn", "capital": "Conakry", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GW", "iso3166_alpha3": "GNB", "iso3166_num": "624", "iso3166_name": "Guinea-Bissau", "name": "Guinea-Bissau", "languages": ["pt-GW", "pov"], "tld": ".gw", "capital": "Bissau", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GY", "iso3166_alpha3": "GUY", "iso3166_num": "328", "iso3166_name": "Guyana", "name": "Guyana", "languages": ["en-GY"], "tld": ".gy", "capital": "Georgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HT", "iso3166_alpha3": "HTI", "iso3166_num": "332", "iso3166_name": "Haiti", "name": "Haiti", "languages": ["ht", "fr-HT"], "tld": ".ht", "capital": "Port-au-Prince", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HM", "iso3166_alpha3": "HMD", "iso3166_num": "334", "iso3166_name": "Heard & McDonald Islands", "name": "Heard & McDonald Islands", "languages": [""], "tld": ".hm", "capital": "", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "VA", "iso3166_alpha3": "VAT", "iso3166_num": "336", "iso3166_name": "Vatican City", "name": "Vatican City", "languages": ["la", "it", "fr"], "tld": ".va", "capital": "Vatican City", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "HN", "iso3166_alpha3": "HND", "iso3166_num": "340", "iso3166_name": "Honduras", "name": "Honduras", "languages": ["es-HN", "cab", "miq"], "tld": ".hn", "capital": "Tegucigalpa", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HU", "iso3166_alpha3": "HUN", "iso3166_num": "348", "iso3166_name": "Hungary", "name": "Hungary", "languages": ["hu-HU"], "tld": ".hu", "capital": "Budapest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "IS", "iso3166_alpha3": "ISL", "iso3166_num": "352", "iso3166_name": "Iceland", "name": "Iceland", "languages": ["is", "en", "de", "da", "sv", "no"], "tld": ".is", "capital": "Reykjavik", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IN", "iso3166_alpha3": "IND", "iso3166_num": "356", "iso3166_name": "India", "name": "India", "languages": ["en-IN", "hi", "bn", "te", "mr", "ta", "ur", "gu", "kn", "ml", "or", "pa", "as", "bh", "sat", "ks", "ne", "sd", "kok", "doi", "mni", "sit", "sa", "fr", "lus", "inc"], "tld": ".in", "capital": "New Delhi", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ID", "iso3166_alpha3": "IDN", "iso3166_num": "360", "iso3166_name": "Indonesia", "name": "Indonesia", "languages": ["id", "en", "nl", "jv"], "tld": ".id", "capital": "Jakarta", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "IR", "iso3166_alpha3": "IRN", "iso3166_num": "364", "iso3166_name": "Iran", "name": "Iran", "languages": ["fa-IR", "ku"], "tld": ".ir", "capital": "Tehran", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "IQ", "iso3166_alpha3": "IRQ", "iso3166_num": "368", "iso3166_name": "Iraq", "name": "Iraq", "languages": ["ar-IQ", "ku", "hy"], "tld": ".iq", "capital": "Baghdad", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IE", "iso3166_alpha3": "IRL", "iso3166_num": "372", "iso3166_name": "Ireland", "name": "Ireland", "languages": ["en-IE", "ga-IE"], "tld": ".ie", "capital": "Dublin", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IM", "iso3166_alpha3": "IMN", "iso3166_num": "833", "iso3166_name": "Isle of Man", "name": "Isle of Man", "languages": ["en", "gv"], "tld": ".im", "capital": "Douglas", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IL", "iso3166_alpha3": "ISR", "iso3166_num": "376", "iso3166_name": "Israel", "name": "Israel", "languages": ["he", "ar-IL", "en-IL", ""], "tld": ".il", "capital": "Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IT", "iso3166_alpha3": "ITA", "iso3166_num": "380", "iso3166_name": "Italy", "name": "Italy", "languages": ["it-IT", "de-IT", "fr-IT", "sc", "ca", "co", "sl"], "tld": ".it", "capital": "Rome", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "JM", "iso3166_alpha3": "JAM", "iso3166_num": "388", "iso3166_name": "Jamaica", "name": "Jamaica", "languages": ["en-JM"], "tld": ".jm", "capital": "Kingston", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "JP", "iso3166_alpha3": "JPN", "iso3166_num": "392", "iso3166_name": "Japan", "name": "Japan", "languages": ["ja"], "tld": ".jp", "capital": "Tokyo", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "JE", "iso3166_alpha3": "JEY", "iso3166_num": "832", "iso3166_name": "Jersey", "name": "Jersey", "languages": ["en", "fr", "nrf"], "tld": ".je", "capital": "Saint Helier", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "JO", "iso3166_alpha3": "JOR", "iso3166_num": "400", "iso3166_name": "Jordan", "name": "Jordan", "languages": ["ar-JO", "en"], "tld": ".jo", "capital": "Amman", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KZ", "iso3166_alpha3": "KAZ", "iso3166_num": "398", "iso3166_name": "Kazakhstan", "name": "Kazakhstan", "languages": ["kk", "ru"], "tld": ".kz", "capital": "Nur-Sultan", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "KE", "iso3166_alpha3": "KEN", "iso3166_num": "404", "iso3166_name": "Kenya", "name": "Kenya", "languages": ["en-KE", "sw-KE"], "tld": ".ke", "capital": "Nairobi", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KI", "iso3166_alpha3": "KIR", "iso3166_num": "296", "iso3166_name": "Kiribati", "name": "Kiribati", "languages": ["en-KI", "gil"], "tld": ".ki", "capital": "Tarawa", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "KW", "iso3166_alpha3": "KWT", "iso3166_num": "414", "iso3166_name": "Kuwait", "name": "Kuwait", "languages": ["ar-KW", "en"], "tld": ".kw", "capital": "Kuwait City", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KG", "iso3166_alpha3": "KGZ", "iso3166_num": "417", "iso3166_name": "Kyrgyzstan", "name": "Kyrgyzstan", "languages": ["ky", "uz", "ru"], "tld": ".kg", "capital": "Bishkek", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "LA", "iso3166_alpha3": "LAO", "iso3166_num": "418", "iso3166_name": "Laos", "name": "Laos", "languages": ["lo", "fr", "en"], "tld": ".la", "capital": "Vientiane", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "LV", "iso3166_alpha3": "LVA", "iso3166_num": "428", "iso3166_name": "Latvia", "name": "Latvia", "languages": ["lv", "ru", "lt"], "tld": ".lv", "capital": "Riga", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LB", "iso3166_alpha3": "LBN", "iso3166_num": "422", "iso3166_name": "Lebanon", "name": "Lebanon", "languages": ["ar-LB", "fr-LB", "en", "hy"], "tld": ".lb", "capital": "Beirut", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "LS", "iso3166_alpha3": "LSO", "iso3166_num": "426", "iso3166_name": "Lesotho", "name": "Lesotho", "languages": ["en-LS", "st", "zu", "xh"], "tld": ".ls", "capital": "Maseru", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LR", "iso3166_alpha3": "LBR", "iso3166_num": "430", "iso3166_name": "Liberia", "name": "Liberia", "languages": ["en-LR"], "tld": ".lr", "capital": "Monrovia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LY", "iso3166_alpha3": "LBY", "iso3166_num": "434", "iso3166_name": "Libya", "name": "Libya", "languages": ["ar-LY", "it", "en"], "tld": ".ly", "capital": "Tripoli", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "LI", "iso3166_alpha3": "LIE", "iso3166_num": "438", "iso3166_name": "Liechtenstein", "name": "Liechtenstein", "languages": ["de-LI"], "tld": ".li", "capital": "Vaduz", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "LT", "iso3166_alpha3": "LTU", "iso3166_num": "440", "iso3166_name": "Lithuania", "name": "Lithuania", "languages": ["lt", "ru", "pl"], "tld": ".lt", "capital": "Vilnius", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LU", "iso3166_alpha3": "LUX", "iso3166_num": "442", "iso3166_name": "Luxembourg", "name": "Luxembourg", "languages": ["lb", "de-LU", "fr-LU"], "tld": ".lu", "capital": "Luxembourg", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MG", "iso3166_alpha3": "MDG", "iso3166_num": "450", "iso3166_name": "Madagascar", "name": "Madagascar", "languages": ["fr-MG", "mg"], "tld": ".mg", "capital": "Antananarivo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MW", "iso3166_alpha3": "MWI", "iso3166_num": "454", "iso3166_name": "Malawi", "name": "Malawi", "languages": ["ny", "yao", "tum", "swk"], "tld": ".mw", "capital": "Lilongwe", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MY", "iso3166_alpha3": "MYS", "iso3166_num": "458", "iso3166_name": "Malaysia", "name": "Malaysia", "languages": ["ms-MY", "en", "zh", "ta", "te", "ml", "pa", "th"], "tld": ".my", "capital": "Kuala Lumpur", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MV", "iso3166_alpha3": "MDV", "iso3166_num": "462", "iso3166_name": "Maldives", "name": "Maldives", "languages": ["dv", "en"], "tld": ".mv", "capital": "Male", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ML", "iso3166_alpha3": "MLI", "iso3166_num": "466", "iso3166_name": "Mali", "name": "Mali", "languages": ["fr-ML", "bm"], "tld": ".ml", "capital": "Bamako", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MT", "iso3166_alpha3": "MLT", "iso3166_num": "470", "iso3166_name": "Malta", "name": "Malta", "languages": ["mt", "en-MT"], "tld": ".mt", "capital": "Valletta", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MH", "iso3166_alpha3": "MHL", "iso3166_num": "584", "iso3166_name": "Marshall Islands", "name": "Marshall Islands", "languages": ["mh", "en-MH"], "tld": ".mh", "capital": "Majuro", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MQ", "iso3166_alpha3": "MTQ", "iso3166_num": "474", "iso3166_name": "Martinique", "name": "Martinique", "languages": ["fr-MQ"], "tld": ".mq", "capital": "Fort-de-France", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MR", "iso3166_alpha3": "MRT", "iso3166_num": "478", "iso3166_name": "Mauritania", "name": "Mauritania", "languages": ["ar-MR", "fuc", "snk", "fr", "mey", "wo"], "tld": ".mr", "capital": "Nouakchott", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MU", "iso3166_alpha3": "MUS", "iso3166_num": "480", "iso3166_name": "Mauritius", "name": "Mauritius", "languages": ["en-MU", "bho", "fr"], "tld": ".mu", "capital": "Port Louis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "YT", "iso3166_alpha3": "MYT", "iso3166_num": "175", "iso3166_name": "Mayotte", "name": "Mayotte", "languages": ["fr-YT"], "tld": ".yt", "capital": "Mamoudzou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MX", "iso3166_alpha3": "MEX", "iso3166_num": "484", "iso3166_name": "Mexico", "name": "Mexico", "languages": ["es-MX"], "tld": ".mx", "capital": "Mexico City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FM", "iso3166_alpha3": "FSM", "iso3166_num": "583", "iso3166_name": "Micronesia", "name": "Micronesia", "languages": ["en-FM", "chk", "pon", "yap", "kos", "uli", "woe", "nkr", "kpg"], "tld": ".fm", "capital": "Palikir", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MC", "iso3166_alpha3": "MCO", "iso3166_num": "492", "iso3166_name": "Monaco", "name": "Monaco", "languages": ["fr-MC", "en", "it"], "tld": ".mc", "capital": "Monaco", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MN", "iso3166_alpha3": "MNG", "iso3166_num": "496", "iso3166_name": "Mongolia", "name": "Mongolia", "languages": ["mn", "ru"], "tld": ".mn", "capital": "Ulaanbaatar", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "ME", "iso3166_alpha3": "MNE", "iso3166_num": "499", "iso3166_name": "Montenegro", "name": "Montenegro", "languages": ["sr", "hu", "bs", "sq", "hr", "rom"], "tld": ".me", "capital": "Podgorica", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MS", "iso3166_alpha3": "MSR", "iso3166_num": "500", "iso3166_name": "Montserrat", "name": "Montserrat", "languages": ["en-MS"], "tld": ".ms", "capital": "Plymouth", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MA", "iso3166_alpha3": "MAR", "iso3166_num": "504", "iso3166_name": "Morocco", "name": "Morocco", "languages": ["ar-MA", "ber", "fr"], "tld": ".ma", "capital": "Rabat", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "MZ", "iso3166_alpha3": "MOZ", "iso3166_num": "508", "iso3166_name": "Mozambique", "name": "Mozambique", "languages": ["pt-MZ", "vmw"], "tld": ".mz", "capital": "Maputo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MM", "iso3166_alpha3": "MMR", "iso3166_num": "104", "iso3166_name": "Myanmar", "name": "Myanmar", "languages": ["my"], "tld": ".mm", "capital": "Nay Pyi Taw", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "NA", "iso3166_alpha3": "NAM", "iso3166_num": "516", "iso3166_name": "Namibia", "name": "Namibia", "languages": ["en-NA", "af", "de", "hz", "naq"], "tld": ".na", "capital": "Windhoek", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NR", "iso3166_alpha3": "NRU", "iso3166_num": "520", "iso3166_name": "Nauru", "name": "Nauru", "languages": ["na", "en-NR"], "tld": ".nr", "capital": "Yaren", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NP", "iso3166_alpha3": "NPL", "iso3166_num": "524", "iso3166_name": "Nepal", "name": "Nepal", "languages": ["ne", "en"], "tld": ".np", "capital": "Kathmandu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "NL", "iso3166_alpha3": "NLD", "iso3166_num": "528", "iso3166_name": "Netherlands", "name": "Netherlands", "languages": ["nl-NL", "fy-NL"], "tld": ".nl", "capital": "Amsterdam", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "NC", "iso3166_alpha3": "NCL", "iso3166_num": "540", "iso3166_name": "New Caledonia", "name": "New Caledonia", "languages": ["fr-NC"], "tld": ".nc", "capital": "Noumea", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "NZ", "iso3166_alpha3": "NZL", "iso3166_num": "554", "iso3166_name": "New Zealand", "name": "New Zealand", "languages": ["en-NZ", "mi"], "tld": ".nz", "capital": "Wellington", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "NI", "iso3166_alpha3": "NIC", "iso3166_num": "558", "iso3166_name": "Nicaragua", "name": "Nicaragua", "languages": ["es-NI", "en"], "tld": ".ni", "capital": "Managua", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "NE", "iso3166_alpha3": "NER", "iso3166_num": "562", "iso3166_name": "Niger", "name": "Niger", "languages": ["fr-NE", "ha", "kr", "dje"], "tld": ".ne", "capital": "Niamey", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NG", "iso3166_alpha3": "NGA", "iso3166_num": "566", "iso3166_name": "Nigeria", "name": "Nigeria", "languages": ["en-NG", "ha", "yo", "ig", "ff"], "tld": ".ng", "capital": "Abuja", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NU", "iso3166_alpha3": "NIU", "iso3166_num": "570", "iso3166_name": "Niue", "name": "Niue", "languages": ["niu", "en-NU"], "tld": ".nu", "capital": "Alofi", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "NF", "iso3166_alpha3": "NFK", "iso3166_num": "574", "iso3166_name": "Norfolk Island", "name": "Norfolk Island", "languages": ["en-NF"], "tld": ".nf", "capital": "Kingston", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "MP", "iso3166_alpha3": "MNP", "iso3166_num": "580", "iso3166_name": "Northern Mariana Islands", "name": "Northern Mariana Islands", "languages": ["fil", "tl", "zh", "ch-MP", "en-MP"], "tld": ".mp", "capital": "Saipan", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NO", "iso3166_alpha3": "NOR", "iso3166_num": "578", "iso3166_name": "Norway", "name": "Norway", "languages": ["no", "nb", "nn", "se", "fi"], "tld": ".no", "capital": "Oslo", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "OM", "iso3166_alpha3": "OMN", "iso3166_num": "512", "iso3166_name": "Oman", "name": "Oman", "languages": ["ar-OM", "en", "bal", "ur"], "tld": ".om", "capital": "Muscat", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "PK", "iso3166_alpha3": "PAK", "iso3166_num": "586", "iso3166_name": "Pakistan", "name": "Pakistan", "languages": ["ur-PK", "en-PK", "pa", "sd", "ps", "brh"], "tld": ".pk", "capital": "Islamabad", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PW", "iso3166_alpha3": "PLW", "iso3166_num": "585", "iso3166_name": "Palau", "name": "Palau", "languages": ["pau", "sov", "en-PW", "tox", "ja", "fil", "zh"], "tld": ".pw", "capital": "Melekeok", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "PA", "iso3166_alpha3": "PAN", "iso3166_num": "591", "iso3166_name": "Panama", "name": "Panama", "languages": ["es-PA", "en"], "tld": ".pa", "capital": "Panama City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PG", "iso3166_alpha3": "PNG", "iso3166_num": "598", "iso3166_name": "Papua New Guinea", "name": "Papua New Guinea", "languages": ["en-PG", "ho", "meu", "tpi"], "tld": ".pg", "capital": "Port Moresby", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "PY", "iso3166_alpha3": "PRY", "iso3166_num": "600", "iso3166_name": "Paraguay", "name": "Paraguay", "languages": ["es-PY", "gn"], "tld": ".py", "capital": "Asuncion", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PE", "iso3166_alpha3": "PER", "iso3166_num": "604", "iso3166_name": "Peru", "name": "Peru", "languages": ["es-PE", "qu", "ay"], "tld": ".pe", "capital": "Lima", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PH", "iso3166_alpha3": "PHL", "iso3166_num": "608", "iso3166_name": "Philippines", "name": "Philippines", "languages": ["tl", "en-PH", "fil", "ceb", "tgl", "ilo", "hil", "war", "pam", "bik", "bcl", "pag", "mrw", "tsg", "mdh", "cbk", "krj", "sgd", "msb", "akl", "ibg", "yka", "mta", "abx"], "tld": ".ph", "capital": "Manila", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "PN", "iso3166_alpha3": "PCN", "iso3166_num": "612", "iso3166_name": "Pitcairn Islands", "name": "Pitcairn Islands", "languages": ["en-PN"], "tld": ".pn", "capital": "Adamstown", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "PL", "iso3166_alpha3": "POL", "iso3166_num": "616", "iso3166_name": "Poland", "name": "Poland", "languages": ["pl"], "tld": ".pl", "capital": "Warsaw", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "PT", "iso3166_alpha3": "PRT", "iso3166_num": "620", "iso3166_name": "Portugal", "name": "Portugal", "languages": ["pt-PT", "mwl"], "tld": ".pt", "capital": "Lisbon", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "PR", "iso3166_alpha3": "PRI", "iso3166_num": "630", "iso3166_name": "Puerto Rico", "name": "Puerto Rico", "languages": ["en-PR", "es-PR"], "tld": ".pr", "capital": "San Juan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "QA", "iso3166_alpha3": "QAT", "iso3166_num": "634", "iso3166_name": "Qatar", "name": "Qatar", "languages": ["ar-QA", "es"], "tld": ".qa", "capital": "Doha", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KR", "iso3166_alpha3": "KOR", "iso3166_num": "410", "iso3166_name": "South Korea", "name": "South Korea", "languages": ["ko-KR", "en"], "tld": ".kr", "capital": "Seoul", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MD", "iso3166_alpha3": "MDA", "iso3166_num": "498", "iso3166_name": "Moldova", "name": "Moldova", "languages": ["ro", "ru", "gag", "tr"], "tld": ".md", "capital": "Chisinau", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RO", "iso3166_alpha3": "ROU", "iso3166_num": "642", "iso3166_name": "Romania", "name": "Romania", "languages": ["ro", "hu", "rom"], "tld": ".ro", "capital": "Bucharest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RU", "iso3166_alpha3": "RUS", "iso3166_num": "643", "iso3166_name": "Russia", "name": "Russia", "languages": ["ru", "tt", "xal", "cau", "ady", "kv", "ce", "tyv", "cv", "udm", "tut", "mns", "bua", "myv", "mdf", "chm", "ba", "inh", "tut", "kbd", "krc", "av", "sah", "nog"], "tld": ".ru", "capital": "Moscow", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RW", "iso3166_alpha3": "RWA", "iso3166_num": "646", "iso3166_name": "Rwanda", "name": "Rwanda", "languages": ["rw", "en-RW", "fr-RW", "sw"], "tld": ".rw", "capital": "Kigali", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RE", "iso3166_alpha3": "REU", "iso3166_num": "638", "iso3166_name": "R\u00e9union", "name": "R\u00e9union", "languages": ["fr-RE"], "tld": ".re", "capital": "Saint-Denis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BL", "iso3166_alpha3": "BLM", "iso3166_num": "652", "iso3166_name": "St. Barth\u00e9lemy", "name": "St. Barth\u00e9lemy", "languages": ["fr"], "tld": ".gp", "capital": "Gustavia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SH", "iso3166_alpha3": "SHN", "iso3166_num": "654", "iso3166_name": "St. Helena", "name": "St. Helena", "languages": ["en-SH"], "tld": ".sh", "capital": "Jamestown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KN", "iso3166_alpha3": "KNA", "iso3166_num": "659", "iso3166_name": "St. Kitts & Nevis", "name": "St. Kitts & Nevis", "languages": ["en-KN"], "tld": ".kn", "capital": "Basseterre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "LC", "iso3166_alpha3": "LCA", "iso3166_num": "662", "iso3166_name": "St. Lucia", "name": "St. Lucia", "languages": ["en-LC"], "tld": ".lc", "capital": "Castries", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MF", "iso3166_alpha3": "MAF", "iso3166_num": "663", "iso3166_name": "St. Martin", "name": "St. Martin", "languages": ["fr"], "tld": ".gp", "capital": "Marigot", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PM", "iso3166_alpha3": "SPM", "iso3166_num": "666", "iso3166_name": "St. Pierre & Miquelon", "name": "St. Pierre & Miquelon", "languages": ["fr-PM"], "tld": ".pm", "capital": "Saint-Pierre", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "VC", "iso3166_alpha3": "VCT", "iso3166_num": "670", "iso3166_name": "St. Vincent & Grenadines", "name": "St. Vincent & Grenadines", "languages": ["en-VC", "fr"], "tld": ".vc", "capital": "Kingstown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "WS", "iso3166_alpha3": "WSM", "iso3166_num": "882", "iso3166_name": "Samoa", "name": "Samoa", "languages": ["sm", "en-WS"], "tld": ".ws", "capital": "Apia", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "SM", "iso3166_alpha3": "SMR", "iso3166_num": "674", "iso3166_name": "San Marino", "name": "San Marino", "languages": ["it-SM"], "tld": ".sm", "capital": "San Marino", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "ST", "iso3166_alpha3": "STP", "iso3166_num": "678", "iso3166_name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "languages": ["pt-ST"], "tld": ".st", "capital": "Sao Tome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SA", "iso3166_alpha3": "SAU", "iso3166_num": "682", "iso3166_name": "Saudi Arabia", "name": "Saudi Arabia", "languages": ["ar-SA"], "tld": ".sa", "capital": "Riyadh", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SN", "iso3166_alpha3": "SEN", "iso3166_num": "686", "iso3166_name": "Senegal", "name": "Senegal", "languages": ["fr-SN", "wo", "fuc", "mnk"], "tld": ".sn", "capital": "Dakar", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RS", "iso3166_alpha3": "SRB", "iso3166_num": "688", "iso3166_name": "Serbia", "name": "Serbia", "languages": ["sr", "hu", "bs", "rom"], "tld": ".rs", "capital": "Belgrade", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SC", "iso3166_alpha3": "SYC", "iso3166_num": "690", "iso3166_name": "Seychelles", "name": "Seychelles", "languages": ["en-SC", "fr-SC"], "tld": ".sc", "capital": "Victoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SL", "iso3166_alpha3": "SLE", "iso3166_num": "694", "iso3166_name": "Sierra Leone", "name": "Sierra Leone", "languages": ["en-SL", "men", "tem"], "tld": ".sl", "capital": "Freetown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SG", "iso3166_alpha3": "SGP", "iso3166_num": "702", "iso3166_name": "Singapore", "name": "Singapore", "languages": ["cmn", "en-SG", "ms-SG", "ta-SG", "zh-SG"], "tld": ".sg", "capital": "Singapore", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "SX", "iso3166_alpha3": "SXM", "iso3166_num": "534", "iso3166_name": "Sint Maarten", "name": "Sint Maarten", "languages": ["nl", "en"], "tld": ".sx", "capital": "Philipsburg", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SK", "iso3166_alpha3": "SVK", "iso3166_num": "703", "iso3166_name": "Slovakia", "name": "Slovakia", "languages": ["sk", "hu"], "tld": ".sk", "capital": "Bratislava", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "SI", "iso3166_alpha3": "SVN", "iso3166_num": "705", "iso3166_name": "Slovenia", "name": "Slovenia", "languages": ["sl", "sh"], "tld": ".si", "capital": "Ljubljana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SB", "iso3166_alpha3": "SLB", "iso3166_num": "090", "iso3166_name": "Solomon Islands", "name": "Solomon Islands", "languages": ["en-SB", "tpi"], "tld": ".sb", "capital": "Honiara", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "SO", "iso3166_alpha3": "SOM", "iso3166_num": "706", "iso3166_name": "Somalia", "name": "Somalia", "languages": ["so-SO", "ar-SO", "it", "en-SO"], "tld": ".so", "capital": "Mogadishu", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZA", "iso3166_alpha3": "ZAF", "iso3166_num": "710", "iso3166_name": "South Africa", "name": "South Africa", "languages": ["zu", "xh", "af", "nso", "en-ZA", "tn", "st", "ts", "ss", "ve", "nr"], "tld": ".za", "capital": "Pretoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GS", "iso3166_alpha3": "SGS", "iso3166_num": "239", "iso3166_name": "South Georgia & South Sandwich Islands", "name": "South Georgia & South Sandwich Islands", "languages": ["en"], "tld": ".gs", "capital": "Grytviken", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SS", "iso3166_alpha3": "SSD", "iso3166_num": "728", "iso3166_name": "South Sudan", "name": "South Sudan", "languages": ["en"], "tld": "", "capital": "Juba", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ES", "iso3166_alpha3": "ESP", "iso3166_num": "724", "iso3166_name": "Spain", "name": "Spain", "languages": ["es-ES", "ca", "gl", "eu", "oc"], "tld": ".es", "capital": "Madrid", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "LK", "iso3166_alpha3": "LKA", "iso3166_num": "144", "iso3166_name": "Sri Lanka", "name": "Sri Lanka", "languages": ["si", "ta", "en"], "tld": ".lk", "capital": "Colombo", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PS", "iso3166_alpha3": "PSE", "iso3166_num": "275", "iso3166_name": "Palestine", "name": "Palestine", "languages": ["ar-PS"], "tld": ".ps", "capital": "East Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SD", "iso3166_alpha3": "SDN", "iso3166_num": "729", "iso3166_name": "Sudan", "name": "Sudan", "languages": ["ar-SD", "en", "fia"], "tld": ".sd", "capital": "Khartoum", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SR", "iso3166_alpha3": "SUR", "iso3166_num": "740", "iso3166_name": "Suriname", "name": "Suriname", "languages": ["nl-SR", "en", "srn", "hns", "jv"], "tld": ".sr", "capital": "Paramaribo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SJ", "iso3166_alpha3": "SJM", "iso3166_num": "744", "iso3166_name": "Svalbard & Jan Mayen", "name": "Svalbard & Jan Mayen", "languages": ["no", "ru"], "tld": ".sj", "capital": "Longyearbyen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "SZ", "iso3166_alpha3": "SWZ", "iso3166_num": "748", "iso3166_name": "Swaziland", "name": "Eswatini", "languages": ["en-SZ", "ss-SZ"], "tld": ".sz", "capital": "Mbabane", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SE", "iso3166_alpha3": "SWE", "iso3166_num": "752", "iso3166_name": "Sweden", "name": "Sweden", "languages": ["sv-SE", "se", "sma", "fi-SE"], "tld": ".se", "capital": "Stockholm", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "CH", "iso3166_alpha3": "CHE", "iso3166_num": "756", "iso3166_name": "Switzerland", "name": "Switzerland", "languages": ["de-CH", "fr-CH", "it-CH", "rm"], "tld": ".ch", "capital": "Bern", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "SY", "iso3166_alpha3": "SYR", "iso3166_num": "760", "iso3166_name": "Syria", "name": "Syria", "languages": ["ar-SY", "ku", "hy", "arc", "fr", "en"], "tld": ".sy", "capital": "Damascus", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TJ", "iso3166_alpha3": "TJK", "iso3166_num": "762", "iso3166_name": "Tajikistan", "name": "Tajikistan", "languages": ["tg", "ru"], "tld": ".tj", "capital": "Dushanbe", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TH", "iso3166_alpha3": "THA", "iso3166_num": "764", "iso3166_name": "Thailand", "name": "Thailand", "languages": ["th", "en"], "tld": ".th", "capital": "Bangkok", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MK", "iso3166_alpha3": "MKD", "iso3166_num": "807", "iso3166_name": "Macedonia", "name": "North Macedonia", "languages": ["mk", "sq", "tr", "rmm", "sr"], "tld": ".mk", "capital": "Skopje", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "TL", "iso3166_alpha3": "TLS", "iso3166_num": "626", "iso3166_name": "Timor-Leste", "name": "Timor-Leste", "languages": ["tet", "pt-TL", "id", "en"], "tld": ".tl", "capital": "Dili", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "TG", "iso3166_alpha3": "TGO", "iso3166_num": "768", "iso3166_name": "Togo", "name": "Togo", "languages": ["fr-TG", "ee", "hna", "kbp", "dag", "ha"], "tld": ".tg", "capital": "Lome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TK", "iso3166_alpha3": "TKL", "iso3166_num": "772", "iso3166_name": "Tokelau", "name": "Tokelau", "languages": ["tkl", "en-TK"], "tld": ".tk", "capital": "", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TO", "iso3166_alpha3": "TON", "iso3166_num": "776", "iso3166_name": "Tonga", "name": "Tonga", "languages": ["to", "en-TO"], "tld": ".to", "capital": "Nuku'alofa", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TT", "iso3166_alpha3": "TTO", "iso3166_num": "780", "iso3166_name": "Trinidad & Tobago", "name": "Trinidad & Tobago", "languages": ["en-TT", "hns", "fr", "es", "zh"], "tld": ".tt", "capital": "Port of Spain", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TN", "iso3166_alpha3": "TUN", "iso3166_num": "788", "iso3166_name": "Tunisia", "name": "Tunisia", "languages": ["ar-TN", "fr"], "tld": ".tn", "capital": "Tunis", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "TR", "iso3166_alpha3": "TUR", "iso3166_num": "792", "iso3166_name": "Turkey", "name": "Turkey", "languages": ["tr-TR", "ku", "diq", "az", "av"], "tld": ".tr", "capital": "Ankara", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TM", "iso3166_alpha3": "TKM", "iso3166_num": "795", "iso3166_name": "Turkmenistan", "name": "Turkmenistan", "languages": ["tk", "ru", "uz"], "tld": ".tm", "capital": "Ashgabat", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TC", "iso3166_alpha3": "TCA", "iso3166_num": "796", "iso3166_name": "Turks & Caicos Islands", "name": "Turks & Caicos Islands", "languages": ["en-TC"], "tld": ".tc", "capital": "Cockburn Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TV", "iso3166_alpha3": "TUV", "iso3166_num": "798", "iso3166_name": "Tuvalu", "name": "Tuvalu", "languages": ["tvl", "en", "sm", "gil"], "tld": ".tv", "capital": "Funafuti", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "UG", "iso3166_alpha3": "UGA", "iso3166_num": "800", "iso3166_name": "Uganda", "name": "Uganda", "languages": ["en-UG", "lg", "sw", "ar"], "tld": ".ug", "capital": "Kampala", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UA", "iso3166_alpha3": "UKR", "iso3166_num": "804", "iso3166_name": "Ukraine", "name": "Ukraine", "languages": ["uk", "ru-UA", "rom", "pl", "hu"], "tld": ".ua", "capital": "Kyiv", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "AE", "iso3166_alpha3": "ARE", "iso3166_num": "784", "iso3166_name": "United Arab Emirates", "name": "United Arab Emirates", "languages": ["ar-AE", "fa", "en", "hi", "ur"], "tld": ".ae", "capital": "Abu Dhabi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "GB", "iso3166_alpha3": "GBR", "iso3166_num": "826", "iso3166_name": "UK", "name": "United Kingdom", "languages": ["en-GB", "cy-GB", "gd"], "tld": ".uk", "capital": "London", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "TZ", "iso3166_alpha3": "TZA", "iso3166_num": "834", "iso3166_name": "Tanzania", "name": "Tanzania", "languages": ["sw-TZ", "en", "ar"], "tld": ".tz", "capital": "Dodoma", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UM", "iso3166_alpha3": "UMI", "iso3166_num": "581", "iso3166_name": "U.S. Outlying Islands", "name": "U.S. Outlying Islands", "languages": ["en-UM"], "tld": ".um", "capital": "", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "VI", "iso3166_alpha3": "VIR", "iso3166_num": "850", "iso3166_name": "U.S. Virgin Islands", "name": "U.S. Virgin Islands", "languages": ["en-VI"], "tld": ".vi", "capital": "Charlotte Amalie", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "US", "iso3166_alpha3": "USA", "iso3166_num": "840", "iso3166_name": "US", "name": "United States", "languages": ["en-US", "es-US", "haw", "fr"], "tld": ".us", "capital": "Washington", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "UY", "iso3166_alpha3": "URY", "iso3166_num": "858", "iso3166_name": "Uruguay", "name": "Uruguay", "languages": ["es-UY"], "tld": ".uy", "capital": "Montevideo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "UZ", "iso3166_alpha3": "UZB", "iso3166_num": "860", "iso3166_name": "Uzbekistan", "name": "Uzbekistan", "languages": ["uz", "ru", "tg"], "tld": ".uz", "capital": "Tashkent", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "VU", "iso3166_alpha3": "VUT", "iso3166_num": "548", "iso3166_name": "Vanuatu", "name": "Vanuatu", "languages": ["bi", "en-VU", "fr-VU"], "tld": ".vu", "capital": "Port Vila", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "VE", "iso3166_alpha3": "VEN", "iso3166_num": "862", "iso3166_name": "Venezuela", "name": "Venezuela", "languages": ["es-VE"], "tld": ".ve", "capital": "Caracas", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "VN", "iso3166_alpha3": "VNM", "iso3166_num": "704", "iso3166_name": "Vietnam", "name": "Vietnam", "languages": ["vi", "en", "fr", "zh", "km"], "tld": ".vn", "capital": "Hanoi", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "WF", "iso3166_alpha3": "WLF", "iso3166_num": "876", "iso3166_name": "Wallis & Futuna", "name": "Wallis & Futuna", "languages": ["wls", "fud", "fr-WF"], "tld": ".wf", "capital": "Mata Utu", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "EH", "iso3166_alpha3": "ESH", "iso3166_num": "732", "iso3166_name": "Western Sahara", "name": "Western Sahara", "languages": ["ar", "mey"], "tld": ".eh", "capital": "El-Aaiun", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "YE", "iso3166_alpha3": "YEM", "iso3166_num": "887", "iso3166_name": "Yemen", "name": "Yemen", "languages": ["ar-YE"], "tld": ".ye", "capital": "Sanaa", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "ZM", "iso3166_alpha3": "ZMB", "iso3166_num": "894", "iso3166_name": "Zambia", "name": "Zambia", "languages": ["en-ZM", "bem", "loz", "lun", "lue", "ny", "toi"], "tld": ".zm", "capital": "Lusaka", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZW", "iso3166_alpha3": "ZWE", "iso3166_num": "716", "iso3166_name": "Zimbabwe", "name": "Zimbabwe", "languages": ["en-ZW", "sn", "nr", "nd"], "tld": ".zw", "capital": "Harare", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AX", "iso3166_alpha3": "ALA", "iso3166_num": "248", "iso3166_name": "\u00c5land Islands", "name": "\u00c5land Islands", "languages": ["sv-AX"], "tld": ".ax", "capital": "Mariehamn", "region_code": "150", "sub_region_code": "154"}] From 18e6df08bd682588d2e1d2398407630ded5dab09 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:32:34 +0200 Subject: [PATCH 035/105] add pycountry for countries --- ooniapi/services/ooniprobe/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml index 1d87bd2ae..23f728e21 100644 --- a/ooniapi/services/ooniprobe/pyproject.toml +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "ujson ~= 5.9.0", "urllib3 ~= 2.1.0", "python-dateutil ~= 2.8.2", + "pycountry ~= 26.2.16", "pydantic-settings ~= 2.1.0", "statsd ~= 4.0.1", "uvicorn ~= 0.25.0", From 16f1af76bee856e8dc02846b7d7b18aaf7b917a5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:32:55 +0200 Subject: [PATCH 036/105] import types from typing --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 05c2fc641..18d010923 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -9,7 +9,7 @@ from itertools import product from urllib.parse import urljoin, urlencode -from typing import Dict, Any, Tuple +from typing import Annotated, Dict, Any, Tuple, List, Optional import logging import math From 3c60bf0bc84f56d698dd7eed5f52ef586ff48c1e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:33:18 +0200 Subject: [PATCH 037/105] remove old api imports --- .../ooniprobe/src/ooniprobe/routers/private.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 18d010923..46314bf6a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -16,22 +16,6 @@ from sqlalchemy import sql -from ooniapi.auth import role_required -from ooniapi.config import metrics -from ooniapi.countries import lookup_country -from ooniapi.data import dnscheck_inputs, stunreachability_inputs -from ooniapi.models import TEST_GROUPS -from ooniapi.prio import generate_test_list -from ooniapi.utils import cachedjson, nocachejson, jerror, req_json - -from ooniapi.probe_services import ( - probe_geoip, - extract_probe_ipaddr_octect, - generate_test_helpers_conf, - generate_report_id, -) - - from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 from pydantic import AnyUrl, conint, constr From f06c0d4eb1214f67a1f8d3d37730b3575a80bec2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:33:50 +0200 Subject: [PATCH 038/105] import pydantic Field --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 46314bf6a..5fea8a694 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -18,7 +18,7 @@ from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 -from pydantic import AnyUrl, conint, constr +from pydantic import AnyUrl, conint, Field from ..common.clickhouse_utils import query_click, query_click_one_row from ..common.dependencies import role_required From 1b16703510a19f77a3a5c2aeaff9943aa0833b08 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:35:01 +0200 Subject: [PATCH 039/105] fix deps from new api --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5fea8a694..abbab6b33 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -20,9 +20,15 @@ from pydantic_extra_types.country import CountryAlpha2 from pydantic import AnyUrl, conint, Field +from .v1.probe_services import probe_geoip, generate_test_helpers_conf from ..common.clickhouse_utils import query_click, query_click_one_row from ..common.dependencies import role_required +from ..common.prio import generate_test_list from ..common.routers import BaseModel +from ..metrics import Metrics +from ..countries import lookup_country +from ..data import dnscheck_inputs, stunreachability_inputs +from ..utils import generate_report_id # The private API is exposed under the prefix /api/_ From 01c1e5adb913ce554b1d501988da41227d344596 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:36:31 +0200 Subject: [PATCH 040/105] fix types --- .../src/ooniprobe/routers/private.py | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index abbab6b33..4478818e8 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -157,10 +157,9 @@ class TestName(BaseModel): id: str = Field(..., description="test id") name: str = Field(..., description="test name") -TestNamesResponse: List[TestName] -@router.get("/test_names", tags=["private_api"], response_model=TestNamesResponse) -def api_private_test_names() -> Response: +@router.get("/test_names", tags=["private_api"], response_model=List[TestName]) +def api_private_test_names() -> List[TestName]: """Provides test names and descriptions to Explorer --- responses: @@ -194,7 +193,7 @@ def api_private_test_names() -> Response: "web_connectivity": "Web Connectivity", "whatsapp": "WhatsApp", } - validated: TestNamesResponse = [TestName(id=k, name=v) for k, v in TEST_NAMES.items()] + validated: List[TestNames] = [TestName(id=k, name=v) for k, v in TEST_NAMES.items()] return validated @@ -204,11 +203,8 @@ class CountryStat(BaseModel): name: str = Field(..., description="Country Name") -AllCountryStats : List[CountryStat] - - -@router.get("/countries", tags=["private_api"], response_model=AllCountryStats) -def api_private_countries() -> AllCountryStats: +@router.get("/countries", tags=["private_api"], response_model=List[CountryStat]) +def api_private_countries() -> List[CountryStat]: """Summary of countries --- responses: @@ -231,17 +227,17 @@ def api_private_countries() -> AllCountryStats: except KeyError: pass - validated: AllCountryStats = c + validated: List[CountryStat] = c return validated @router.get( "/quotas_summary", - response_model=AllCountryStats, + response_model=List[CountryStat], tags=["private_api"], dependencies=[Depends(role_required(["admin"]))], ) -def api_private_quotas_summary() -> Response: +def api_private_quotas_summary() -> List[CountryStat]: """Summary on rate-limiting quotas. [(first ipaddr octet, remaining daily quota), ... ] """ @@ -416,13 +412,10 @@ class MeasurementsByASN(BaseModel): probe_asn: str = Field(..., description="Autonomous System Number") -WebsiteNetworksResponse: List[MeasurementsByASN] - - -@router.get("/website_networks", response_model=WebsiteNetworksResponse, tags=["private"]) +@router.get("/website_networks", response_model=List[MeasurementsByASN], tags=["private"]) def api_private_website_network_tests( probe_cc: CountryAlpha2 = Query(..., description="Country Code") - ) -> WebsiteNetworksResponse: + ) -> List[MeasurementsByASN]: """TODO --- parameters: @@ -446,7 +439,7 @@ def api_private_website_network_tests( ORDER BY count DESC """ results = query_click(sql.text(s), {"probe_cc": probe_cc}) - validated: WebsiteNetworksResponse = [MeasurementsByASN(**x) for x in results] + validated: List[MeasurementsByASN] = [MeasurementsByASN(**x) for x in results] return validated class DayStats(BaseModel): @@ -685,13 +678,10 @@ class IMNetworkStats(BaseModel): last_tested: date -IMNetworksResponse: Dict[str, IMNetworksStats] - - -@router.get("/im_networks", response_model=IMNetworksResponse, tags=["private"]) +@router.get("/im_networks", response_model=Dict[str, IMNetworkStats], tags=["private"]) def api_private_im_networks( probe_cc: CountryAlpha2 = Query(..., description="Country Code") -) -> IMNetworksResponse: +) -> Dict[str, IMNetworkStats]: """Instant messaging networks statistics --- responses: From 5597e4d329706f87e441cbd76d4a244ef7be5028 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:37:50 +0200 Subject: [PATCH 041/105] fix typos --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 4478818e8..ba71d9f07 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -454,7 +454,7 @@ class WebsiteStatsResponse(BaseModel): @router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) def api_private_website_stats( - input: AnyUrl = Query(..., "Website to query stats"), + input: AnyUrl = Query(..., description="Website to query stats"), probe_cc: CountryAlpha2 = Query(..., description="Country Code"), probe_asn: int = Query(..., description="ASN (integer)"), ) -> Response: @@ -707,7 +707,7 @@ def api_private_im_networks( results = IMNetworksResponse() for r in q: # get stats for test_name or create a new IMNetworksStats - stats = results.get(r["test_name"], IMNetworkStats(anomaly_networks=[], ok_networks=[], last_tested)) + stats = results.get(r["test_name"], IMNetworkStats(anomaly_networks=[], ok_networks=[], last_tested=None)) # create and add a new entry entry = NetworkEntry(asn=r["probe_asn"], name="", total_count=r["total_count"], last_tested=r["last_tested"]) @@ -1199,7 +1199,7 @@ def api_private_networks() -> Response: """ try: results = query_click(sql.text(q), {}) - return MeasuredNetworksResponse(results=results v=0) + return MeasuredNetworksResponse(results=results, v=0) except Exception as e: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) From f54bbabd648dd4cdd7a3aface5a0ef2ea60667ad Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:38:08 +0200 Subject: [PATCH 042/105] add pydantic_extra_types for country code validator --- ooniapi/services/ooniprobe/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ooniapi/services/ooniprobe/pyproject.toml b/ooniapi/services/ooniprobe/pyproject.toml index 23f728e21..b66269ce6 100644 --- a/ooniapi/services/ooniprobe/pyproject.toml +++ b/ooniapi/services/ooniprobe/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "python-dateutil ~= 2.8.2", "pycountry ~= 26.2.16", "pydantic-settings ~= 2.1.0", + "pydantic_extra_types ~= 2.11.1", "statsd ~= 4.0.1", "uvicorn ~= 0.25.0", "psycopg2 ~= 2.9.9", From bacbd061e033bdccb7b4354e844891e20d6cc1a9 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:38:24 +0200 Subject: [PATCH 043/105] fix conflicted field name --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ba71d9f07..ddff8b545 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -980,7 +980,7 @@ def api_private_circumvention_stats_by_country() -> Response: class CircumventionRuntimeStat(BaseModel): test_name: str = Field(..., description="Name of test") probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") - date: date = Field(..., description="Date of metric") + metric_date: date = Field(..., alias="date", description="Date of metric") v: Tuple[float, float, int] = Field((), description="(p50, p90, cnt)") From 5002c548d942c957e1aab6eb27c3948e8437e2f6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:38:51 +0200 Subject: [PATCH 044/105] fix spacing --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ddff8b545..5d406da76 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -911,6 +911,7 @@ class GlobalOverviewMonthResponse(BaseModel): countries_by_month: List[GlobalOverviewStat] measurement_by_month: List[GlobalOverviewStat] + @router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) def api_private_global_by_month() -> Response: """Provide global summary of measurements From 27500915339e0def545b5d708ac03c0cbc3544b6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:39:41 +0200 Subject: [PATCH 045/105] fix model validator DomainStr --- .../ooniprobe/src/ooniprobe/routers/private.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5d406da76..e5397eb4d 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1045,10 +1045,13 @@ def api_private_circumvention_runtime_stats() -> Response: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) -DomainStr = constr( - strip_whitespace=True, - regex=r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[A-Za-z]{2,}$" -) +DomainStr = Annotated[ + str, + Field( + strip_whitespace=True, + pattern=r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[A-Za-z]{2,}$" + ) +] class DomainMetadataResponse(BaseModel): From b517b3eb3234323c33e4e43cb86515e75b2e61ad Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:40:04 +0200 Subject: [PATCH 046/105] fix type --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index e5397eb4d..a9b762a6a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1126,7 +1126,7 @@ def api_private_domain_metadata( class ASNMetadataResponse(BaseModel): - org_name: string = Field("Unknown", description="ORG Name of ASN") + org_name: str = Field("Unknown", description="ORG Name of ASN") @router.get("/asnmeta", response_model=ASNMetadataResponse, tags=["private"]) From 7e0c6fbcaaab7c337c990f98e32aa46091d65030 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:40:40 +0200 Subject: [PATCH 047/105] fix Field usage --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a9b762a6a..88cc70988 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -873,9 +873,9 @@ def api_private_country_overview( class GlobalOverviewResponse(BaseModel): - network_count: int = Field(..., "Number of networks measured") - country_count: int = Field(..., "Nubmer of countries measured") - measurement_count: int = Field(..., "Number of total measurements") + network_count: int = Field(..., description="Number of networks measured") + country_count: int = Field(..., description="Number of countries measured") + measurement_count: int = Field(..., description="Number of total measurements") @router.get("/global_overview", response_model=GlobalOverviewResponse, tags=["private"]) From 24fa684ddda5e268b0f18349a7e8fae2ed923bb9 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:41:46 +0200 Subject: [PATCH 048/105] remove check-in, not on migration list --- .../src/ooniprobe/routers/private.py | 222 ------------------ 1 file changed, 222 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 88cc70988..ff986d1b3 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1255,225 +1255,3 @@ def api_private_domains() -> DomainsMeasuredResponse: return DomainsMeasuredResponse(v=0, results=results) except Exception as e: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) - - -@api_private_blueprint.route("/check-in", methods=["POST"]) -def private_api_check_in() -> Response: - """Private, experimental check-in - --- - produces: - - application/json - consumes: - - application/json - parameters: - - in: body - name: probe self-description - required: true - schema: - type: object - properties: - probe_cc: - type: string - description: Two letter, uppercase country code - example: IT - probe_asn: - type: string - description: ASN, two uppercase letters followed by number - example: AS1234 - platform: - type: string - example: android - software_name: - type: string - example: ooniprobe - software_version: - type: string - example: 0.0.1 - on_wifi: - type: boolean - charging: - description: set only on devices with battery; true when charging - type: boolean - run_type: - type: string - description: timed or manual - example: timed - web_connectivity: - type: object - properties: - category_codes: - description: List/array of URL categories, all uppercase - type: array - items: - type: string - example: NEWS - description: probe_asn and probe_cc are not provided if unknown - - responses: - '200': - description: Give a URL test list to a probe running web_connectivity - tests; additional data for other tests; - schema: - type: object - properties: - v: - type: integer - description: response format version - probe_cc: - type: string - description: probe CC inferred from GeoIP or ZZ - probe_asn: - type: string - description: probe ASN inferred from GeoIP or AS0 - probe_network_name: - type: string - description: probe network name inferred from GeoIP or None - utc_time: - type: string - description: current UTC time as YYYY-mm-ddTHH:MM:SSZ - conf: - type: object - description: auxiliary configuration parameters - features: - type: object - description: feature flags - nettests: - type: array - items: - type: object - description: test-specific configuration - properties: - test_name: string - report_id: string - targets: - type: array - items: - type: object - properties: - attributes: - type: object - input: string - options: - type: object - - """ - # TODO: Implement throttling - data = req_json() - run_type = data.get("run_type", "timed") - charging = data.get("charging", True) - probe_cc = data.get("probe_cc", "ZZ").upper() - probe_asn = data.get("probe_asn", "AS0") - - resp, probe_cc, asn_i = probe_geoip(probe_cc, probe_asn) - resp["v"] = 2 - - # On run_type=manual preserve the old behavior: test the whole list - # On timed runs test few URLs, especially when on battery - if run_type == "manual": - url_limit = 9999 # same as prio.py - elif charging: - url_limit = 100 - else: - url_limit = 20 - - if "web_connectivity" in data: - catcodes = data["web_connectivity"].get("category_codes", []) - if isinstance(catcodes, str): - category_codes = catcodes.split(",") - else: - category_codes = catcodes - - else: - category_codes = [] - - for c in category_codes: - assert c.isalpha() - - try: - webconn_test_items, _1, _2 = generate_test_list( - probe_cc, category_codes, asn_i, url_limit, False - ) - except Exception as e: - log.error(e, exc_info=True) - # TODO: use same failover as prio.py:list_test_urls - # failover_generate_test_list runs without any database interaction - # test_items = failover_generate_test_list(country_code, category_codes, limit) - webconn_test_items = [] - - metrics.gauge("check-in-test-list-count", len(webconn_test_items)) - conf: Dict[str, Any] = dict(features={}) - - # set webconnectivity_0.5 feature flag for some probes - octect = extract_probe_ipaddr_octect(1, 0) - if octect in (34, 239): - conf["features"]["webconnectivity_0.5"] = True - - conf["test_helpers"] = generate_test_helpers_conf() - - resp["conf"] = conf - resp["utc_time"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - - test_names = ( - "bridge_reachability", - "dash", - "dns_consistency", - "dnscheck", - "dnsping", - "echcheck", - "facebook_messenger", - "http_header_field_manipulation", - "http_host", - "http_invalid_request_line", - "http_requests", - "meek_fronted_requests_test", - "multi_protocol_traceroute", - "ndt", - "portfiltering", - "psiphon", - "quicping", - "riseupvpn", - "signal", - "simplequicping", - "sniblocking", - "stunreachability", - "tcp_connect", - "tcpping", - "telegram", - "tlsmiddlebox", - "tlsping", - "tor", - "torsf", - "urlgetter", - "vanilla_tor", - "web_connectivity", - "whatsapp", - ) - resp["nettests"] = [] - # Add one example for each nettest - for tn in test_names: - rid = generate_report_id(tn, probe_cc, asn_i) - targets = [] - if tn == "web_connectivity": - for d in webconn_test_items: - tgt = { - "attributes": { - "category_code": d["category_code"], - "country_code": d["country_code"], - }, - "input": d["url"], - "options": {}, - } - targets.append(tgt) - elif tn == "dnscheck": - for url in dnscheck_inputs: - tgt = {"attributes": {}, "input": url, "options": {}} - targets.append(tgt) - elif tn == "stunreachability": - for url in stunreachability_inputs: - tgt = {"attributes": {}, "input": url, "options": {}} - targets.append(tgt) - - block = {"test_name": tn, "report_id": rid, "targets": targets} - resp["nettests"].append(block) - - return nocachejson(**resp) From 3154b86ebeed0187ccd8ba641a6da33006be8c1f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 24 Apr 2026 10:47:03 +0200 Subject: [PATCH 049/105] add old api tests for private api --- .../ooniprobe/tests/integ/test_private_api.py | 434 ++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 ooniapi/services/ooniprobe/tests/integ/test_private_api.py diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py new file mode 100644 index 000000000..af7486ee5 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -0,0 +1,434 @@ +# +# Most of the private API runs statistics on the last N days. As such, tests +# are not deterministic. +# + +import pytest + +from tests.utils import mock_load_json + + +def privapi(client, subpath): + response = client.get(f"/api/_/{subpath}") + assert response.status_code == 200 + assert response.is_json + return response.json + + +# TODO: improve tests + + +def test_private_api_asn_by_month(client): + url = "asn_by_month" + response = privapi(client, url) + assert len(response) > 0, response + r = response[0] + assert sorted(r.keys()) == ["date", "value"] + assert r["value"] > 10 + assert r["value"] < 10**6 + assert r["date"].endswith("T00:00:00+00:00") + + +def test_private_api_countries_by_month(client): + url = "countries_by_month" + response = privapi(client, url) + assert len(response) > 0, response + r = response[0] + assert sorted(r.keys()) == ["date", "value"] + assert r["value"] > 1 + assert r["value"] < 1000 + assert r["date"].endswith("T00:00:00+00:00") + + +def test_private_api_test_names(client, log): + url = "test_names" + response = privapi(client, url) + assert response == { + "test_names": [ + {"id": "bridge_reachability", "name": "Bridge Reachability"}, + {"id": "dash", "name": "DASH"}, + {"id": "dns_consistency", "name": "DNS Consistency"}, + {"id": "dnscheck", "name": "DNS Check"}, + {"id": "facebook_messenger", "name": "Facebook Messenger"}, + { + "id": "http_header_field_manipulation", + "name": "HTTP Header Field Manipulation", + }, + {"id": "http_host", "name": "HTTP Host"}, + {"id": "http_invalid_request_line", "name": "HTTP Invalid Request Line"}, + {"id": "http_requests", "name": "HTTP Requests"}, + {"id": "meek_fronted_requests_test", "name": "Meek Fronted Requests"}, + {"id": "multi_protocol_traceroute", "name": "Multi Protocol Traceroute"}, + {"id": "ndt", "name": "NDT"}, + {"id": "psiphon", "name": "Psiphon"}, + {"id": "riseupvpn", "name": "RiseupVPN"}, + {"id": "signal", "name": "Signal"}, + {"id": "stunreachability", "name": "STUN Reachability"}, + {"id": "tcp_connect", "name": "TCP Connect"}, + {"id": "telegram", "name": "Telegram"}, + {"id": "tor", "name": "Tor"}, + {"id": "torsf", "name": "Tor Snowflake"}, + {"id": "urlgetter", "name": "URL Getter"}, + {"id": "vanilla_tor", "name": "Vanilla Tor"}, + {"id": "web_connectivity", "name": "Web Connectivity"}, + {"id": "whatsapp", "name": "WhatsApp"}, + ] + } + + +def test_private_api_countries_total(client, log): + url = "countries" + response = privapi(client, url) + assert "countries" in response + assert len(response["countries"]) >= 20 + for a in response["countries"]: + if a["alpha_2"] == "CA": + assert a["count"] > 100 + assert a["name"] == "Canada" + return + + assert 0, "CA not found" + + +def test_private_api_test_coverage(client, log): + url = "test_coverage?probe_cc=BR" + resp = privapi(client, url) + assert 190 < len(resp["test_coverage"]) < 220 + assert 27 < len(resp["network_coverage"]) < 32 + assert sorted(resp["test_coverage"][0]) == ["count", "test_day", "test_group"] + assert sorted(resp["network_coverage"][0]) == ["count", "test_day"] + + +def test_private_api_test_coverage_with_groups(client, log): + url = "test_coverage?probe_cc=BR&test_groups=websites" + resp = privapi(client, url) + assert len(resp["test_coverage"]) > 10 + assert sorted(resp["test_coverage"][0]) == ["count", "test_day", "test_group"] + assert 27 < len(resp["network_coverage"]) < 32 + + +def test_private_api_domain_metadata1(client): + url = "domain_metadata?domain=facebook.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "www.facebook.com" + + +def test_private_api_domain_metadata2(client): + url = "domain_metadata?domain=www.facebook.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "www.facebook.com" + + +def test_private_api_domain_metadata3(client): + url = "domain_metadata?domain=www.twitter.com" + resp = privapi(client, url) + assert resp["category_code"] == "GRP" + assert resp["canonical_domain"] == "twitter.com" + + +def test_private_api_domain_metadata4(client): + url = "domain_metadata?domain=www.this-domain-is-not-in-the-test-lists-for-sure.com" + resp = privapi(client, url) + assert resp["category_code"] == "MISC" + exp = "this-domain-is-not-in-the-test-lists-for-sure.com" + assert resp["canonical_domain"] == exp + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_networks(client, log): + url = "website_networks?probe_cc=US" + resp = privapi(client, url) + assert len(resp["results"]) > 100 + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_stats(client, log): + url = "website_stats?probe_cc=DE&probe_asn=3320&input=http:%2F%2Fwww.backtrack-linux.org%2F" + resp = privapi(client, url) + assert len(resp["results"]) > 2 + assert sorted(resp["results"][0].keys()) == [ + "anomaly_count", + "confirmed_count", + "failure_count", + "test_day", + "total_count", + ] + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_website_urls(client, log): + url = "website_urls?probe_cc=US&probe_asn=209" + response = privapi(client, url) + r = response["metadata"] + assert r["total_count"] > 0 + del r["total_count"] + assert r == { + "current_page": 1, + "limit": 10, + "next_url": "https://api.ooni.io/api/_/website_urls?limit=10&offset=10&probe_asn=209&probe_cc=US", + "offset": 0, + } + assert len(response["results"]) == 10 + + +def test_private_api_vanilla_tor_stats(client): + url = "vanilla_tor_stats?probe_cc=BR" + resp = privapi(client, url) + assert "notok_networks" in resp + return # FIXME: implement tests with mocked db + assert resp["notok_networks"] >= 0 + assert len(resp["networks"]) > 10 + assert sorted(resp["networks"][0].keys()) == [ + "failure_count", + "last_tested", + "probe_asn", + "success_count", + "test_runtime_avg", + "test_runtime_max", + "test_runtime_min", + "total_count", + ] + assert resp["last_tested"].startswith("20") + + +def test_private_api_vanilla_tor_stats_empty(client): + url = "vanilla_tor_stats?probe_cc=XY" + resp = privapi(client, url) + assert resp["notok_networks"] == 0 + assert len(resp["networks"]) == 0 + assert resp["last_tested"] is None + + +def test_private_api_im_networks(client): + url = "im_networks?probe_cc=BR" + resp = privapi(client, url) + return # FIXME: implement tests with mocked db + assert len(resp) > 1 + assert len(resp["facebook_messenger"]["ok_networks"]) > 5 + if "telegram" in resp: + assert len(resp["telegram"]["ok_networks"]) > 5 + assert len(resp["signal"]["ok_networks"]) > 5 + # assert len(resp["whatsapp"]["ok_networks"]) > 5 + + +def test_private_api_im_stats_basic(client): + url = "im_stats?probe_cc=CH&probe_asn=3303&test_name=facebook_messenger" + resp = privapi(client, url) + assert 20 < len(resp["results"]) < 34 + assert resp["results"][0]["total_count"] > -1 + assert resp["results"][0]["anomaly_count"] is None + assert len(resp["results"][0]["test_day"]) == 25 + + +@pytest.mark.skip("FIXME not deterministic") +def test_private_api_im_stats(client): + url = "im_stats?probe_cc=CH&probe_asn=3303&test_name=facebook_messenger" + resp = privapi(client, url) + assert len(resp["results"]) > 10 + assert resp["results"][0]["total_count"] > -1 + assert resp["results"][0]["anomaly_count"] is None + assert len(resp["results"][0]["test_day"]) == 25 + assert sum(e["total_count"] for e in resp["results"]) > 0, resp + + +def test_private_api_network_stats(client): + # TODO: the stats are not implemented + url = "network_stats?probe_cc=GB" + response = privapi(client, url) + assert response == { + "metadata": { + "current_page": 1, + "limit": 10, + "next_url": None, + "offset": 0, + "total_count": 0, + }, + "results": [], + } + + +def test_private_api_country_overview(client): + url = "country_overview?probe_cc=BR" + resp = privapi(client, url) + assert resp["first_bucket_date"].startswith("20"), resp + assert resp["measurement_count"] > 1000 + assert resp["network_count"] > 10 + + +def test_private_api_global_overview(client): + url = "global_overview" + response = privapi(client, url) + assert "country_count" in response + assert "measurement_count" in response + assert "network_count" in response + + +def test_private_api_global_overview_by_month(client): + url = "global_overview_by_month" + resp = privapi(client, url) + assert sorted(resp["networks_by_month"][0].keys()) == ["date", "value"] + assert sorted(resp["countries_by_month"][0].keys()) == ["date", "value"] + assert sorted(resp["measurements_by_month"][0].keys()) == ["date", "value"] + assert resp["networks_by_month"][0]["date"].endswith("T00:00:00+00:00") + + +@pytest.mark.skip(reason="cannot be tested") +def test_private_api_quotas_summary(client): + resp = privapi(client, "quotas_summary") + + +def test_private_api_check_report_id(client, log): + rid = "20210709T004340Z_webconnectivity_MY_4818_n1_YCM7J9mGcEHds2K3" + url = f"check_report_id?report_id={rid}" + response = privapi(client, url) + assert response == {"v": 0, "found": True} + + +def test_private_api_check_bogus_report_id_is_found(client, log): + # The API always returns True + url = f"check_report_id?report_id=BOGUS_REPORT_ID" + response = privapi(client, url) + assert response == {"v": 0, "found": True} + + +# # /circumvention_stats_by_country + + +@pytest.mark.skip(reason="depends on fresh data") +def test_private_api_circumvention_stats_by_country(client, log): + url = "circumvention_stats_by_country" + resp = privapi(client, url) + assert resp["v"] == 0 + assert len(resp["results"]) > 3 + + +# # /circumvention_runtime_stats + + +@pytest.mark.skip(reason="depends on fresh data") +def test_private_api_circumvention_runtime_stats(client, log): + url = "circumvention_runtime_stats" + resp = privapi(client, url) + assert resp["v"] == 0 + assert "error" not in resp, resp + assert len(resp["results"]) > 3, resp + + +# # /asnmeta + + +def test_private_api_ansmeta(client, log): + resp = privapi(client, "asnmeta?asn=0") + assert resp == {"org_name": "Unknown"} + + +# # /networks + + +def test_private_api_networks(client, log): + resp = privapi(client, "networks") + assert resp["v"] == 0 + assert len(resp["results"]) > 50 + assert resp["results"][0] == {"cnt": 380, "org_name": "", "probe_asn": 58224} + + +# # /domains + + +def test_private_api_domains(client, log): + resp = privapi(client, "domains") + assert resp["v"] == 0 + rows = resp["results"] + assert len(rows) > 5 + assert sorted(rows[0]) == ["category_code", "domain_name", "measurement_count"] + d = {r["domain_name"]: r["category_code"] for r in rows} + assert d["facebook.com"] == "GRP" + assert d["ncac.org"] == "NEWS" + assert d["twitter.com"] == "GRP" + + +# # /check-in + +from mock import patch + + +def postj(client, url, **kw): + response = client.post(url, json=kw) + assert response.status_code == 200 + assert response.is_json + return response.json + + +def test_private_check_in_basic(client): + j = dict( + probe_cc="US", + probe_asn="AS1234", + on_wifi=True, + charging=False, + ) + with patch("ooniapi.probe_services._load_json", new_callable=mock_load_json): + c = postj(client, "/api/_/check-in", **j) + + assert c["v"] == 2 + assert c["nettests"] + for t in c["nettests"]: + assert sorted(t) == ["report_id", "targets", "test_name"] + + for t in c["nettests"]: + if t["test_name"] == "dnscheck": + assert t["targets"] + assert t["targets"][0] == { + "attributes": {}, + "input": "https://dns.google/dns-query", + "options": {}, + } + break + + for t in c["nettests"]: + if t["test_name"] == "stunreachability": + assert t["targets"] + assert t["targets"][0] == { + "attributes": {}, + "input": "stun://stun.voip.blackberry.com:3478", + "options": {}, + } + break + + exp = [ + "bridge_reachability", + "dash", + "dns_consistency", + "dnscheck", + "dnsping", + "echcheck", + "facebook_messenger", + "http_header_field_manipulation", + "http_host", + "http_invalid_request_line", + "http_requests", + "meek_fronted_requests_test", + "multi_protocol_traceroute", + "ndt", + "portfiltering", + "psiphon", + "quicping", + "riseupvpn", + "signal", + "simplequicping", + "sniblocking", + "stunreachability", + "tcp_connect", + "tcpping", + "telegram", + "tlsmiddlebox", + "tlsping", + "tor", + "torsf", + "urlgetter", + "vanilla_tor", + "web_connectivity", + "whatsapp", + ] + assert sorted([t["test_name"] for t in c["nettests"]]) == exp From 0b52a712948f143d90382154acdca8b35d502d4f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:21:47 +0200 Subject: [PATCH 050/105] add ClickhouseDep; update query_click usage --- .../ooniprobe/src/ooniprobe/routers/private.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ff986d1b3..fb6a7f652 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -22,7 +22,7 @@ from .v1.probe_services import probe_geoip, generate_test_helpers_conf from ..common.clickhouse_utils import query_click, query_click_one_row -from ..common.dependencies import role_required +from ..common.dependencies import role_required, ClickhouseDep from ..common.prio import generate_test_list from ..common.routers import BaseModel from ..metrics import Metrics @@ -103,7 +103,9 @@ class ASNCount(BaseModel): @router.get("/asn_by_month", tags=["private_api"], response_model=ASNByMonthResponse) -def api_private_asn_by_month() -> ASNByMonthResponse: +def api_private_asn_by_month( + clickhouse: ClickhouseDep, +) -> ASNByMonthResponse: """Network count by month --- responses: @@ -118,7 +120,7 @@ def api_private_asn_by_month() -> ASNByMonthResponse: AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) GROUP BY date ORDER BY date """ - li = list(query_click(q, {})) + li = list(query_click(clickhouse, q, {})) expand_dates(li) validated: ASNByMonthResponse = [ASNCount(**item) for item in li] return validated @@ -1220,7 +1222,9 @@ class DomainsMeasuredResponse(BaseModel): @router.get("/domains", response_model=DomainsMeasuredResponse, tags=["private"]) -def api_private_domains() -> DomainsMeasuredResponse: +def api_private_domains( + clickhouse: ClickhouseDep, +) -> DomainsMeasuredResponse: """List all the domains in the test-lists with their measurement count --- responses: @@ -1251,7 +1255,7 @@ def api_private_domains() -> DomainsMeasuredResponse: ORDER BY domain_name """ try: - results = query_click(sql.text(q), {}) + results = query_click(clickhouse, sql.text(q), {}) return DomainsMeasuredResponse(v=0, results=results) except Exception as e: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) From a1a5478e17d54c2b2acb02ab9c1ae320ba2b89e5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:22:28 +0200 Subject: [PATCH 051/105] add log fixture --- ooniapi/services/ooniprobe/tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ooniapi/services/ooniprobe/tests/conftest.py b/ooniapi/services/ooniprobe/tests/conftest.py index 44907c7dc..72a7db599 100644 --- a/ooniapi/services/ooniprobe/tests/conftest.py +++ b/ooniapi/services/ooniprobe/tests/conftest.py @@ -1,4 +1,5 @@ import json +import logging import pathlib import time from datetime import datetime @@ -35,6 +36,10 @@ from .utils import setup_user +@pytest.fixture +def log(): + return logging.getLogger(__name__) + def make_override_get_settings(**kw): def override_get_settings(): return Settings(**kw) From 75400aff1a757e837b39b73f8731c88e853d4fbb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:22:56 +0200 Subject: [PATCH 052/105] remove unused mock_load_json --- ooniapi/services/ooniprobe/tests/integ/test_private_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py index af7486ee5..458504065 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -5,8 +5,6 @@ import pytest -from tests.utils import mock_load_json - def privapi(client, subpath): response = client.get(f"/api/_/{subpath}") From b146341bfaa53f9d1548ba2f66bd650e9bbb27cb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:23:09 +0200 Subject: [PATCH 053/105] fix testclient api usage (httpx) --- ooniapi/services/ooniprobe/tests/integ/test_private_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py index 458504065..ab2a30819 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -9,7 +9,7 @@ def privapi(client, subpath): response = client.get(f"/api/_/{subpath}") assert response.status_code == 200 - assert response.is_json + assert response.json return response.json From 0cd634351533cddf623282d4d8ef0790e65e75a2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:24:03 +0200 Subject: [PATCH 054/105] remove test for private check-in API: unported --- .../ooniprobe/tests/integ/test_private_api.py | 85 ------------------- 1 file changed, 85 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py index ab2a30819..247be1297 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -345,88 +345,3 @@ def test_private_api_domains(client, log): assert d["facebook.com"] == "GRP" assert d["ncac.org"] == "NEWS" assert d["twitter.com"] == "GRP" - - -# # /check-in - -from mock import patch - - -def postj(client, url, **kw): - response = client.post(url, json=kw) - assert response.status_code == 200 - assert response.is_json - return response.json - - -def test_private_check_in_basic(client): - j = dict( - probe_cc="US", - probe_asn="AS1234", - on_wifi=True, - charging=False, - ) - with patch("ooniapi.probe_services._load_json", new_callable=mock_load_json): - c = postj(client, "/api/_/check-in", **j) - - assert c["v"] == 2 - assert c["nettests"] - for t in c["nettests"]: - assert sorted(t) == ["report_id", "targets", "test_name"] - - for t in c["nettests"]: - if t["test_name"] == "dnscheck": - assert t["targets"] - assert t["targets"][0] == { - "attributes": {}, - "input": "https://dns.google/dns-query", - "options": {}, - } - break - - for t in c["nettests"]: - if t["test_name"] == "stunreachability": - assert t["targets"] - assert t["targets"][0] == { - "attributes": {}, - "input": "stun://stun.voip.blackberry.com:3478", - "options": {}, - } - break - - exp = [ - "bridge_reachability", - "dash", - "dns_consistency", - "dnscheck", - "dnsping", - "echcheck", - "facebook_messenger", - "http_header_field_manipulation", - "http_host", - "http_invalid_request_line", - "http_requests", - "meek_fronted_requests_test", - "multi_protocol_traceroute", - "ndt", - "portfiltering", - "psiphon", - "quicping", - "riseupvpn", - "signal", - "simplequicping", - "sniblocking", - "stunreachability", - "tcp_connect", - "tcpping", - "telegram", - "tlsmiddlebox", - "tlsping", - "tor", - "torsf", - "urlgetter", - "vanilla_tor", - "web_connectivity", - "whatsapp", - ] - assert sorted([t["test_name"] for t in c["nettests"]]) == exp From 6585852bfc1e9547250c33a04952155c140bd88d Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Mon, 27 Apr 2026 11:40:45 +0200 Subject: [PATCH 055/105] remove unused validation methods --- .../ooniprobe/src/ooniprobe/routers/private.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index fb6a7f652..312ce5f41 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -73,21 +73,6 @@ def daterange(start_date, end_date): yield start_date + timedelta(n) -def validate_probe_cc_query_param(): - probe_cc = request.args.get("probe_cc") - if probe_cc is None or len(probe_cc) != 2: - raise BadRequest("missing or incorrect probe_cc") - return probe_cc - - -def validate_probe_asn_query_param(): - probe_asn = request.args.get("probe_asn") - if probe_asn is None: - # TODO: validate and return int? - raise BadRequest("missing or incorrect probe_asn") - return probe_asn - - def expand_dates(li): """Replaces 'date' key in a list of dict""" for i in li: From 9ac169333b1fe2ab308014aa10d96d246c1966f2 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 08:36:13 +0000 Subject: [PATCH 056/105] update private_api client wrapper; TestClient uses httpx --- ooniapi/services/ooniprobe/tests/integ/test_private_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py index 247be1297..8fae5ab30 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -9,8 +9,7 @@ def privapi(client, subpath): response = client.get(f"/api/_/{subpath}") assert response.status_code == 200 - assert response.json - return response.json + return response.json() # TODO: improve tests From 619ec4930919052f2c0fd2f7b2e1b3de978a6f5a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 12:18:09 +0200 Subject: [PATCH 057/105] update test fixture fastpath table definition --- .../services/ooniprobe/tests/fixtures/initdb/01-scheme.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql b/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql index 25e6af123..d3ca78a36 100644 --- a/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql +++ b/ooniapi/services/ooniprobe/tests/fixtures/initdb/01-scheme.sql @@ -31,14 +31,15 @@ CREATE TABLE default.fastpath `server_as_name` String, `update_time` DateTime64(3) MATERIALIZED now64(), `test_version` String, - `test_runtime` Float32, `architecture` String, `engine_name` String, `engine_version` String, + `test_runtime` Float32, `blocking_type` String, `test_helper_address` LowCardinality(String), `test_helper_type` LowCardinality(String), - `ooni_run_link_id` Nullable(UInt64) + `ooni_run_link_id` Nullable(UInt64), + `is_verified` LowCardinality(String) DEFAULT 'u' ) ENGINE = ReplacingMergeTree ORDER BY (measurement_start_time, report_id, input) From fb76dd08e7dcb6a0a86a2b13bf4d8c0e0bf365d0 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:26:24 +0200 Subject: [PATCH 058/105] fix date format --- .../services/ooniprobe/src/ooniprobe/routers/private.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 312ce5f41..a04a5201c 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -5,7 +5,7 @@ not rely on these as they are likely to change, break in unexpected ways. Also there is no versioning on them. """ -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, timezone from itertools import product from urllib.parse import urljoin, urlencode @@ -82,6 +82,10 @@ def expand_dates(li): class ASNCount(BaseModel): date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") value: int = Field(..., description="Count of unique ASN seen.") + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + ASNByMonthResponse = List[ASNCount] @@ -115,6 +119,9 @@ class CountryCount(BaseModel): date: datetime = Field(..., description="Timestamp for the measurement (ISO 8601).") value: int = Field(..., description="Count of unique countries seen.") + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } CountryByMonthResponse = List[ASNCount] From febb1a3a81b6a59983d1985aee8965f15336e63b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:28:56 +0200 Subject: [PATCH 059/105] fix response type --- .../src/ooniprobe/routers/private.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a04a5201c..41b78bf61 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -123,11 +123,12 @@ class CountryCount(BaseModel): "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } } -CountryByMonthResponse = List[ASNCount] -@router.get("/countries_by_month", tags=["private_api"], response_model=CountryByMonthResponse) -def api_private_countries_by_month() -> Response: +@router.get("/countries_by_month", tags=["private_api"], response_model=List[CountryCount]) +def api_private_countries_by_month( + clickhouse: ClickhouseDep, +) -> List[CountryCount]: """Countries count by month --- responses: @@ -144,16 +145,20 @@ def api_private_countries_by_month() -> Response: """ li = list(query_click(q, {})) expand_dates(li) - validated: CountryByMonthResponse = [CountryCount(**item) for item in li] - return validated + return [CountryCount(**item) for item in li] + class TestName(BaseModel): id: str = Field(..., description="test id") name: str = Field(..., description="test name") -@router.get("/test_names", tags=["private_api"], response_model=List[TestName]) -def api_private_test_names() -> List[TestName]: +class TestNameResponse(BaseModel): + test_names: List[TestName] + + +@router.get("/test_names", tags=["private_api"], response_model=TestNameResponse) +def api_private_test_names() -> TestNameResponse: """Provides test names and descriptions to Explorer --- responses: @@ -187,8 +192,7 @@ def api_private_test_names() -> List[TestName]: "web_connectivity": "Web Connectivity", "whatsapp": "WhatsApp", } - validated: List[TestNames] = [TestName(id=k, name=v) for k, v in TEST_NAMES.items()] - return validated + return TestNameResponse(test_names=[TestName(id=k, name=v) for k, v in TEST_NAMES.items()]) class CountryStat(BaseModel): @@ -197,8 +201,14 @@ class CountryStat(BaseModel): name: str = Field(..., description="Country Name") -@router.get("/countries", tags=["private_api"], response_model=List[CountryStat]) -def api_private_countries() -> List[CountryStat]: +class CountryStatResponse(BaseModel): + countries: List[CountryStat] = Field(..., description="List of countries") + + +@router.get("/countries", tags=["private_api"], response_model=CountryStatResponse) +def api_private_countries( + clickhouse: ClickhouseDep, +) -> CountryStatResponse: """Summary of countries --- responses: @@ -221,8 +231,7 @@ def api_private_countries() -> List[CountryStat]: except KeyError: pass - validated: List[CountryStat] = c - return validated + return CountryStatResponse(countries=c) @router.get( From ba999b24c0d1b071a48636766453bcff1031cccb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:30:11 +0200 Subject: [PATCH 060/105] pass clickhouse to query_click --- .../src/ooniprobe/routers/private.py | 82 ++++++++++++------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 41b78bf61..a1d6bf578 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -143,7 +143,7 @@ def api_private_countries_by_month( AND measurement_start_time > toStartOfMonth(subtractMonths(now(), 24)) GROUP BY date ORDER BY date """ - li = list(query_click(q, {})) + li = list(query_click(clickhouse, q, {})) expand_dates(li) return [CountryCount(**item) for item in li] @@ -221,7 +221,7 @@ def api_private_countries( GROUP BY probe_cc ORDER BY probe_cc """ c = [] - rows = query_click(q, {}) + rows = query_click(clickhouse, q, {}) for r in rows: try: name = lookup_country(r["probe_cc"]) @@ -318,12 +318,12 @@ def pivot_test_coverage(rows, test_group_names, days): return test_coverage -def get_recent_test_coverage_ch(probe_cc): +def get_recent_test_coverage_ch(clickhouse, probe_cc): """Returns [{"count": 4888, "test_day": "2021-10-16", "test_group": "websites"}, ... ] """ q = "SELECT DISTINCT(test_group) FROM test_groups ORDER BY test_group" - rows = query_click(sql.text(q), {}) + rows = query_click(clickhouse, sql.text(q), {}) test_group_names = [r["test_group"] for r in rows] q = """SELECT @@ -337,13 +337,13 @@ def get_recent_test_coverage_ch(probe_cc): AND probe_cc = :probe_cc GROUP BY measurement_start_day, test_group """ - rows = query_click(sql.text(q), dict(probe_cc=probe_cc)) + rows = query_click(clickhouse, sql.text(q), dict(probe_cc=probe_cc)) rows = tuple(rows) l30d = tuple(last_30days(32, 2)) return pivot_test_coverage(rows, test_group_names, l30d) -def get_recent_network_coverage_ch(probe_cc, test_groups): +def get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups): """Count ASNs with at least one measurements, grouped by day, for a given CC, and filtered by test groups Return [{"count": 58, "test_day": "2021-10-16" }, ... ]""" @@ -374,7 +374,7 @@ def get_recent_network_coverage_ch(probe_cc, test_groups): s = s.replace("--mark--", "") d = {"probe_cc": probe_cc} - return query_click(sql.text(s), d) + return query_click(clickhouse, sql.text(s), d) class TestCoverageResponse(BaseModel): @@ -384,6 +384,7 @@ class TestCoverageResponse(BaseModel): @router.get("/test_coverage", response_model=TestCoverageResponse, tags=["private"]) def api_private_test_coverage( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), test_groups: str = Query(None, description="XXX: What is this?") ) -> TestCoverageResponse: @@ -404,8 +405,8 @@ def api_private_test_coverage( if test_groups is not None: test_groups = test_groups.split(",") - tc = get_recent_test_coverage_ch(probe_cc) - nc = get_recent_network_coverage_ch(probe_cc, test_groups) + tc = get_recent_test_coverage_ch(clickhouse, probe_cc) + nc = get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups) validated = TestCoverageResponse(network_coverage=nc, test_coverage=tc) return validated @@ -417,8 +418,9 @@ class MeasurementsByASN(BaseModel): @router.get("/website_networks", response_model=List[MeasurementsByASN], tags=["private"]) def api_private_website_network_tests( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") - ) -> List[MeasurementsByASN]: +) -> List[MeasurementsByASN]: """TODO --- parameters: @@ -441,7 +443,7 @@ def api_private_website_network_tests( GROUP BY probe_asn ORDER BY count DESC """ - results = query_click(sql.text(s), {"probe_cc": probe_cc}) + results = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) validated: List[MeasurementsByASN] = [MeasurementsByASN(**x) for x in results] return validated @@ -457,6 +459,7 @@ class WebsiteStatsResponse(BaseModel): @router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) def api_private_website_stats( + clickhouse: ClickhouseDep, input: AnyUrl = Query(..., description="Website to query stats"), probe_cc: CountryAlpha2 = Query(..., description="Country Code"), probe_asn: int = Query(..., description="ASN (integer)"), @@ -487,7 +490,7 @@ def api_private_website_stats( GROUP BY test_day ORDER BY test_day """ d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} - results = query_click(sql.text(s), d) + results = query_click(clickhouse, sql.text(s), d) return WebsiteStatsResponse(result=results) @@ -512,6 +515,7 @@ class WebsiteURLsResponse(BaseModel): @router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) def api_private_website_test_urls( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), probe_asn: str = Query(..., description="ASN, e.g. AS1234"), limit: int = Query(10, description="Limit results"), @@ -539,7 +543,7 @@ def api_private_website_test_urls( AND probe_cc = :probe_cc AND probe_asn = :probe_asn """ - q = query_click_one_row(sql.text(s), dict(probe_cc=probe_cc, probe_asn=probe_asn)) + q = query_click_one_row(clickhouse, sql.text(s), dict(probe_cc=probe_cc, probe_asn=probe_asn)) total_count = q["input_count"] if q else 0 # Group msmts by CC / ASN / period with LIMIT and OFFSET @@ -566,7 +570,7 @@ def api_private_website_test_urls( "limit": limit, "offset": offset, } - results = query_click(sql.text(s), d) + results = query_click(clickhouse, sql.text(s), d) current_page = math.ceil(offset / limit) + 1 metadata = { "offset": offset, @@ -614,6 +618,7 @@ class TorStatsResponse(BaseModel): @router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) def api_private_vanilla_tor_stats( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") ) -> TorStatsResponse: """Tor statistics over ASN for a given CC @@ -641,7 +646,7 @@ def api_private_vanilla_tor_stats( AND probe_cc = :probe_cc GROUP BY probe_asn """ - q = query_click(sql.text(s), {"probe_cc": probe_cc}) + q = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) extras = { "test_runtime_avg": None, "test_runtime_max": None, @@ -683,6 +688,7 @@ class IMNetworkStats(BaseModel): @router.get("/im_networks", response_model=Dict[str, IMNetworkStats], tags=["private"]) def api_private_im_networks( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") ) -> Dict[str, IMNetworkStats]: """Instant messaging networks statistics @@ -706,7 +712,7 @@ def api_private_im_networks( ORDER BY test_name ASC, total_count DESC """ test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] - q = query_click(sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) + q = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) results = IMNetworksResponse() for r in q: # get stats for test_name or create a new IMNetworksStats @@ -745,6 +751,7 @@ class IMStatsResponse(BaseModel): @router.get("/im_stats", response_model=IMStatsResponse, tags=["private"]) def api_private_im_stats( + clickhouse: ClickhouseDep, probe_asn: str = Query(..., description="ASN, e.g. AS1234"), probe_cc: CountryAlpha2 = Query(..., description="Country Code"), test_name: str = Query(..., description="Test name") @@ -778,7 +785,7 @@ def api_private_im_stats( "probe_asn": probe_asn, "test_name": test_name, } - q = query_click(sql.text(s), query_params) + q = query_click(clickhouse, sql.text(s), query_params) tmp = {r["test_day"]: r for r in q} items: List[IMStatsItem] = [] days = [date.today() + timedelta(days=(d - 31)) for d in range(32)] @@ -845,6 +852,7 @@ class CountryOverviewResponse(BaseModel): @router.get("/country_overview", response_model=CountryOverviewResponse, tags=["private"]) def api_private_country_overview( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), ) -> CountryOverviewResponse: """Country-specific overview @@ -866,7 +874,7 @@ def api_private_country_overview( WHERE probe_cc = :probe_cc AND measurement_start_time > '2012-12-01' """ - r = query_click_one_row(sql.text(s), {"probe_cc": probe_cc}) + r = query_click_one_row(clickhouse, sql.text(s), {"probe_cc": probe_cc}) assert r result = CountryOverviewResponse( first_bucket_date=r["first_bucket_date"], @@ -882,7 +890,9 @@ class GlobalOverviewResponse(BaseModel): @router.get("/global_overview", response_model=GlobalOverviewResponse, tags=["private"]) -def api_private_global_overview() -> GlobalOverviewResponse: +def api_private_global_overview( + clickhouse: ClickhouseDep, +) -> GlobalOverviewResponse: """Provide global summary of measurements Sources: global_stats db table --- @@ -896,7 +906,7 @@ def api_private_global_overview() -> GlobalOverviewResponse: COUNT(*) AS measurement_count FROM fastpath """ - r = query_click_one_row(q, {}) + r = query_click_one_row(clickhouse, q, {}) assert r result = GlobalOverViewResponse( network_count=r["network_count"], @@ -916,7 +926,9 @@ class GlobalOverviewMonthResponse(BaseModel): @router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) -def api_private_global_by_month() -> Response: +def api_private_global_by_month( + clickhouse: ClickhouseDep, +) -> GlobalOverviewMonthResponse: """Provide global summary of measurements Sources: global_by_month db table --- @@ -934,7 +946,7 @@ def api_private_global_by_month() -> Response: AND measurement_start_time < toStartOfMonth(today() + interval 1 month) GROUP BY month ORDER BY month """ - rows = query_click(sql.text(q), {}) + rows = query_click(clickhouse, sql.text(q), {}) rows = list(rows) n = [{"date": r["month"], "value": r["networks_by_month"]} for r in rows] @@ -958,7 +970,9 @@ class CircumventionStatsResponse(BaseModel): @router.get("/circumvention_stats_by_country", response_model=CircumventionStatsResponse, tags=["private"]) -def api_private_circumvention_stats_by_country() -> Response: +def api_private_circumvention_stats_by_country( + clickhouse: ClickhouseDep, +) -> CircumventionStatsResponse: """Aggregated statistics on protocols used for circumvention, grouped by country. --- @@ -974,7 +988,7 @@ def api_private_circumvention_stats_by_country() -> Response: GROUP BY probe_cc ORDER BY probe_cc """ try: - result = query_click(sql.text(q), {}) + result = query_click(clickhouse, sql.text(q), {}) return CountryCircumVentionStatsResponse(results=result) except Exception as e: @@ -1018,7 +1032,9 @@ class CircumventionRuntimeStatsResponse(BaseModel): @router.get("/circumvention_runtime_stats", response_model=CircumventionRuntimeStatsResponse, tags=["private"]) -def api_private_circumvention_runtime_stats() -> Response: +def api_private_circumvention_runtime_stats( + clickhouse: ClickhouseDep, +) -> CircumventionRuntimeStatsResponse: """Runtime statistics on protocols used for circumvention, grouped by date, country, test_name. --- @@ -1041,7 +1057,7 @@ def api_private_circumvention_runtime_stats() -> Response: GROUP BY date, probe_cc, test_name """ try: - r = query_click(sql.text(q), {}) + r = query_click(clickhouse, sql.text(q), {}) return CircumventionRuntimeStatsResponse(results=pivot_circumvention_runtime_stats(r), v=0) except Exception as e: @@ -1064,6 +1080,7 @@ class DomainMetadataResponse(BaseModel): @router.get("/domain_metadata", response_model=DomainMetadataResponse, tags=["private"]) def api_private_domain_metadata( + clickhouse: ClickhouseDep, domain: DomainStr = Query(..., description="Domain Name"), ) -> DomainMetadataResponse: """Return the primary category code of a certain domain_name and its @@ -1109,7 +1126,7 @@ def api_private_domain_metadata( ORDER BY length(url) LIMIT 1 """ - res = query_click_one_row(sql.text(q), dict(domains=domains)) + res = query_click_one_row(clickhouse, sql.text(q), dict(domains=domains)) if not res: # case 2: domain only inside a country list, so we just select the # shortest domain among the one with and without www @@ -1119,7 +1136,7 @@ def api_private_domain_metadata( ORDER BY length(url) LIMIT 1 """ - res = query_click_one_row(sql.text(q), dict(domains=domains)) + res = query_click_one_row(clickhouse, sql.text(q), dict(domains=domains)) if res: category_code = res["category_code"] @@ -1134,6 +1151,7 @@ class ASNMetadataResponse(BaseModel): @router.get("/asnmeta", response_model=ASNMetadataResponse, tags=["private"]) def api_private_asnmeta( + clickhouse: ClickhouseDep, asn: int = Query(..., description="Autonomous System Number, e.g. 1234"), ) -> ASNMetadataResponse: """Look up organization name by ASN @@ -1158,7 +1176,7 @@ def api_private_asnmeta( ORDER BY changed DESC LIMIT 1 """ - res = query_click_one_row(sql.text(q), dict(asn=asn)) + res = query_click_one_row(clickhouse, sql.text(q), dict(asn=asn)) org_name = res["org_name"] if res else "Unknown" return ASNMetadataResponse(org_name=org_name) @@ -1175,7 +1193,9 @@ class MeasuredNetworksResponse(BaseModel): @router.get("/networks", response_model=MeasuredNetworksResponse, tags=["private"]) -def api_private_networks() -> Response: +def api_private_networks( + clickhouse: ClickhouseDep, +) -> MeasuredNetworksResponse: """List all networks that have measurements --- responses: @@ -1205,7 +1225,7 @@ def api_private_networks() -> Response: ON (asorgs.asn = cnts.probe_asn) """ try: - results = query_click(sql.text(q), {}) + results = query_click(clickhouse, sql.text(q), {}) return MeasuredNetworksResponse(results=results, v=0) except Exception as e: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) From 9d7516f7fe2502db88e5d45065101dcdd61e8fcb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:30:31 +0200 Subject: [PATCH 061/105] remove old request.args --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a1d6bf578..d148abf0e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1106,9 +1106,6 @@ def api_private_domain_metadata( } """ category_code = "MISC" - domain = request.args.get("domain") - if domain is None: - raise BadRequest("missing domain") if domain.startswith("www."): canonical_domain = domain[4:] From 172d8338a35663c15da7d527303f146100094975 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:53:06 +0200 Subject: [PATCH 062/105] fix response type --- .../ooniprobe/src/ooniprobe/routers/private.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index d148abf0e..1d1ce1061 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -87,14 +87,10 @@ class ASNCount(BaseModel): } - -ASNByMonthResponse = List[ASNCount] - - -@router.get("/asn_by_month", tags=["private_api"], response_model=ASNByMonthResponse) +@router.get("/asn_by_month", tags=["private_api"], response_model=List[ASNCount]) def api_private_asn_by_month( clickhouse: ClickhouseDep, -) -> ASNByMonthResponse: +) -> List[ASNCount]: """Network count by month --- responses: @@ -111,8 +107,7 @@ def api_private_asn_by_month( """ li = list(query_click(clickhouse, q, {})) expand_dates(li) - validated: ASNByMonthResponse = [ASNCount(**item) for item in li] - return validated + return [ASNCount(**item) for item in li] class CountryCount(BaseModel): From eb9280950c1992c82f56a2acfa5bc196afa78d0b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:53:32 +0200 Subject: [PATCH 063/105] fix spacing --- .../services/ooniprobe/src/ooniprobe/routers/private.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1d1ce1061..1dfde6591 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -442,6 +442,7 @@ def api_private_website_network_tests( validated: List[MeasurementsByASN] = [MeasurementsByASN(**x) for x in results] return validated + class DayStats(BaseModel): test_day: date anomaly_count: conint(ge=0) @@ -449,9 +450,11 @@ class DayStats(BaseModel): failure_count: conint(ge=0) total_count: conint(ge=0) + class WebsiteStatsResponse(BaseModel): results: List[DayStats] + @router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) def api_private_website_stats( clickhouse: ClickhouseDep, @@ -496,6 +499,7 @@ class WebsiteURLItem(BaseModel): failure_count: conint(ge=0) total_count: conint(ge=0) + class PaginationMetadata(BaseModel): offset: int limit: int @@ -503,6 +507,7 @@ class PaginationMetadata(BaseModel): total_count: int next_url: Optional[AnyUrl] = None + class WebsiteURLsResponse(BaseModel): metadata: PaginationMetadata results: List[WebsiteURLItem] @@ -668,6 +673,7 @@ def api_private_vanilla_tor_stats( notok_networks=blocked, ) + class NetworkEntry(BaseModel): asn: int name: str @@ -845,6 +851,7 @@ class CountryOverviewResponse(BaseModel): measurement_count: int = Field(..., description="Number of measurements") network_count: int = Field(..., description="Number of networks measured") + @router.get("/country_overview", response_model=CountryOverviewResponse, tags=["private"]) def api_private_country_overview( clickhouse: ClickhouseDep, From 084a3176a9bb21b4ef92008374aeaa4ab6c898db Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:53:57 +0200 Subject: [PATCH 064/105] fix results type --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1dfde6591..20f6af3ce 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -714,7 +714,7 @@ def api_private_im_networks( """ test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] q = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc, "test_names": test_names}) - results = IMNetworksResponse() + results: Dict[str, IMNetworkStats] = {} for r in q: # get stats for test_name or create a new IMNetworksStats stats = results.get(r["test_name"], IMNetworkStats(anomaly_networks=[], ok_networks=[], last_tested=None)) From ef3d468f99d42ec3052861eb8a0b86be7a651fff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:54:12 +0200 Subject: [PATCH 065/105] fix datetime format --- .../services/ooniprobe/src/ooniprobe/routers/private.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 20f6af3ce..cc7b68804 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -744,6 +744,9 @@ class IMStatsItem(BaseModel): anomaly_count: Optional[int] = None test_day: datetime total_count: int + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } class IMStatsResponse(BaseModel): @@ -920,6 +923,10 @@ def api_private_global_overview( class GlobalOverviewStat(BaseModel): date: datetime value: int + model_config = { + "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } + } + class GlobalOverviewMonthResponse(BaseModel): networks_by_month: List[GlobalOverviewStat] From 7824990973dadd61c2fbe52cb928fd3690256195 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:55:11 +0200 Subject: [PATCH 066/105] fix typo: field name, class name --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index cc7b68804..ac12e5017 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -913,7 +913,7 @@ def api_private_global_overview( """ r = query_click_one_row(clickhouse, q, {}) assert r - result = GlobalOverViewResponse( + result = GlobalOverviewResponse( network_count=r["network_count"], country_count=r["country_count"], measurement_count=r["measurement_count"]) @@ -931,7 +931,7 @@ class GlobalOverviewStat(BaseModel): class GlobalOverviewMonthResponse(BaseModel): networks_by_month: List[GlobalOverviewStat] countries_by_month: List[GlobalOverviewStat] - measurement_by_month: List[GlobalOverviewStat] + measurements_by_month: List[GlobalOverviewStat] @router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) From bdce53da30fa7e0577387a37c35a61d3458380fb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 11:55:49 +0000 Subject: [PATCH 067/105] fix bool value --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ac12e5017..f988dd88b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -278,7 +278,7 @@ def check_report_id() -> CheckReportIDResponse: example: { "found": true, "v": 0 } """ - return CheckReportIDResponse(v=0, found=true) + return CheckReportIDResponse(v=0, found=True) def last_30days(begin=31, end=1): From 18fcc73ed74af31d2896a9e3a017b575636b595f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:56:47 +0200 Subject: [PATCH 068/105] remove old request.args --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index f988dd88b..f5b0644e5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -1172,9 +1172,6 @@ def api_private_asnmeta( 200: description: JSON object """ - asn = request.args.get("asn") - if asn is None: - raise BadRequest("missing asn") q = """SELECT org_name FROM asnmeta From dbe4ca89ee7455af58b6233582e28ac0e83c6e20 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 14:05:15 +0200 Subject: [PATCH 069/105] add sample fastpath data and tweak test values --- .../tests/fixtures/initdb/04-fastpath.sql | 1 + .../ooniprobe/tests/integ/test_private_api.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql diff --git a/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql b/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql new file mode 100644 index 000000000..912ecf1f4 --- /dev/null +++ b/ooniapi/services/ooniprobe/tests/fixtures/initdb/04-fastpath.sql @@ -0,0 +1 @@ +INSERT INTO table default.fastpath (`measurement_uid`, `report_id`, `input`, `probe_cc`, `probe_asn`, `test_name`, `test_start_time`, `measurement_start_time`, `filename`, `scores`, `platform`, `anomaly`, `confirmed`, `msm_failure`, `domain`, `software_name`, `software_version`, `control_failure`, `blocking_general`, `is_ssl_expected`, `page_len`, `page_len_ratio`, `server_cc`, `server_asn`, `server_as_name`, `test_version`, `architecture`, `engine_name`, `engine_version`, `test_runtime`, `blocking_type`, `test_helper_address`, `test_helper_type`, `ooni_run_link_id`, `is_verified`) VALUES ('20260110135447.088775_RS_signal_e7b599ccd49ca171', '20260110T135446Z_signal_RS_8771_n4_kVFlePYMNxHYcbxh', '', 'RS', 8771, 'signal', '2026-01-10 13:54:46', '2026-01-10 13:54:46', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.49980247, '', '', '', NULL, '0'), ('20260123110034.624460_VE_signal_aaeea63fd70b5899', '20260123T110032Z_signal_VE_8048_n4_0aicHCzlncXLgrDf', '', 'VE', 8048, 'signal', '2026-01-23 11:00:32', '2026-01-23 11:00:33', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.24.0', 1.0396768, '', '', '', NULL, '0'), ('20260117141659.519197_IT_signal_6ad8f1c0b89de448', '20260117T141658Z_signal_IT_24608_n4_h0OQyBSdutunwhcj', '', 'IT', 24608, 'signal', '2026-01-17 14:16:58', '2026-01-17 14:16:58', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.6059591, '', '', '', NULL, '0'), ('20260127043315.750799_ES_signal_9dc615c1cb2a4249', '20260127T043315Z_signal_ES_57269_n4_HnUIVcoBr1iaTQwU', '', 'ES', 57269, 'signal', '2026-01-27 04:33:15', '2026-01-27 04:33:15', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.35447666, '', '', '', NULL, '0'), ('20260105215811.053944_AU_signal_143a953edc5475f9', '20260105T215809Z_signal_AU_4764_n4_QcLeZ4AQIiTLM9yO', '', 'AU', 4764, 'signal', '2026-01-05 21:58:09', '2026-01-05 21:58:10', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.67212117, '', '', '', NULL, '0'), ('20260129043446.605662_SV_signal_f0e2dae8ab70b21e', '20260129T043445Z_signal_SV_27773_n4_d2SLo5doCXWMMMe6', '', 'SV', 27773, 'signal', '2026-01-29 04:34:45', '2026-01-29 04:34:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4660445, '', '', '', NULL, '0'), ('20260121064037.490724_ES_signal_b97aa744a3e8b7c9', '20260121T064036Z_signal_ES_57269_n4_qriPsy5OaTcOVIJK', '', 'ES', 57269, 'signal', '2026-01-21 06:40:36', '2026-01-21 06:40:36', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.47944492, '', '', '', NULL, '0'), ('20260120175831.837469_DE_signal_2712804df5faf5b8', '20260120T175831Z_signal_DE_680_n4_ytyvRjo0Ba05PfTN', '', 'DE', 680, 'signal', '2026-01-20 17:58:31', '2026-01-20 17:58:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.1062024, '', '', '', NULL, '0'), ('20260108174143.943125_FR_signal_a93aed855fbbc9bb', '20260108T174142Z_signal_FR_3215_n4_TG5PsH67W6fYJ6Qp', '', 'FR', 3215, 'signal', '2026-01-08 17:42:40', '2026-01-08 17:42:40', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.6', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.25.0', 0.5855959, '', '', '', NULL, '0'), ('20260128182122.005734_DE_signal_bcd5ee367a2525cb', '20260128T182121Z_signal_DE_680_n4_vykMKDVFlM5xyRGB', '', 'DE', 680, 'signal', '2026-01-28 18:21:21', '2026-01-28 18:21:21', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.09559728, '', '', '', NULL, '0'), ('20260110050954.007509_CM_signal_83b2054892b391a3', '20260110T050951Z_signal_CM_15964_n4_TeZN64NZzGN0PoPe', '', 'CM', 15964, 'signal', '2026-01-10 05:09:51', '2026-01-10 05:09:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 2.0773945, '', '', '', NULL, '0'), ('20260127061350.718282_GN_signal_a586a62ab2982dfa', '20260127T061348Z_signal_GN_37461_n4_jSZliomLWIofzQsz', '', 'GN', 37461, 'signal', '2026-01-27 06:13:47', '2026-01-27 06:13:47', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.6756234, '', '', '', NULL, '0'), ('20260102041813.309231_DE_signal_dc028560a210b6c1', '20260102T041812Z_signal_DE_3209_n4_Gu8rwQe9vcmCdKMk', '', 'DE', 3209, 'signal', '2026-01-02 04:18:12', '2026-01-02 04:18:12', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.61336344, '', '', '', NULL, '0'), ('20260120140327.403850_JP_signal_f20d6b7ff13153c1', '20260120T140325Z_signal_JP_45102_n4_My2S5QoD6HDyQRca', '', 'JP', 45102, 'signal', '2026-01-20 14:03:25', '2026-01-20 14:03:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 1.4022309, '', '', '', NULL, '0'), ('20260110160710.218928_DE_signal_f9eb3c4f05d6a772', '20260110T160709Z_signal_DE_3320_n4_4OJLILnklxI63roa', '', 'DE', 3320, 'signal', '2026-01-10 16:07:09', '2026-01-10 16:07:09', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.56549716, '', '', '', NULL, '0'), ('20260119144448.740559_US_signal_1d614b5a411607dc', '20260119T144448Z_signal_US_14615_n4_TpfUn5Zc5XrTKQil', '', 'US', 14615, 'signal', '2026-01-19 14:44:48', '2026-01-19 14:44:48', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 0.1717979, '', '', '', NULL, '0'), ('20260124031352.083113_ES_signal_d8082edf11a991a0', '20260124T031351Z_signal_ES_60203_n4_N7AKwOaSUkWN9Y3i', '', 'ES', 60203, 'signal', '2026-01-24 03:13:51', '2026-01-24 03:13:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4493603, '', '', '', NULL, '0'), ('20260127161132.817453_CA_signal_d98459cdc733a244', '20260127T161131Z_signal_CA_812_n4_muoBlMoRGlOLubog', '', 'CA', 812, 'signal', '2026-01-27 16:11:32', '2026-01-27 16:11:32', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'android', 'f', 'f', 't', '', 'ooniprobe-android-unattended', '3.7.3', '', 0, 0, 0, 0, '', 0, '', '0.2.2', 'arm64', 'ooniprobe-engine', '3.16.7', 0.6236891, '', '', '', NULL, '0'), ('20260101192636.414972_ES_signal_2e95090292761b6a', '20260101T192635Z_signal_ES_3352_n4_v9KQphoYgSyc9CC7', '', 'ES', 3352, 'signal', '2026-01-01 19:26:35', '2026-01-01 19:26:35', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.5347266, '', '', '', NULL, '0'), ('20260102001427.293388_ZW_signal_7ca4c020bfb8c733', '20260102T001425Z_signal_ZW_37332_n4_U3sNsOPWfQENtPbx', '', 'ZW', 37332, 'signal', '2026-01-02 00:14:25', '2026-01-02 00:14:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.9928689, '', '', '', NULL, '0'), ('20260126193945.103317_US_signal_341e23577ab01d08', '20260126T193944Z_signal_US_7922_n4_NKvVQf2foWl0Svoa', '', 'US', 7922, 'signal', '2026-01-26 19:39:44', '2026-01-26 19:39:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.2488854, '', '', '', NULL, '0'), ('20260113081502.171083_US_signal_c493a4cd3b4618b0', '20260113T081500Z_signal_US_11492_n4_aUQDYczPvKxMmlxc', '', 'US', 11492, 'signal', '2026-01-13 08:15:00', '2026-01-13 08:15:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.76373357, '', '', '', NULL, '0'), ('20260130045403.620913_NL_signal_1b76dba3ec160675', '20260130T045402Z_signal_NL_1136_n4_VMJMlf58x8LtrcdL', '', 'NL', 1136, 'signal', '2026-01-30 04:54:03', '2026-01-30 04:54:03', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.51608795, '', '', '', NULL, '0'), ('20260110154809.542690_NZ_signal_77c0e4b1ea472944', '20260110T154807Z_signal_NZ_4771_n4_i5IibPcfcFc60xtX', '', 'NZ', 4771, 'signal', '2026-01-10 15:48:07', '2026-01-10 15:48:07', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.8938294, '', '', '', NULL, '0'), ('20260103074148.111315_FR_signal_e22f2578c70032d8', '20260103T074147Z_signal_FR_12322_n4_BgqUeiKoAgkTHbe8', '', 'FR', 12322, 'signal', '2026-01-03 07:41:47', '2026-01-03 07:41:47', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.44495875, '', '', '', NULL, '0'), ('20260112045926.845756_ZA_signal_5dee9e0dda8a2b6b', '20260112T045924Z_signal_ZA_328471_n4_5gQlMl1mzxj3oGA3', '', 'ZA', 328471, 'signal', '2026-01-12 04:59:24', '2026-01-12 04:59:24', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.9813594, '', '', '', NULL, '0'), ('20260101180538.691977_ET_signal_8814d8efb9dfd565', '20260101T180537Z_signal_ET_24757_n4_vLNoqzjMyUotVNye', '', 'ET', 24757, 'signal', '2026-01-01 18:05:38', '2026-01-01 18:05:38', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.904734, '', '', '', NULL, '0'), ('20260126062200.282691_US_signal_434e3d22004d02a3', '20260126T062159Z_signal_US_5650_n4_xCUw4TgErIyyVVp2', '', 'US', 5650, 'signal', '2026-01-26 06:22:00', '2026-01-26 06:22:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.72052324, '', '', '', NULL, '0'), ('20260131233019.265756_DZ_signal_8337900470d1d647', '20260131T233018Z_signal_DZ_36947_n4_jIBGzDnVQmj0poGW', '', 'DZ', 36947, 'signal', '2026-01-31 23:30:17', '2026-01-31 23:30:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.7189328, '', '', '', NULL, '0'), ('20260118221838.842575_US_signal_0a3512ee77378581', '20260118T221837Z_signal_US_11427_n4_pqw4mf1hB91ubI3J', '', 'US', 11427, 'signal', '2026-01-18 22:18:37', '2026-01-18 22:18:37', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.43586272, '', '', '', NULL, '0'), ('20260121213502.700887_FI_signal_d03ed86cb78733cf', '20260121T213502Z_signal_FI_719_n4_PdCnHJ7kXFWHWAv9', '', 'FI', 719, 'signal', '2026-01-21 21:35:02', '2026-01-21 21:35:02', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.07162474, '', '', '', NULL, '0'), ('20260120234837.045187_FR_signal_d21a8e1e8b757cc6', '20260120T234836Z_signal_FR_3215_n4_G5EQDEXkA27Wx1M0', '', 'FR', 3215, 'signal', '2026-01-20 23:48:36', '2026-01-20 23:48:36', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.19238907, '', '', '', NULL, '0'), ('20260121044125.972617_FI_signal_311cf8bbfdcf8672', '20260121T044125Z_signal_FI_719_n4_TbzDNCzGiqnDLeit', '', 'FI', 719, 'signal', '2026-01-21 04:41:25', '2026-01-21 04:41:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.084080726, '', '', '', NULL, '0'), ('20260121052901.443389_US_signal_1a074c793d403bc7', '20260121T052901Z_signal_US_397005_n4_wIYnABVJEh0orZP2', '', 'US', 397005, 'signal', '2026-01-21 05:29:00', '2026-01-21 05:29:01', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.14746083, '', '', '', NULL, '0'), ('20260102020535.145478_RU_signal_5ca6f103dd82b676', '20260102T020514Z_signal_RU_25513_n4_5foqozJNdueTiBL8', '', 'RU', 25513, 'signal', '2026-01-02 02:05:14', '2026-01-02 02:05:14', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'windows', 't', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 20.087662, '', '', '', NULL, '0'), ('20260125103153.448150_BR_signal_6d0b131ce71115af', '20260125T103152Z_signal_BR_28210_n4_YnN5X9pxKeiXhDsZ', '', 'BR', 28210, 'signal', '2026-01-25 10:31:52', '2026-01-25 10:31:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.1573689, '', '', '', NULL, '0'), ('20260129222159.563844_IN_signal_a2b24f411322cc53', '20260129T222157Z_signal_IN_17465_n4_rlFpg9LwkMJr4IbF', '', 'IN', 17465, 'signal', '2026-01-29 22:22:00', '2026-01-29 22:22:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.038376, '', '', '', NULL, '0'), ('20260120002609.685381_IT_signal_42648e1be5cb6da5', '20260120T002608Z_signal_IT_3269_n4_5gj1XUhi3GTNhQ2c', '', 'IT', 3269, 'signal', '2026-01-20 00:26:08', '2026-01-20 00:26:08', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.75125754, '', '', '', NULL, '0'), ('20260127033137.903663_US_signal_95796985eb2062eb', '20260127T033137Z_signal_US_20115_n4_CTnZJDdcZ9NNJo9q', '', 'US', 20115, 'signal', '2026-01-27 03:31:37', '2026-01-27 03:31:37', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.35777518, '', '', '', NULL, '0'), ('20260126191421.710076_US_signal_0525c8c13e158738', '20260126T191421Z_signal_US_7922_n4_KYf66iopHyD6ohhY', '', 'US', 7922, 'signal', '2026-01-26 19:14:20', '2026-01-26 19:14:21', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.3128446, '', '', '', NULL, '0'), ('20260118033803.250613_CM_signal_b52936a77ea8b059', '20260118T033800Z_signal_CM_15964_n4_bCoPC05wnzJoHB7x', '', 'CM', 15964, 'signal', '2026-01-18 03:38:00', '2026-01-18 03:38:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.7432516, '', '', '', NULL, '0'), ('20260123090840.650629_CA_signal_71840b6978320000', '20260123T090839Z_signal_CA_577_n4_PIhBTPyLi8PHsoDy', '', 'CA', 577, 'signal', '2026-01-23 09:08:39', '2026-01-23 09:08:40', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.27.0', 0.19444399, '', '', '', NULL, '0'), ('20260112134453.775759_US_signal_785e7b66566d2be1', '20260112T134452Z_signal_US_11090_n4_pxUe58CIjkV9HBiz', '', 'US', 11090, 'signal', '2026-01-12 13:44:52', '2026-01-12 13:44:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.5586467, '', '', '', NULL, '0'), ('20260103103802.609988_FR_signal_0da8cc27f84d5ffd', '20260103T103801Z_signal_FR_12322_n4_DdojMxeEyUYGlzsq', '', 'FR', 12322, 'signal', '2026-01-03 10:37:59', '2026-01-03 10:37:59', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.1015654, '', '', '', NULL, '0'), ('20260106215331.674612_CM_signal_dc63250fe642e00f', '20260106T215328Z_signal_CM_15964_n4_UdbjBXmonpuftMiV', '', 'CM', 15964, 'signal', '2026-01-06 21:53:28', '2026-01-06 21:53:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 2.0551805, '', '', '', NULL, '0'), ('20260110181818.481389_RU_signal_d117815babb1c63b', '20260110T181818Z_signal_RU_39087_n4_kmgsaRh6rfCX8jl1', '', 'RU', 39087, 'signal', '2026-01-06 04:09:49', '2026-01-06 04:09:50', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 10.440062, '', '', '', NULL, '0'), ('20260120122355.499063_TR_signal_c91c358c4e1168dd', '20260120T122354Z_signal_TR_47331_n4_xJeMQAojMLfKZgjL', '', 'TR', 47331, 'signal', '2026-01-20 12:23:54', '2026-01-20 12:23:54', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.60208714, '', '', '', NULL, '0'), ('20260129151311.456862_ES_signal_9c0a9695e0265bf4', '20260129T151310Z_signal_ES_3352_n4_s3RXoTvKt4R2jUav', '', 'ES', 3352, 'signal', '2026-01-29 15:13:11', '2026-01-29 15:13:11', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.56744546, '', '', '', NULL, '0'), ('20260107073559.543536_IT_signal_64816471ddef9ede', '20260107T073558Z_signal_IT_30722_n4_6axwuwALvRqD47Og', '', 'IT', 30722, 'signal', '2026-01-07 07:35:58', '2026-01-07 07:35:58', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 0.8084099, '', '', '', NULL, '0'), ('20260108001413.438299_CM_signal_e1b01bec82c0230b', '20260108T001410Z_signal_CM_15964_n4_8XY0N6z9x8lrzlj2', '', 'CM', 15964, 'signal', '2026-01-08 00:14:09', '2026-01-08 00:14:10', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 2.145543, '', '', '', NULL, '0'), ('20260118061633.900624_PK_signal_f19d350c2a18c773', '20260118T061613Z_signal_PK_135407_n4_oTEQYpvi1viu8ElG', '', 'PK', 135407, 'signal', '2026-01-18 06:16:13', '2026-01-18 06:16:13', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 20.223848, '', '', '', NULL, '0'), ('20260102072053.373246_US_signal_a449924ebfb9e157', '20260102T072051Z_signal_US_33363_n4_UPuo0PoKSfXSELiQ', '', 'US', 33363, 'signal', '2026-01-02 07:20:52', '2026-01-02 07:20:52', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.5', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.24.0', 1.0205637, '', '', '', NULL, '0'), ('20260129143549.899949_US_signal_1fe8b6ad4eb95bbf', '20260129T143548Z_signal_US_7018_n4_J2zo41KS2WzBfUHg', '', 'US', 7018, 'signal', '2026-01-29 14:35:49', '2026-01-29 14:35:49', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.61999345, '', '', '', NULL, '0'), ('20260105113221.305238_GB_signal_05b461d7c031620a', '20260105T113218Z_signal_GB_328309_n4_CLE8li5t181WljPa', '', 'GB', 328309, 'signal', '2026-01-05 11:32:18', '2026-01-05 11:32:19', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 1.7276975, '', '', '', NULL, '0'), ('20260110182705.937428_US_signal_5a066278fa5d5fce', '20260110T182705Z_signal_US_7922_n4_UxZt0pglkpSVrn1W', '', 'US', 7922, 'signal', '2026-01-10 18:27:05', '2026-01-10 18:27:05', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.2958476, '', '', '', NULL, '0'), ('20260119082028.968978_US_signal_6c38e75da1910c76', '20260119T082028Z_signal_US_62658_n4_1oIOQ5LY5EabuzRV', '', 'US', 62658, 'signal', '2026-01-19 08:20:28', '2026-01-19 08:20:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.2513034, '', '', '', NULL, '0'), ('20260110211551.852077_AU_signal_9c5540ccf5ab7b76', '20260110T211550Z_signal_AU_4739_n4_DSb1hru8mzKHbOj5', '', 'AU', 4739, 'signal', '2026-01-10 21:15:50', '2026-01-10 21:15:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.0.6', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.25.0', 0.7667587, '', '', '', NULL, '0'), ('20260123012736.613823_FR_signal_1d12081f301db0bf', '20260123T012735Z_signal_FR_15557_n4_m9AMWnojHrJFlXVd', '', 'FR', 15557, 'signal', '2026-01-23 01:27:39', '2026-01-23 01:27:39', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm', 'ooniprobe-engine', '3.28.0', 0.5139983, '', '', '', NULL, '0'), ('20260126214344.619344_ES_signal_ea4412e4cb1b6296', '20260126T214344Z_signal_ES_57269_n4_Oh9Pk4F49VFNS5HD', '', 'ES', 57269, 'signal', '2026-01-26 21:43:44', '2026-01-26 21:43:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.3829573, '', '', '', NULL, '0'), ('20260128193853.330808_BR_signal_191aa206398ad2b2', '20260128T193852Z_signal_BR_28210_n4_vwLcSw36MAdDg6xA', '', 'BR', 28210, 'signal', '2026-01-28 19:38:53', '2026-01-28 19:38:53', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.71647525, '', '', '', NULL, '0'), ('20260125042659.141540_CY_signal_baa784af572c279d', '20260125T042657Z_signal_CY_35432_n4_plK8SnDPLvulz8on', '', 'CY', 35432, 'signal', '2026-01-25 04:27:00', '2026-01-25 04:27:00', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.1204401, '', '', '', NULL, '0'), ('20260117015457.248827_SI_signal_fc6af267bb729798', '20260117T015456Z_signal_SI_3212_n4_nmRG5b8LnLhWPsQ0', '', 'SI', 3212, 'signal', '2026-01-17 01:54:56', '2026-01-17 01:54:56', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.60261893, '', '', '', NULL, '0'), ('20260131204919.230824_SE_signal_2c4204c6e30ede77', '20260131T204918Z_signal_SE_8473_n4_37HchTGzYHU2oGMk', '', 'SE', 8473, 'signal', '2026-01-31 20:49:18', '2026-01-31 20:49:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'linux', 'f', 'f', 'f', '', 'ooniprobe-cli-unattended', '3.27.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.39307818, '', '', '', NULL, '0'), ('20260109230709.648875_RU_signal_083185be8c783b72', '20260109T230658Z_signal_RU_25159_n4_LpoDkzA3ZEz5mtHb', '', 'RU', 25159, 'signal', '2026-01-09 23:06:58', '2026-01-09 23:06:58', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'android', 't', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 10.777376, '', '', '', NULL, '0'), ('20260102135516.354188_ES_signal_63305644f94d7800', '20260102T135515Z_signal_ES_3352_n4_cMVCYOuKJPdKyWvs', '', 'ES', 3352, 'signal', '2026-01-02 13:55:14', '2026-01-02 13:55:14', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4341947, '', '', '', NULL, '0'), ('20260109074220.142662_ES_signal_393f01c95fafdd1a', '20260109T074219Z_signal_ES_3352_n4_4g9zQPdSHQZiFon0', '', 'ES', 3352, 'signal', '2026-01-09 07:42:18', '2026-01-09 07:42:18', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4999763, '', '', '', NULL, '0'), ('20260108162529.001785_US_signal_acd93e5d19dfa896', '20260108T162527Z_signal_US_10796_n4_lNZP314S7LCyQuTo', '', 'US', 10796, 'signal', '2026-01-08 16:25:28', '2026-01-08 16:25:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6204547, '', '', '', NULL, '0'), ('20260129124738.638259_BR_signal_16e194fe04c17935', '20260129T124737Z_signal_BR_266121_n4_bX8sQE36Zbb4IKQO', '', 'BR', 266121, 'signal', '2026-01-29 12:47:38', '2026-01-29 12:47:38', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.5957706, '', '', '', NULL, '0'), ('20260108084022.731865_US_signal_b6756029807297e6', '20260108T084021Z_signal_US_20115_n4_87I8o9k7OwrXwi2g', '', 'US', 20115, 'signal', '2026-01-08 08:40:21', '2026-01-08 08:40:22', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4671578, '', '', '', NULL, '0'), ('20260105060506.962435_US_signal_2b3091d1974a289b', '20260105T060505Z_signal_US_7018_n4_j2yYYu5s0fhAiM22', '', 'US', 7018, 'signal', '2026-01-05 06:05:04', '2026-01-05 06:05:04', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.57331777, '', '', '', NULL, '0'), ('20260115021531.127320_VN_signal_df98957524a319cb', '20260115T021530Z_signal_VN_7552_n4_4QvRK9tJliFJp5xq', '', 'VN', 7552, 'signal', '2026-01-15 02:15:29', '2026-01-15 02:15:29', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8183516, '', '', '', NULL, '0'), ('20260126010331.893272_US_signal_bb73f06817b94957', '20260126T010331Z_signal_US_33387_n4_B3oBoJDmW5wpFqvC', '', 'US', 33387, 'signal', '2026-01-26 01:03:31', '2026-01-26 01:03:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.10474518, '', '', '', NULL, '0'), ('20260128100621.739976_RO_signal_25e202dac905fae5', '20260128T100620Z_signal_RO_8708_n4_Vi17A70VX1exQlXO', '', 'RO', 8708, 'signal', '2026-01-28 10:06:19', '2026-01-28 10:06:19', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.667061, '', '', '', NULL, '0'), ('20260109052451.264065_CA_signal_1eb36573803fa762', '20260109T052450Z_signal_CA_577_n4_J3zYJ0llJEvbDFWb', '', 'CA', 577, 'signal', '2026-01-09 05:24:50', '2026-01-09 05:24:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.2.2', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.27.0', 0.6373465, '', '', '', NULL, '0'), ('20260106174916.480115_FR_signal_97df58aec56b5582', '20260106T174915Z_signal_FR_15557_n4_Z0PB4a69L1InxQWO', '', 'FR', 15557, 'signal', '2026-01-06 17:49:15', '2026-01-06 17:49:15', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6112673, '', '', '', NULL, '0'), ('20260130175951.568421_TR_signal_da2133354225608e', '20260130T175950Z_signal_TR_47331_n4_P6wTfiWSRYW9lWVI', '', 'TR', 47331, 'signal', '2026-01-30 17:59:51', '2026-01-30 17:59:51', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6442853, '', '', '', NULL, '0'), ('20260127044812.840537_GB_signal_61ba50916d8cffac', '20260127T044812Z_signal_GB_6871_n4_8uHbgFez8O2nCsox', '', 'GB', 6871, 'signal', '2026-01-27 04:48:12', '2026-01-27 04:48:12', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.09698934, '', '', '', NULL, '0'), ('20260116085126.865917_TH_signal_2f6e1bb32fc42308', '20260116T085125Z_signal_TH_45758_n4_iPrmlsbjjTozbmYR', '', 'TH', 45758, 'signal', '2026-01-16 08:51:25', '2026-01-16 08:51:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.7706264, '', '', '', NULL, '0'), ('20260124075855.723591_DE_signal_b8d817b2c8c35a59', '20260124T075855Z_signal_DE_31898_n4_IFIclPYPWpPPqoXs', '', 'DE', 31898, 'signal', '2026-01-24 07:58:55', '2026-01-24 07:58:55', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.05214951, '', '', '', NULL, '0'), ('20260130164932.080917_MX_signal_a7891bea830d0fa5', '20260130T164931Z_signal_MX_8151_n4_BfQpxYU7DDbzLKeN', '', 'MX', 8151, 'signal', '2026-01-30 16:49:31', '2026-01-30 16:49:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.27090517, '', '', '', NULL, '0'), ('20260123050626.011753_UA_signal_7781bcc2a8fb0faa', '20260123T050625Z_signal_UA_6876_n4_oZ4fYJJ27Q4wxUCO', '', 'UA', 6876, 'signal', '2026-01-23 05:06:26', '2026-01-23 05:06:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.4885918, '', '', '', NULL, '0'), ('20260120045040.224875_DE_signal_b868eac6460f197b', '20260120T045039Z_signal_DE_8881_n4_Htoc7IKmn9n5tu2x', '', 'DE', 8881, 'signal', '2026-01-20 04:50:39', '2026-01-20 04:50:39', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.29948318, '', '', '', NULL, '0'), ('20260129060428.575194_CY_signal_25f09d8c3a288f5c', '20260129T060427Z_signal_CY_15805_n4_gwqonwh2Ax2n4dor', '', 'CY', 15805, 'signal', '2026-01-29 06:04:26', '2026-01-29 06:04:26', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.779632, '', '', '', NULL, '0'), ('20260115083527.901995_IN_signal_80baaedabdc895de', '20260115T083526Z_signal_IN_24560_n4_czpqbQ7j7Z6tMbwk', '', 'IN', 24560, 'signal', '2026-01-15 08:35:25', '2026-01-15 08:35:25', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8536811, '', '', '', NULL, '0'), ('20260128060546.823485_US_signal_58a7b8e78ea2f182', '20260128T060545Z_signal_US_203020_n4_KyJQZ7MA7SAvpG3W', '', 'US', 203020, 'signal', '2026-01-28 06:05:44', '2026-01-28 06:05:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 1.0699235, '', '', '', NULL, '0'), ('20260102010446.359409_ES_signal_151c38de04d7ed1f', '20260102T010445Z_signal_ES_57269_n4_RCxu595agyf2h3l4', '', 'ES', 57269, 'signal', '2026-01-02 01:04:45', '2026-01-02 01:04:45', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'ios', 'f', 'f', 'f', '', 'ooniprobe-ios-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.45330188, '', '', '', NULL, '0'), ('20260114014644.641698_US_signal_110c28b6d721c302', '20260114T014644Z_signal_US_701_n4_KZ2YJ8Nsa5ZIjo81', '', 'US', 701, 'signal', '2026-01-14 01:46:44', '2026-01-14 01:46:44', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 0.26397383, '', '', '', NULL, '0'), ('20260125214505.275170_ES_signal_9f8649a2dc39dfe4', '20260125T214504Z_signal_ES_57269_n4_sQgWIakYDwIMoPUo', '', 'ES', 57269, 'signal', '2026-01-25 21:45:04', '2026-01-25 21:45:04', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'macos', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.39790463, '', '', '', NULL, '0'), ('20260112102028.237552_VN_signal_78f9fa738b840749', '20260112T102027Z_signal_VN_45899_n4_WRISDRVfjsS1MVhD', '', 'VN', 45899, 'signal', '2026-01-12 10:20:26', '2026-01-12 10:20:27', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.23.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.23.0', 0.8143061, '', '', '', NULL, '0'), ('20260119235541.383591_GB_signal_9a52a520cbf53453', '20260119T235531Z_signal_GB_6871_n4_Ayw1cubCzQQomBD9', '', 'GB', 6871, 'signal', '2026-01-19 23:55:31', '2026-01-19 23:55:31', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 10.05995, '', '', '', NULL, '0'), ('20260112213831.467389_KZ_signal_a16ff4187ffd2374', '20260112T213830Z_signal_KZ_60286_n4_4jfMB2x6dRPw1zue', '', 'KZ', 60286, 'signal', '2026-01-12 21:38:09', '2026-01-12 21:38:09', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 0.8840163, '', '', '', NULL, '0'), ('20260129100428.809164_RU_signal_0fd185e6a396f133', '20260129T100408Z_signal_RU_12389_n4_4ckhwMuo9RwCX34e', '', 'RU', 12389, 'signal', '2026-01-29 10:04:08', '2026-01-29 10:04:09', '', '{"blocking_general":1.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"analysis":{"signal_backend_failure":"generic_timeout_error"}}', 'windows', 't', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.26.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.26.0', 20.172535, '', '', '', NULL, '0'), ('20260117012950.855475_ES_signal_4b764baa80b700db', '20260117T012949Z_signal_ES_3352_n4_oHfQvBg28Q7SoDou', '', 'ES', 3352, 'signal', '2026-01-17 01:29:49', '2026-01-17 01:29:50', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.6271005, '', '', '', NULL, '0'), ('20260108161611.436103_FR_signal_d5559d8cbfb9f757', '20260108T161610Z_signal_FR_12322_n4_jh7D6RFRDpNxobbn', '', 'FR', 12322, 'signal', '2026-01-08 16:16:08', '2026-01-08 16:16:08', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.4508722, '', '', '', NULL, '0'), ('20260122062316.714352_CA_signal_ed9118e794ddd148', '20260122T062315Z_signal_CA_852_n4_vCuP2C3zwUWi9D54', '', 'CA', 852, 'signal', '2026-01-22 06:23:15', '2026-01-22 06:23:16', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.13090481, '', '', '', NULL, '0'), ('20260101045021.148218_ES_signal_c6a0ac5e18361571', '20260101T045020Z_signal_ES_57269_n4_Kj915bUjDJ1q8F7Y', '', 'ES', 57269, 'signal', '2026-01-01 04:50:20', '2026-01-01 04:50:20', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.43395877, '', '', '', NULL, '0'), ('20260117225217.575965_KE_signal_9b2985f63256df7e', '20260117T225216Z_signal_KE_37061_n4_65qgh7tL9ImwL4Eo', '', 'KE', 37061, 'signal', '2026-01-17 22:52:14', '2026-01-17 22:52:14', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'windows', 'f', 'f', 'f', '', 'ooniprobe-desktop-unattended', '3.24.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'amd64', 'ooniprobe-engine', '3.24.0', 1.1794214, '', '', '', NULL, '0'), ('20260101181803.339683_US_signal_dc3e20af849caa69', '20260101T181802Z_signal_US_395466_n4_RVRxwCzLJANrPAFm', '', 'US', 395466, 'signal', '2026-01-01 18:18:03', '2026-01-01 18:18:03', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.70013964, '', '', '', NULL, '0'), ('20260123204628.537028_US_signal_dbd08de12970f3d9', '20260123T204627Z_signal_US_174_n4_1od1R2Ja8akVVEzM', '', 'US', 174, 'signal', '2026-01-23 20:46:27', '2026-01-23 20:46:28', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0,"accuracy":0.0}', 'linux', 'f', 'f', 't', '', 'iThena-ooniprobe', '1.0.0', '', 0, 0, 0, 0, '', 0, '', '0.2.0', '', 'ooniprobe-engine', '3.10.0-beta.3', 0.27568492, '', '', '', NULL, '0'), ('20260114224456.610555_FR_signal_2acdb40bf99cbebf', '20260114T224455Z_signal_FR_16276_n4_wiuEVNNkvDGju0no', '', 'FR', 16276, 'signal', '2026-01-14 22:44:55', '2026-01-14 22:44:55', '', '{"blocking_general":0.0,"blocking_global":0.0,"blocking_country":0.0,"blocking_isp":0.0,"blocking_local":0.0}', 'android', 'f', 'f', 'f', '', 'ooniprobe-android-unattended', '5.3.0', '', 0, 0, 0, 0, '', 0, '', '0.2.5', 'arm64', 'ooniprobe-engine', '3.28.0', 0.5232244, '', '', '', NULL, '0'); diff --git a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py index 8fae5ab30..624081985 100644 --- a/ooniapi/services/ooniprobe/tests/integ/test_private_api.py +++ b/ooniapi/services/ooniprobe/tests/integ/test_private_api.py @@ -32,7 +32,7 @@ def test_private_api_countries_by_month(client): assert len(response) > 0, response r = response[0] assert sorted(r.keys()) == ["date", "value"] - assert r["value"] > 1 + assert r["value"] > 10 assert r["value"] < 1000 assert r["date"].endswith("T00:00:00+00:00") @@ -80,7 +80,7 @@ def test_private_api_countries_total(client, log): assert len(response["countries"]) >= 20 for a in response["countries"]: if a["alpha_2"] == "CA": - assert a["count"] > 100 + assert a["count"] > 3 assert a["name"] == "Canada" return @@ -250,8 +250,8 @@ def test_private_api_country_overview(client): url = "country_overview?probe_cc=BR" resp = privapi(client, url) assert resp["first_bucket_date"].startswith("20"), resp - assert resp["measurement_count"] > 1000 - assert resp["network_count"] > 10 + assert resp["measurement_count"] > 1 + assert resp["network_count"] > 1 def test_private_api_global_overview(client): @@ -328,7 +328,15 @@ def test_private_api_networks(client, log): resp = privapi(client, "networks") assert resp["v"] == 0 assert len(resp["results"]) > 50 - assert resp["results"][0] == {"cnt": 380, "org_name": "", "probe_asn": 58224} + assert resp["results"][0] + res = resp["results"][0] + for k in ["cnt", "org_name", "probe_asn"]: + assert k in res + assert isinstance(res["probe_asn"], int) + assert res["probe_asn"] > 0 + assert isinstance(res["cnt"], int) + assert res["cnt"] > 0 + assert isinstance(res["org_name"], str) # # /domains From 2525baba72fe4d36dbdf1a8e03ae5dca1634dde3 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 28 Apr 2026 13:13:46 +0000 Subject: [PATCH 070/105] FIXME: this exception isn't caught on input XY --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index f5b0644e5..7c0f9bb72 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -619,7 +619,7 @@ class TorStatsResponse(BaseModel): @router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) def api_private_vanilla_tor_stats( clickhouse: ClickhouseDep, - probe_cc: CountryAlpha2 = Query(..., description="Country Code") + probe_cc: str = Query(..., description="Country Code") ) -> TorStatsResponse: """Tor statistics over ASN for a given CC --- @@ -633,6 +633,10 @@ def api_private_vanilla_tor_stats( '200': description: TODO """ + try: + CountryAlpha2(probe_cc) + except Exception: + raise blocked = 0 nets = [] s = """SELECT From 8232ecc36d768c529e9be287524e21da3fa8e53c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 15:08:42 +0200 Subject: [PATCH 071/105] normalize api tags --- .../ooniprobe/src/ooniprobe/routers/private.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7c0f9bb72..32c4bc64b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -87,7 +87,7 @@ class ASNCount(BaseModel): } -@router.get("/asn_by_month", tags=["private_api"], response_model=List[ASNCount]) +@router.get("/asn_by_month", tags=["private"], response_model=List[ASNCount]) def api_private_asn_by_month( clickhouse: ClickhouseDep, ) -> List[ASNCount]: @@ -120,7 +120,7 @@ class CountryCount(BaseModel): -@router.get("/countries_by_month", tags=["private_api"], response_model=List[CountryCount]) +@router.get("/countries_by_month", tags=["private"], response_model=List[CountryCount]) def api_private_countries_by_month( clickhouse: ClickhouseDep, ) -> List[CountryCount]: @@ -152,7 +152,7 @@ class TestNameResponse(BaseModel): test_names: List[TestName] -@router.get("/test_names", tags=["private_api"], response_model=TestNameResponse) +@router.get("/test_names", tags=["private"], response_model=TestNameResponse) def api_private_test_names() -> TestNameResponse: """Provides test names and descriptions to Explorer --- @@ -200,7 +200,7 @@ class CountryStatResponse(BaseModel): countries: List[CountryStat] = Field(..., description="List of countries") -@router.get("/countries", tags=["private_api"], response_model=CountryStatResponse) +@router.get("/countries", tags=["private"], response_model=CountryStatResponse) def api_private_countries( clickhouse: ClickhouseDep, ) -> CountryStatResponse: @@ -232,7 +232,7 @@ def api_private_countries( @router.get( "/quotas_summary", response_model=List[CountryStat], - tags=["private_api"], + tags=["private"], dependencies=[Depends(role_required(["admin"]))], ) def api_private_quotas_summary() -> List[CountryStat]: @@ -251,7 +251,7 @@ class CheckReportIDResponse(BaseModel): @router.get("/check_report_id", response_model=CheckReportIDResponse, - tags=["private_api"], + tags=["private"], ) def check_report_id() -> CheckReportIDResponse: """Legacy. Used to check if a report_id existed in the fastpath table. From 5ff6bd73e58ddad5e8103b814d0017cf70c4a3fa Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 15:43:58 +0200 Subject: [PATCH 072/105] omit responses; let FastAPI generate docstring --- .../ooniprobe/src/ooniprobe/routers/private.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 32c4bc64b..3b6671561 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -92,11 +92,8 @@ def api_private_asn_by_month( clickhouse: ClickhouseDep, ) -> List[ASNCount]: """Network count by month - --- - responses: - '200': - description: [{"date":"2018-08-31","value":4411}, ... ] """ + q = """SELECT COUNT(DISTINCT(probe_asn)) AS value, toStartOfMonth(measurement_start_time) AS date @@ -125,10 +122,6 @@ def api_private_countries_by_month( clickhouse: ClickhouseDep, ) -> List[CountryCount]: """Countries count by month - --- - responses: - '200': - description: TODO """ q = """SELECT COUNT(DISTINCT(probe_cc)) AS value, @@ -155,10 +148,6 @@ class TestNameResponse(BaseModel): @router.get("/test_names", tags=["private"], response_model=TestNameResponse) def api_private_test_names() -> TestNameResponse: """Provides test names and descriptions to Explorer - --- - responses: - '200': - description: TODO """ # TODO: eventually drop this, once we see nobody is using it TEST_NAMES = { @@ -205,10 +194,6 @@ def api_private_countries( clickhouse: ClickhouseDep, ) -> CountryStatResponse: """Summary of countries - --- - responses: - '200': - description: {"countries": [{"alpha_2": x, "count": y, "name": z}, ... ]} """ q = """ SELECT probe_cc, COUNT() AS measurement_count From 3f4223918fb302070f6a720f7e74434d718a994c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 15:55:05 +0200 Subject: [PATCH 073/105] omit responses docstring from legacy check_report_id generate API docstring with FastAPI --- .../src/ooniprobe/routers/private.py | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 3b6671561..ee7031dc4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -240,28 +240,7 @@ class CheckReportIDResponse(BaseModel): ) def check_report_id() -> CheckReportIDResponse: """Legacy. Used to check if a report_id existed in the fastpath table. - Used by https://github.com/ooni/probe/issues/1034 - --- - produces: - - application/json - parameters: - - name: report_id - in: query - type: string - responses: - 200: - description: Always returns True. - schema: - type: object - properties: - v: - type: integer - description: version number of this response - found: - type: boolean - description: True - example: { "found": true, "v": 0 } - + Used by https://github.com/ooni/probe/issues/1034. Always returns True. """ return CheckReportIDResponse(v=0, found=True) From 8b651d1d445fff277a4354aa4163f426f03f94e6 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 15:57:33 +0200 Subject: [PATCH 074/105] add response model validation for /api/_/test_coverage omit responses; let FastAPI generate API docs --- .../src/ooniprobe/routers/private.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ee7031dc4..361e6a928 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -336,28 +336,29 @@ def get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups): return query_click(clickhouse, sql.text(s), d) +class NetworkCoveragePoint(BaseModel): + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)", example="2021-10-16") + count: int = Field(..., description="Count of unique ASNs seen that day", example=58) + + +class TestCoveragePoint(BaseModel): + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)", example="2021-10-16") + test_group: str = Field(..., description="Test group name", example="websites") + count: int = Field(..., description="Number of measurements for this test group on that day", example=4888) + + class TestCoverageResponse(BaseModel): - network_coverage: List[Any] - test_coverage: List[Any] + network_coverage: List[NetworkCoveragePoint] = Field(..., description="Daily network coverage (ASNs per day)") + test_coverage: List[TestCoveragePoint] = Field(..., description="Per-test-group coverage per day") @router.get("/test_coverage", response_model=TestCoverageResponse, tags=["private"]) def api_private_test_coverage( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), - test_groups: str = Query(None, description="XXX: What is this?") + test_groups: str = Query(None, description="Comma-separated list of test group keys to filter results", example="websites,im") ) -> TestCoverageResponse: """Return number of measurements per day across test categories - --- - parameters: - - name: probe_cc - in: query - type: string - minLength: 2 - required: true - responses: - '200': - description: '{"network_coverage: [...], "test_coverage": [...]}' """ # TODO: merge the two queries into one? # TODO: remove test categories or move aggregation to the front-end? From a7ac388c2636d3026d089a753c024b0b8728f57a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:09:04 +0200 Subject: [PATCH 075/105] add response model validation for /api/_/websites_networks adds method docstring --- .../src/ooniprobe/routers/private.py | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 361e6a928..5774be5e2 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -381,17 +381,7 @@ def api_private_website_network_tests( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") ) -> List[MeasurementsByASN]: - """TODO - --- - parameters: - - name: probe_cc - in: query - type: string - description: The two letter country code - responses: - '200': - description: TODO - """ + """Daily counts of website measurements per ASN for the past 31 days, returned as a list of (probe_asn, count) ordered by count descending.""" s = """SELECT COUNT() AS count, probe_asn @@ -409,15 +399,15 @@ def api_private_website_network_tests( class DayStats(BaseModel): - test_day: date - anomaly_count: conint(ge=0) - confirmed_count: conint(ge=0) - failure_count: conint(ge=0) - total_count: conint(ge=0) + test_day: date = Field(..., description="Date for the aggregated counts (YYYY-MM-DD)", example="2026-03-29") + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies on this day", example=5) + confirmed_count: int = Field(..., description="Number of anomalies confirmed on this day", example=2) + failure_count: int = Field(..., description="Number of measurements that failed on this day", example=10) + total_count: int = Field(..., description="Total number of measurements for this day", example=100) class WebsiteStatsResponse(BaseModel): - results: List[DayStats] + results: List[DayStats] = Field(..., description="Daily aggregated statistics for the queried website, country, and ASN (ordered by day)") @router.get("/website_stats", response_model=WebsiteStatsResponse, tags=["private"]) @@ -426,13 +416,8 @@ def api_private_website_stats( input: AnyUrl = Query(..., description="Website to query stats"), probe_cc: CountryAlpha2 = Query(..., description="Country Code"), probe_asn: int = Query(..., description="ASN (integer)"), -) -> Response: - """Returns daily aggregated stats for the given website, country and ASN - --- - responses: - '200': - description: daily website stats - """ +) -> WebsiteStatsResponse: + """Daily aggregated website measurement statistics (anomalies, confirmations, failures, and totals) for the past 31 days.""" # uses_pg_index counters_day_cc_asn_input_idx a BRIN index was not used at # all, but BTREE on (measurement_start_day, probe_cc, probe_asn, input) # made queries go from full scan to 50ms From 8a2120e4dd0845ad868c3ddbd7193c7f8056f521 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:15:05 +0200 Subject: [PATCH 076/105] add response model validation, method docstring for /api/_/website_stats --- .../src/ooniprobe/routers/private.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5774be5e2..ad72ba43b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -443,24 +443,24 @@ def api_private_website_stats( class WebsiteURLItem(BaseModel): - input: AnyUrl - anomaly_count: conint(ge=0) - confirmed_count: conint(ge=0) - failure_count: conint(ge=0) - total_count: conint(ge=0) + input: AnyUrl = Field(..., description="Tested URL") + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies for this URL in the past 31 days", example=5) + confirmed_count: int = Field(..., description="Number of anomalies confirmed for this URL in the past 31 days", example=2) + failure_count: int = Field(..., description="Number of measurements that failed for this URL in the past 31 days", example=10) + total_count: int = Field(..., description="Total number of measurements for this URL in the past 31 days", example=100) class PaginationMetadata(BaseModel): - offset: int - limit: int - current_page: int - total_count: int - next_url: Optional[AnyUrl] = None + offset: int = Field(..., description="Current result offset", example=0) + limit: int = Field(..., description="Maximum number of results returned", example=10) + current_page: int = Field(..., description="Current page number (1-based)", example=1) + total_count: int = Field(..., description="Total number of matching URLs", example=123) + next_url: Optional[AnyUrl] = Field(None, description="URL for the next page of results, or null if none", example="https://example.com/api/_/website_urls?limit=10&offset=10") class WebsiteURLsResponse(BaseModel): - metadata: PaginationMetadata - results: List[WebsiteURLItem] + metadata: PaginationMetadata = Field(..., description="Pagination metadata for the results") + results: List[WebsiteURLItem] = Field(..., description="List of URL statistics for the requested CC/ASN") @router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) @@ -471,12 +471,7 @@ def api_private_website_test_urls( limit: int = Query(10, description="Limit results"), offset: int = Query(0, description="Offset results") ) -> WebsiteURLsResponse: - """TODO - --- - responses: - '200': - description: TODO - """ + """Paginated list of tested URLs with per-URL counts (anomalies, confirmations, failures, totals) for the past 31 days.""" # TODO optimize or remove if limit <= 0: limit = 10 From b4aebffdfc29337c81fb206a1eeb727543a2c0bf Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:19:37 +0200 Subject: [PATCH 077/105] add response model validation for /api/_/vanilla_tor_stats --- .../src/ooniprobe/routers/private.py | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ad72ba43b..cbe91f21b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -545,20 +545,20 @@ def api_private_website_test_urls( class NetworkStat(BaseModel): - failure_count: int - last_tested: Optional[date] # SQL toDate() returns a date - probe_asn: int - total_count: int - success_count: int - test_runtime_avg: Optional[float] = None - test_runtime_max: Optional[float] = None - test_runtime_min: Optional[float] = None + failure_count: int = Field(..., description="Number of failed measurements for this ASN", example=3) + last_tested: Optional[date] = Field(None, description="Date of the most recent measurement (YYYY-MM-DD)", example="2026-03-29") + probe_asn: int = Field(..., description="Autonomous System Number (integer)", example=12345) + total_count: int = Field(..., description="Total number of measurements for this ASN", example=100) + success_count: int = Field(..., description="Number of successful measurements for this ASN", example=80) + test_runtime_avg: Optional[float] = Field(None, description="Average test runtime in seconds for this ASN", example=1.23) + test_runtime_max: Optional[float] = Field(None, description="Maximum test runtime in seconds for this ASN", example=2.5) + test_runtime_min: Optional[float] = Field(None, description="Minimum test runtime in seconds for this ASN", example=0.8) class TorStatsResponse(BaseModel): - last_tested: Optional[date] - networks: List[NetworkStat] - notok_networks: int + last_tested: Optional[date] = Field(None, description="Most recent test date across all networks (YYYY-MM-DD)", example="2026-03-29") + networks: List[NetworkStat] = Field(..., description="List of per-ASN Tor test statistics") + notok_networks: int = Field(..., description="Number of networks considered 'not OK' (low success rate)", example=5) @router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) @@ -566,18 +566,7 @@ def api_private_vanilla_tor_stats( clickhouse: ClickhouseDep, probe_cc: str = Query(..., description="Country Code") ) -> TorStatsResponse: - """Tor statistics over ASN for a given CC - --- - parameters: - - name: probe_cc - in: query - type: string - minLength: 2 - required: true - responses: - '200': - description: TODO - """ + """Per-ASN Tor measurement statistics for the given country over the last 6 months, including counts, last-tested date, and a tally of networks with low success rates.""" try: CountryAlpha2(probe_cc) except Exception: From 17514ef3fa93051250fbd062369311112849c208 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:22:40 +0200 Subject: [PATCH 078/105] add response model validation for /api/_/im_networks --- .../src/ooniprobe/routers/private.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index cbe91f21b..1f98f637e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -613,16 +613,16 @@ def api_private_vanilla_tor_stats( class NetworkEntry(BaseModel): - asn: int - name: str - total_count: int - last_tested: date + asn: int = Field(..., description="Autonomous System Number (integer)", example=12345) + name: str = Field(..., description="Network/ASN name", example="Example ISP") + total_count: int = Field(..., description="Total number of measurements for this ASN and test", example=42) + last_tested: date = Field(..., description="Date of the most recent measurement (YYYY-MM-DD)", example="2026-03-29") class IMNetworkStats(BaseModel): - anomaly_networks: List[NetworkEntry] - ok_networks: List[NetworkEntry] - last_tested: date + anomaly_networks: List[NetworkEntry] = Field(..., description="List of networks showing anomalous behaviour for this test") + ok_networks: List[NetworkEntry] = Field(..., description="List of networks considered OK for this test") + last_tested: date = Field(..., description="Most recent measurement date across networks for this test", example="2026-03-29") @router.get("/im_networks", response_model=Dict[str, IMNetworkStats], tags=["private"]) @@ -630,12 +630,7 @@ def api_private_im_networks( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") ) -> Dict[str, IMNetworkStats]: - """Instant messaging networks statistics - --- - responses: - '200': - description: TODO - """ + """Per-test instant messaging network statistics (per-ASN totals and last-tested date) for the past 31 days, keyed by test name.""" s = """SELECT COUNT() AS total_count, '' AS name, From 5fe42838990b58f3f55d3fda0f8c3d15b0c3e2ce Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:47:36 +0200 Subject: [PATCH 079/105] add response model descriptions for /api/_/im_stats --- .../ooniprobe/src/ooniprobe/routers/private.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1f98f637e..e0f9d5b63 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -674,16 +674,16 @@ def isomid(d) -> str: class IMStatsItem(BaseModel): - anomaly_count: Optional[int] = None - test_day: datetime - total_count: int + anomaly_count: Optional[int] = Field(None, description="Number of measurements flagged as anomalies for that day") + test_day: datetime = Field(..., description="Timestamp for the day (ISO 8601, midnight UTC)", example="2020-08-01T00:00:00+00:00") + total_count: int = Field(..., description="Total number of measurements for that day", example=42) model_config = { "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } } class IMStatsResponse(BaseModel): - results: List[IMStatsItem] + results: List[IMStatsItem] = Field(..., description="Daily IM statistics for the requested ASN/CC/test (last 31 days)") @router.get("/im_stats", response_model=IMStatsResponse, tags=["private"]) @@ -693,12 +693,7 @@ def api_private_im_stats( probe_cc: CountryAlpha2 = Query(..., description="Country Code"), test_name: str = Query(..., description="Test name") ) -> IMStatsResponse: - """Instant messaging statistics - --- - responses: - '200': - description: TODO - """ + """Daily instant messaging measurement totals (and optional anomaly counts) for the past 31 days, for the given ASN, country, and test.""" test_names = ["facebook_messenger", "signal", "telegram", "whatsapp"] if test_name not in test_names: raise HTTPException(status_code=400, detail="Invalid test_name") From d3cbbfc37a627c7a47f11fe101cc33a8ed6d55bd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:50:06 +0200 Subject: [PATCH 080/105] add model description, pagination query parameters for /api/_/network_stats this method isn't yet implemented as the necessary database tables are nonexistent --- .../ooniprobe/src/ooniprobe/routers/private.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index e0f9d5b63..45874a8a5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -760,18 +760,11 @@ class NetworkStatsResponse(BaseModel): @router.get("/network_stats", response_model=NetworkStatsResponse, tags=["private"]) def api_private_network_stats( probe_cc: CountryAlpha2 = Query(..., description="Country Code"), + limit: int = Query(10, description="Limit results"), + offset: int = Query(0, description="Offset results"), + clickhouse: ClickhouseDep, ) -> NetworkStatsResponse: - """Network speed statistics - not implemented - --- - parameters: - - name: probe_cc - in: query - type: string - description: The two letter country code - responses: - '200': - description: TODO - """ + # TODO: implement the stats from NDT in fastpath and then here return NetworkStatsResponse(metadata=NetworkMetadata(), results=[]) From f84f05197e45e8bfbdfea4dc56b5caa570900355 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:54:52 +0200 Subject: [PATCH 081/105] let FastAPI generate response type from model --- .../ooniprobe/src/ooniprobe/routers/private.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 45874a8a5..9ab264047 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -781,15 +781,7 @@ def api_private_country_overview( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), ) -> CountryOverviewResponse: - """Country-specific overview - --- - responses: - '200': - description: { - "first_bucket_date":"2012-12-01", - "measurement_count":6659891, - "network_count":333} - """ + """Country-level summary for the requested two-letter code: first available measurement date, total number of measurements since 2012-12-01, and number of distinct ASNs observed (networks).""" # TODO: add circumvention_tools_blocked im_apps_blocked # middlebox_detected_networks websites_confirmed_blocked s = """SELECT @@ -819,13 +811,7 @@ class GlobalOverviewResponse(BaseModel): def api_private_global_overview( clickhouse: ClickhouseDep, ) -> GlobalOverviewResponse: - """Provide global summary of measurements - Sources: global_stats db table - --- - responses: - '200': - description: JSON struct TODO - """ + """Global summary of measurements across all countries: total distinct networks (ASNs), total countries with measurements, and total measurement count (computed from the fastpath table).""" q = """SELECT COUNT(DISTINCT(probe_asn)) AS network_count, COUNT(DISTINCT probe_cc) AS country_count, From 553cc9a7c095f033624b73ceeb0d2b40afd02279 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:56:05 +0200 Subject: [PATCH 082/105] add Field descriptions for FastAPI docs for /api/_/global_overview_by_month --- .../ooniprobe/src/ooniprobe/routers/private.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 9ab264047..4f3b1b650 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -828,30 +828,24 @@ def api_private_global_overview( class GlobalOverviewStat(BaseModel): - date: datetime - value: int + date: datetime = Field(..., description="Month start timestamp (ISO 8601, midnight UTC)", example="2026-03-01T00:00:00+00:00") + value: int = Field(..., description="Count value for the month", example=12345) model_config = { "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } } class GlobalOverviewMonthResponse(BaseModel): - networks_by_month: List[GlobalOverviewStat] - countries_by_month: List[GlobalOverviewStat] - measurements_by_month: List[GlobalOverviewStat] + networks_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly distinct network (ASN) counts") + countries_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly distinct country counts") + measurements_by_month: List[GlobalOverviewStat] = Field(..., description="Monthly total measurement counts") @router.get("/global_overview_by_month", response_model=GlobalOverviewMonthResponse, tags=["private"]) def api_private_global_by_month( clickhouse: ClickhouseDep, ) -> GlobalOverviewMonthResponse: - """Provide global summary of measurements - Sources: global_by_month db table - --- - responses: - '200': - description: JSON struct TODO - """ + """Monthly global time series for the last two years: distinct networks (ASNs), distinct countries, and total measurements per month (month timestamps are start-of-month).""" q = """SELECT COUNT(DISTINCT probe_asn) AS networks_by_month, COUNT(DISTINCT probe_cc) AS countries_by_month, From df5fef661ea6253f942ecac89b4bbefc0536d178 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 16:57:26 +0200 Subject: [PATCH 083/105] let FastAPI generate these response types docs for /api/_/circumvention_stats_by_country, /api/_/circumvention_runtime_stats, /api/_/asnmeta --- .../src/ooniprobe/routers/private.py | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 4f3b1b650..bab74c3b0 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -883,13 +883,7 @@ class CircumventionStatsResponse(BaseModel): def api_private_circumvention_stats_by_country( clickhouse: ClickhouseDep, ) -> CircumventionStatsResponse: - """Aggregated statistics on protocols used for circumvention, - grouped by country. - --- - responses: - 200: - description: List of dicts with keys probe_cc and cnt - """ + """Aggregated statistics on protocols used for circumvention, grouped by country. """ q = """SELECT probe_cc, COUNT(*) as cnt FROM fastpath WHERE measurement_start_time > today() - interval 6 month @@ -945,13 +939,7 @@ class CircumventionRuntimeStatsResponse(BaseModel): def api_private_circumvention_runtime_stats( clickhouse: ClickhouseDep, ) -> CircumventionRuntimeStatsResponse: - """Runtime statistics on protocols used for circumvention, - grouped by date, country, test_name. - --- - responses: - 200: - description: List of dicts with keys probe_cc and cnt - """ + """Runtime statistics on protocols used for circumvention, grouped by date, country, test_name. """ q = """SELECT toDate(measurement_start_time) AS date, test_name, @@ -1008,12 +996,6 @@ def api_private_domain_metadata( notes were taken on what fixes need to be done in the test-lists to ensure all of this works as expected (ex. moving shortest URL representations from the country lists into the global list). - - Returns: - { - "category_code": "CITIZENLAB_CATEGORY_CODE", - "canonical_domain": "canonical.tld" - } """ category_code = "MISC" @@ -1061,18 +1043,7 @@ def api_private_asnmeta( clickhouse: ClickhouseDep, asn: int = Query(..., description="Autonomous System Number, e.g. 1234"), ) -> ASNMetadataResponse: - """Look up organization name by ASN - Sources: ansmeta db table - --- - parameters: - - name: asn - in: query - type: string - description: ASN - responses: - 200: - description: JSON object - """ + """Look up organization name by ASN""" q = """SELECT org_name FROM asnmeta From c8fcf14ec9a8c66beef5ac497312ce991962625d Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 17:04:36 +0200 Subject: [PATCH 084/105] omint conint type; let FastAPI generate api response types +from pydantic import AnyUrl, Field for /api/_/networks, /api/_/domains --- .../src/ooniprobe/routers/private.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index bab74c3b0..bcb252624 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -18,7 +18,7 @@ from fastapi import APIRouter, Depends, Header, Request, Response, Query from pydantic_extra_types.country import CountryAlpha2 -from pydantic import AnyUrl, conint, Field +from pydantic import AnyUrl, Field from .v1.probe_services import probe_geoip, generate_test_helpers_conf from ..common.clickhouse_utils import query_click, query_click_one_row @@ -870,7 +870,7 @@ def api_private_global_by_month( class CountryCircumventionStat(BaseModel): - cnt: conint(ge=0) = Field(..., description="Count of measurements") + cnt: int = Field(..., description="Count of measurements") probe_cc: CountryAlpha2 = Field(..., description="Country code of probe") @@ -1057,9 +1057,9 @@ def api_private_asnmeta( class MeasuredNetworkStat(BaseModel): - cnt: conint(ge=0) = Field(..., description="Number of measurements") - org_name: str = Field("", description="ORG Name of network") - probe_asn: conint(ge=0) = Field(..., description="ASN of network (int)") + cnt: int = Field(..., description="Number of measurements", example=123) + org_name: str = Field("", description="Organization name associated with the ASN", example="Example ISP") + probe_asn: int = Field(..., description="ASN of network (int)") class MeasuredNetworksResponse(BaseModel): @@ -1071,12 +1071,7 @@ class MeasuredNetworksResponse(BaseModel): def api_private_networks( clickhouse: ClickhouseDep, ) -> MeasuredNetworksResponse: - """List all networks that have measurements - --- - responses: - 200: - description: JSON object - """ + """List all networks that have measurements by per-ASN measurement count and associated organization name.""" q = """ SELECT probe_asn, cnt, org_name FROM ( SELECT @@ -1121,12 +1116,7 @@ class DomainsMeasuredResponse(BaseModel): def api_private_domains( clickhouse: ClickhouseDep, ) -> DomainsMeasuredResponse: - """List all the domains in the test-lists with their measurement count - --- - responses: - 200: - description: JSON object - """ + """List all the domains in the test-lists with their measurement count.""" # The nested ORDER BY lower(cc) puts global entries (cc=ZZ) on top so that # any(category_code) picks it up as the most meaningful category code. q = """ From 9157dcc522ef935af7424d11b3d2d64397ca392c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 29 Apr 2026 17:19:10 +0200 Subject: [PATCH 085/105] remove deprecated Field example... --- .../src/ooniprobe/routers/private.py | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index bcb252624..a4a84a2ec 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -337,14 +337,14 @@ def get_recent_network_coverage_ch(clickhouse, probe_cc, test_groups): class NetworkCoveragePoint(BaseModel): - test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)", example="2021-10-16") - count: int = Field(..., description="Count of unique ASNs seen that day", example=58) + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)") + count: int = Field(..., description="Count of unique ASNs seen that day") class TestCoveragePoint(BaseModel): - test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)", example="2021-10-16") - test_group: str = Field(..., description="Test group name", example="websites") - count: int = Field(..., description="Number of measurements for this test group on that day", example=4888) + test_day: date = Field(..., description="Date for the measurement (YYYY-MM-DD)") + test_group: str = Field(..., description="Test group name") + count: int = Field(..., description="Number of measurements for this test group on that day") class TestCoverageResponse(BaseModel): @@ -356,7 +356,7 @@ class TestCoverageResponse(BaseModel): def api_private_test_coverage( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), - test_groups: str = Query(None, description="Comma-separated list of test group keys to filter results", example="websites,im") + test_groups: str = Query(None, description="Comma-separated list of test group keys to filter results") ) -> TestCoverageResponse: """Return number of measurements per day across test categories """ @@ -399,11 +399,11 @@ def api_private_website_network_tests( class DayStats(BaseModel): - test_day: date = Field(..., description="Date for the aggregated counts (YYYY-MM-DD)", example="2026-03-29") - anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies on this day", example=5) - confirmed_count: int = Field(..., description="Number of anomalies confirmed on this day", example=2) - failure_count: int = Field(..., description="Number of measurements that failed on this day", example=10) - total_count: int = Field(..., description="Total number of measurements for this day", example=100) + test_day: date = Field(..., description="Date for the aggregated counts (YYYY-MM-DD)") + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies on this day") + confirmed_count: int = Field(..., description="Number of anomalies confirmed on this day") + failure_count: int = Field(..., description="Number of measurements that failed on this day") + total_count: int = Field(..., description="Total number of measurements for this day") class WebsiteStatsResponse(BaseModel): @@ -444,18 +444,18 @@ def api_private_website_stats( class WebsiteURLItem(BaseModel): input: AnyUrl = Field(..., description="Tested URL") - anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies for this URL in the past 31 days", example=5) - confirmed_count: int = Field(..., description="Number of anomalies confirmed for this URL in the past 31 days", example=2) - failure_count: int = Field(..., description="Number of measurements that failed for this URL in the past 31 days", example=10) - total_count: int = Field(..., description="Total number of measurements for this URL in the past 31 days", example=100) + anomaly_count: int = Field(..., description="Number of measurements flagged as anomalies for this URL in the past 31 days") + confirmed_count: int = Field(..., description="Number of anomalies confirmed for this URL in the past 31 days") + failure_count: int = Field(..., description="Number of measurements that failed for this URL in the past 31 days") + total_count: int = Field(..., description="Total number of measurements for this URL in the past 31 days") class PaginationMetadata(BaseModel): - offset: int = Field(..., description="Current result offset", example=0) - limit: int = Field(..., description="Maximum number of results returned", example=10) - current_page: int = Field(..., description="Current page number (1-based)", example=1) - total_count: int = Field(..., description="Total number of matching URLs", example=123) - next_url: Optional[AnyUrl] = Field(None, description="URL for the next page of results, or null if none", example="https://example.com/api/_/website_urls?limit=10&offset=10") + offset: int = Field(..., description="Current result offset") + limit: int = Field(..., description="Maximum number of results returned") + current_page: int = Field(..., description="Current page number (1-based)") + total_count: int = Field(..., description="Total number of matching URLs") + next_url: Optional[AnyUrl] = Field(None, description="URL for the next page of results, or null if none") class WebsiteURLsResponse(BaseModel): @@ -545,20 +545,20 @@ def api_private_website_test_urls( class NetworkStat(BaseModel): - failure_count: int = Field(..., description="Number of failed measurements for this ASN", example=3) - last_tested: Optional[date] = Field(None, description="Date of the most recent measurement (YYYY-MM-DD)", example="2026-03-29") - probe_asn: int = Field(..., description="Autonomous System Number (integer)", example=12345) - total_count: int = Field(..., description="Total number of measurements for this ASN", example=100) - success_count: int = Field(..., description="Number of successful measurements for this ASN", example=80) - test_runtime_avg: Optional[float] = Field(None, description="Average test runtime in seconds for this ASN", example=1.23) - test_runtime_max: Optional[float] = Field(None, description="Maximum test runtime in seconds for this ASN", example=2.5) - test_runtime_min: Optional[float] = Field(None, description="Minimum test runtime in seconds for this ASN", example=0.8) + failure_count: int = Field(..., description="Number of failed measurements for this ASN") + last_tested: Optional[date] = Field(None, description="Date of the most recent measurement (YYYY-MM-DD)") + probe_asn: int = Field(..., description="Autonomous System Number (integer)") + total_count: int = Field(..., description="Total number of measurements for this ASN") + success_count: int = Field(..., description="Number of successful measurements for this ASN") + test_runtime_avg: Optional[float] = Field(None, description="Average test runtime in seconds for this ASN") + test_runtime_max: Optional[float] = Field(None, description="Maximum test runtime in seconds for this ASN") + test_runtime_min: Optional[float] = Field(None, description="Minimum test runtime in seconds for this ASN") class TorStatsResponse(BaseModel): - last_tested: Optional[date] = Field(None, description="Most recent test date across all networks (YYYY-MM-DD)", example="2026-03-29") + last_tested: Optional[date] = Field(None, description="Most recent test date across all networks (YYYY-MM-DD)") networks: List[NetworkStat] = Field(..., description="List of per-ASN Tor test statistics") - notok_networks: int = Field(..., description="Number of networks considered 'not OK' (low success rate)", example=5) + notok_networks: int = Field(..., description="Number of networks considered 'not OK' (low success rate)") @router.get("/vanilla_tor_stats", response_model=TorStatsResponse, tags=["private"]) @@ -613,16 +613,16 @@ def api_private_vanilla_tor_stats( class NetworkEntry(BaseModel): - asn: int = Field(..., description="Autonomous System Number (integer)", example=12345) - name: str = Field(..., description="Network/ASN name", example="Example ISP") - total_count: int = Field(..., description="Total number of measurements for this ASN and test", example=42) - last_tested: date = Field(..., description="Date of the most recent measurement (YYYY-MM-DD)", example="2026-03-29") + asn: int = Field(..., description="Autonomous System Number (integer)") + name: str = Field(..., description="Network/ASN name") + total_count: int = Field(..., description="Total number of measurements for this ASN and test") + last_tested: date = Field(..., description="Date of the most recent measurement (YYYY-MM-DD)") class IMNetworkStats(BaseModel): anomaly_networks: List[NetworkEntry] = Field(..., description="List of networks showing anomalous behaviour for this test") ok_networks: List[NetworkEntry] = Field(..., description="List of networks considered OK for this test") - last_tested: date = Field(..., description="Most recent measurement date across networks for this test", example="2026-03-29") + last_tested: date = Field(..., description="Most recent measurement date across networks for this test") @router.get("/im_networks", response_model=Dict[str, IMNetworkStats], tags=["private"]) @@ -675,8 +675,8 @@ def isomid(d) -> str: class IMStatsItem(BaseModel): anomaly_count: Optional[int] = Field(None, description="Number of measurements flagged as anomalies for that day") - test_day: datetime = Field(..., description="Timestamp for the day (ISO 8601, midnight UTC)", example="2020-08-01T00:00:00+00:00") - total_count: int = Field(..., description="Total number of measurements for that day", example=42) + test_day: datetime = Field(..., description="Timestamp for the day (ISO 8601, midnight UTC)") + total_count: int = Field(..., description="Total number of measurements for that day") model_config = { "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } } @@ -753,8 +753,8 @@ class NetworkMetadata(BaseModel): class NetworkStatsResponse(BaseModel): - metadata: NetworkMetadata = Field(..., description="Networks metadata") - results: List[NetworkStats] = Field(..., description="List of network stats") + metadata: NetworkMetadata = Field(..., description="Pagination and result metadata") + results: List[NetworkStats] = Field(..., description="List of per-ASN network statistics") @router.get("/network_stats", response_model=NetworkStatsResponse, tags=["private"]) @@ -828,8 +828,8 @@ def api_private_global_overview( class GlobalOverviewStat(BaseModel): - date: datetime = Field(..., description="Month start timestamp (ISO 8601, midnight UTC)", example="2026-03-01T00:00:00+00:00") - value: int = Field(..., description="Count value for the month", example=12345) + date: datetime = Field(..., description="Month start timestamp (ISO 8601, midnight UTC)") + value: int = Field(..., description="Count value for the month") model_config = { "json_encoders": { datetime: lambda dt: dt.astimezone(timezone.utc).replace(microsecond=0).isoformat() } } @@ -1057,8 +1057,8 @@ def api_private_asnmeta( class MeasuredNetworkStat(BaseModel): - cnt: int = Field(..., description="Number of measurements", example=123) - org_name: str = Field("", description="Organization name associated with the ASN", example="Example ISP") + cnt: int = Field(..., description="Number of measurements") + org_name: str = Field("", description="Organization name associated with the ASN") probe_asn: int = Field(..., description="ASN of network (int)") From 8db3e45d33f313513b53ceeb596fcca1a098e25c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 30 Apr 2026 07:40:35 +0200 Subject: [PATCH 086/105] fix default arg order --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a4a84a2ec..483a600cb 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -759,10 +759,10 @@ class NetworkStatsResponse(BaseModel): @router.get("/network_stats", response_model=NetworkStatsResponse, tags=["private"]) def api_private_network_stats( + clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), limit: int = Query(10, description="Limit results"), offset: int = Query(0, description="Offset results"), - clickhouse: ClickhouseDep, ) -> NetworkStatsResponse: # TODO: implement the stats from NDT in fastpath and then here From f7dac659041d3858b0053cfb90f839a064f6623b Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:23:09 +0200 Subject: [PATCH 087/105] fix whitespace. bump codedeploy --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 483a600cb..a34dd478f 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -116,7 +116,6 @@ class CountryCount(BaseModel): } - @router.get("/countries_by_month", tags=["private"], response_model=List[CountryCount]) def api_private_countries_by_month( clickhouse: ClickhouseDep, From 70a672ffb1db9eb576308364fb2a45863bcfdc29 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:42:38 +0200 Subject: [PATCH 088/105] fix arg to WebsiteStatsResponse --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index a34dd478f..1c6271f97 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -438,7 +438,7 @@ def api_private_website_stats( """ d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} results = query_click(clickhouse, sql.text(s), d) - return WebsiteStatsResponse(result=results) + return WebsiteStatsResponse(results=results) class WebsiteURLItem(BaseModel): From 9494718a839364fd09854b2b7cfc2178ff77179c Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:43:34 +0200 Subject: [PATCH 089/105] get base_url from request --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 1c6271f97..5e94a78e1 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -464,6 +464,7 @@ class WebsiteURLsResponse(BaseModel): @router.get("/website_urls", response_model=WebsiteURLsResponse, tags=["private"]) def api_private_website_test_urls( + request: Request, clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code"), probe_asn: str = Query(..., description="ASN, e.g. AS1234"), @@ -534,8 +535,8 @@ def api_private_website_test_urls( ) # TODO: remove BASE_URL? next_url = urljoin( - current_app.config["BASE_URL"], - "/api/_/website_urls?%s" % urlencode(args), + request.base_url.rstrip("/"), + f"/api/_/website_urls?{urlencode(args)}", ) metadata["next_url"] = next_url From 4e92c3a462842b54d34ac348ff90c5b5eef021bb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:44:15 +0200 Subject: [PATCH 090/105] check for last_tested is None --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5e94a78e1..871a2fd74 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -659,7 +659,7 @@ def api_private_im_networks( # XXX: anomaly_networks appears unused # update last_tested if it is the latest measurement - if stats.last_tested < entry.last_tested: + if stats.last_tested is None or stats.last_tested < entry.last_tested: stats.last_tested = entry.last_tested # save the object in results From 637c8a487da9140d91bd96799af6392e3fd40f5e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:44:35 +0200 Subject: [PATCH 091/105] add XXX - api doesn't do what docstring says --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 871a2fd74..21f14756c 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -700,6 +700,8 @@ def api_private_im_stats( probe_asn = int(probe_asn.upper().replace("AS", "")) + # XXX: this method never queries anomaly_count and always returns anomaly_count=None. Why? + s = """SELECT COUNT() as total_count, toDate(measurement_start_time) AS test_day From 2dfe82e6327199be51b0dd1ac665e01612b8fb67 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:44:58 +0200 Subject: [PATCH 092/105] fix incorrect model name --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 21f14756c..02d2a649e 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -895,7 +895,7 @@ def api_private_circumvention_stats_by_country( """ try: result = query_click(clickhouse, sql.text(q), {}) - return CountryCircumVentionStatsResponse(results=result) + return CircumventionStatsResponse(results=result) except Exception as e: raise HTTPException(status_code=400, detail={"error": str(e), "v": 0}) @@ -926,7 +926,7 @@ def pivot_circumvention_runtime_stats(rows) -> List[CircumventionRuntimeStat]: test_names = sorted(test_names) no_data = () result = [ - RuntimeStat(test_name=k[0], probe_cc=k[1], date=k[2], v=tmp.get(k, no_data)) + CircumventionRuntimeStat(test_name=k[0], probe_cc=k[1], date=k[2], v=tmp.get(k, no_data)) for k in product(test_names, ccs, dates) ] return result From 3134222ae086e20f34fd2846f80c9a298277fb8f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 10:47:50 +0200 Subject: [PATCH 093/105] fix model field name metric_date --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 02d2a649e..b05ed57ec 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -926,7 +926,7 @@ def pivot_circumvention_runtime_stats(rows) -> List[CircumventionRuntimeStat]: test_names = sorted(test_names) no_data = () result = [ - CircumventionRuntimeStat(test_name=k[0], probe_cc=k[1], date=k[2], v=tmp.get(k, no_data)) + CircumventionRuntimeStat(test_name=k[0], probe_cc=k[1], metric_date=k[2], v=tmp.get(k, no_data)) for k in product(test_names, ccs, dates) ] return result From cb7ad9054e8827e8acef59c26550509f86f5efa9 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:23:45 +0200 Subject: [PATCH 094/105] fix typo --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index b05ed57ec..fede567a4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -371,7 +371,7 @@ def api_private_test_coverage( class MeasurementsByASN(BaseModel): - count: int = Field(..., description="Number of measurments for eachnetworks in country") + count: int = Field(..., description="Number of measurements for each network in country") probe_asn: str = Field(..., description="Autonomous System Number") From 353a98fd4de175ae971dc7942ba6a61cf0cb36d5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:24:20 +0200 Subject: [PATCH 095/105] fix /api/_/website_networks response type response needs to have a "results" key --- .../services/ooniprobe/src/ooniprobe/routers/private.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index fede567a4..9f64d1c51 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -375,11 +375,15 @@ class MeasurementsByASN(BaseModel): probe_asn: str = Field(..., description="Autonomous System Number") +class WebsiteNetworksResponse(BaseModel): + results: List[MeasurementsByASN] = Field(..., description="List of number of measurements per ASN") + + @router.get("/website_networks", response_model=List[MeasurementsByASN], tags=["private"]) def api_private_website_network_tests( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") -) -> List[MeasurementsByASN]: + ) -> WebsiteNetworksResponse: """Daily counts of website measurements per ASN for the past 31 days, returned as a list of (probe_asn, count) ordered by count descending.""" s = """SELECT COUNT() AS count, @@ -393,7 +397,7 @@ def api_private_website_network_tests( ORDER BY count DESC """ results = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) - validated: List[MeasurementsByASN] = [MeasurementsByASN(**x) for x in results] + validated: WebsiteNetworksResponse(results=[MeasurementsByASN(**x) for x in results]) return validated From 7b2f363492c306eefbe71368e50a044f845c50ac Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:26:04 +0200 Subject: [PATCH 096/105] fix response_model --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 9f64d1c51..5043d2dd4 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -379,7 +379,7 @@ class WebsiteNetworksResponse(BaseModel): results: List[MeasurementsByASN] = Field(..., description="List of number of measurements per ASN") -@router.get("/website_networks", response_model=List[MeasurementsByASN], tags=["private"]) +@router.get("/website_networks", response_model=WebsiteNetworksResponse, tags=["private"]) def api_private_website_network_tests( clickhouse: ClickhouseDep, probe_cc: CountryAlpha2 = Query(..., description="Country Code") From 5eedd1e6d56fde0e1c40870189546a9f72064aeb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:30:44 +0200 Subject: [PATCH 097/105] fix UnboundLocalError --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5043d2dd4..48c0610d0 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -397,7 +397,7 @@ def api_private_website_network_tests( ORDER BY count DESC """ results = query_click(clickhouse, sql.text(s), {"probe_cc": probe_cc}) - validated: WebsiteNetworksResponse(results=[MeasurementsByASN(**x) for x in results]) + validated = WebsiteNetworksResponse(results=[MeasurementsByASN(**x) for x in results]) return validated From f7ec4e0aa294c84cd85e2263e88b1817fa162f81 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:34:34 +0200 Subject: [PATCH 098/105] debug: log GeoLookupRequest --- .../ooniprobe/src/ooniprobe/routers/v1/probe_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 2e9b99738..ae3f18569 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -704,6 +704,7 @@ async def geolookup( ) -> GeoLookupResponse: geolocation = dict() + log.debug(f"GeoLookupRequest: {data}") # for each address provided, call probe_geoip and add the data to our response for ipaddr in data.addresses: try: From b5ba65333f74944ca2d75d2e57f23d0d5d028010 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 12:37:58 +0200 Subject: [PATCH 099/105] website_networks: fix response type --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 48c0610d0..5eede4103 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -372,7 +372,7 @@ def api_private_test_coverage( class MeasurementsByASN(BaseModel): count: int = Field(..., description="Number of measurements for each network in country") - probe_asn: str = Field(..., description="Autonomous System Number") + probe_asn: int = Field(..., description="Autonomous System Number") class WebsiteNetworksResponse(BaseModel): From f95ed6de811dd02635d13bcfaded0f1074c68158 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:01:23 +0200 Subject: [PATCH 100/105] fix website_stats query --- .../ooniprobe/src/ooniprobe/routers/private.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 5eede4103..01dc8ba1c 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -428,14 +428,15 @@ def api_private_website_stats( s = """SELECT toDate(measurement_start_time) AS test_day, - countIf(anomaly = 't') as anomaly_count, - countIf(confirmed = 't') as confirmed_count, - countIf(msm_failure = 't') as failure_count, - COUNT() AS total_count + countIf(anomaly = 't') AS anomaly_count, + countIf(confirmed = 't') AS confirmed_count, + countIf(msm_failure = 't') AS failure_count, + count() AS total_count FROM fastpath - WHERE measurement_start_time >= today() - interval '31 day' + WHERE + measurement_start_time >= (today() - INTERVAL 31 DAY) AND measurement_start_time < today() - AND probe_cc = :probe_cc + AND probe_cc = :probe_cc AND probe_asn = :probe_asn AND input = :input GROUP BY test_day ORDER BY test_day From 2896901ecac472ead566c68294e217a0fc2e7564 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:09:07 +0200 Subject: [PATCH 101/105] fix GROUP BY for clickhouse (cannot refer to test_day) --- .../ooniprobe/src/ooniprobe/routers/private.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 01dc8ba1c..7c8c1aaf7 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -426,20 +426,24 @@ def api_private_website_stats( # made queries go from full scan to 50ms url = input - s = """SELECT + s = """ + SELECT toDate(measurement_start_time) AS test_day, countIf(anomaly = 't') AS anomaly_count, countIf(confirmed = 't') AS confirmed_count, countIf(msm_failure = 't') AS failure_count, count() AS total_count - FROM fastpath - WHERE + FROM fastpath + WHERE measurement_start_time >= (today() - INTERVAL 31 DAY) AND measurement_start_time < today() AND probe_cc = :probe_cc AND probe_asn = :probe_asn AND input = :input - GROUP BY test_day ORDER BY test_day + GROUP BY + toDate(measurement_start_time) + ORDER BY + test_day """ d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} results = query_click(clickhouse, sql.text(s), d) From f567ae625202ddfa877f091244c78303e84d7e29 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:15:47 +0200 Subject: [PATCH 102/105] fix ORDER BY also --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 7c8c1aaf7..46d0cca1a 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -440,10 +440,8 @@ def api_private_website_stats( AND probe_cc = :probe_cc AND probe_asn = :probe_asn AND input = :input - GROUP BY - toDate(measurement_start_time) - ORDER BY - test_day + GROUP BY toDate(measurement_start_time) + ORDER BY toDate(measurement_start_time) """ d = {"probe_cc": probe_cc, "probe_asn": probe_asn, "input": url} results = query_click(clickhouse, sql.text(s), d) From 5494a7576b4a136c03ce35b3dec4d52cb2449362 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:27:45 +0200 Subject: [PATCH 103/105] fix quote input --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 46d0cca1a..18180559b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -437,9 +437,9 @@ def api_private_website_stats( WHERE measurement_start_time >= (today() - INTERVAL 31 DAY) AND measurement_start_time < today() - AND probe_cc = :probe_cc - AND probe_asn = :probe_asn - AND input = :input + AND probe_cc = %(probe_cc)s + AND probe_asn = %(probe_asn)s + AND input = %(input)s GROUP BY toDate(measurement_start_time) ORDER BY toDate(measurement_start_time) """ From 94ef0406a99cd0486304147961ef8354ee539d87 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:43:56 +0200 Subject: [PATCH 104/105] fix query parameters --- .../services/ooniprobe/src/ooniprobe/routers/private.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index 18180559b..ea685b105 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -424,8 +424,7 @@ def api_private_website_stats( # uses_pg_index counters_day_cc_asn_input_idx a BRIN index was not used at # all, but BTREE on (measurement_start_day, probe_cc, probe_asn, input) # made queries go from full scan to 50ms - url = input - + url = str(input) s = """ SELECT toDate(measurement_start_time) AS test_day, @@ -437,9 +436,9 @@ def api_private_website_stats( WHERE measurement_start_time >= (today() - INTERVAL 31 DAY) AND measurement_start_time < today() - AND probe_cc = %(probe_cc)s - AND probe_asn = %(probe_asn)s - AND input = %(input)s + AND probe_cc = {probe_cc:String} + AND probe_asn = {probe_asn:Int64} + AND input = {input:String} GROUP BY toDate(measurement_start_time) ORDER BY toDate(measurement_start_time) """ From b0ee043e70304d13c6afe04cad852ed09c490ff1 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 7 Jul 2026 13:52:20 +0200 Subject: [PATCH 105/105] not valid input --- ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py index ea685b105..2573aec8b 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/private.py @@ -436,9 +436,9 @@ def api_private_website_stats( WHERE measurement_start_time >= (today() - INTERVAL 31 DAY) AND measurement_start_time < today() - AND probe_cc = {probe_cc:String} - AND probe_asn = {probe_asn:Int64} - AND input = {input:String} + AND probe_cc = :probe_cc + AND probe_asn = :probe_asn + AND input = :input GROUP BY toDate(measurement_start_time) ORDER BY toDate(measurement_start_time) """