From b22e25e5a8dcda9bc29ae62d2ab0033a2da74855 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 16:36:00 +0200 Subject: [PATCH 01/34] Extract replay data to request struct --- lib/plausible/ingestion/request.ex | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index e8b15a97f152..853ad899e3bf 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -59,6 +59,10 @@ defmodule Plausible.Ingestion.Request do on_ee do field :revenue_source, :map + + # fields meant for replayed events only + field :replay_id, :string + field :replay_session_id, :string end field :query_params, :map @@ -91,6 +95,7 @@ defmodule Plausible.Ingestion.Request do |> put_uri(request_body) |> put_hostname() |> put_user_agent(conn) + |> put_replay_data(conn) |> put_request_params(request_body) |> put_referrer(request_body) |> put_pathname() @@ -131,6 +136,41 @@ defmodule Plausible.Ingestion.Request do defp put_revenue_source(changeset, _request_body), do: changeset end + on_ee do + @replay_id_header "x-replay-event-id" + @replay_session_id_header "x-replay-session-id" + @replay_time_header "x-replay-time" + + defp put_replay_data(changeset, conn) do + id = + conn + |> Plug.Conn.get_req_header(@replay_id_header) + |> List.first() + + if id do + session_id = + conn + |> Plug.Conn.get_req_header(@replay_session_id_header) + |> List.first() + + time = + conn + |> Plug.Conn.get_req_header(@replay_time_header) + |> List.first() + |> NaiveDateTime.from_iso8601!() + + changeset + |> Changeset.put_change(:replay_id, id) + |> Changeset.put_change(:replay_session_id, session_id) + |> Changeset.put_change(:timestamp, time) + else + changeset + end + end + else + defp put_replay_data(changeset, _conn), do: changeset + end + defp put_remote_ip(changeset, conn) do Changeset.put_change(changeset, :remote_ip, PlausibleWeb.RemoteIP.get(conn)) end From 53818b49c06aea1972b4b6b100d3193f9667b2fa Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 19:43:10 +0200 Subject: [PATCH 02/34] Add virtual `replay_session_id` field to CH event and session structs --- lib/plausible/clickhouse_event_v2.ex | 6 +++++- lib/plausible/clickhouse_session_v2.ex | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 7e2d8861a22f..79b2b2571176 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -50,6 +50,9 @@ defmodule Plausible.ClickhouseEventV2 do # Virtual field used during event processing field :interactive?, :boolean, default: true, virtual: true, writable: :never + + # Virtual field used during event replay + field :replay_session_id, :string, default: nil, virtual: true, writable: :never end def new(attrs) do @@ -71,7 +74,8 @@ defmodule Plausible.ClickhouseEventV2 do :revenue_source_currency, :revenue_reporting_amount, :revenue_reporting_currency, - :interactive? + :interactive?, + :replay_session_id ] ) |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp]) diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex index e9614df5b9bb..e7fc405b96b2 100644 --- a/lib/plausible/clickhouse_session_v2.ex +++ b/lib/plausible/clickhouse_session_v2.ex @@ -75,6 +75,9 @@ defmodule Plausible.ClickhouseSessionV2 do field :transferred_from, :string field :acquisition_channel, Ch, type: "LowCardinality(String)", writable: :never + + # Virtual field used during event replay + field :replay_session_id, :string, default: nil, virtual: true, writable: :never end def random_uint64() do From 710807bb149e9aaad766503c426757d76912abb1 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 19:44:22 +0200 Subject: [PATCH 03/34] Merge replay_session_id from request into event attrs --- lib/plausible/ingestion/event.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index ee4d99e1f8c8..7a1a7ec1b24b 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -285,7 +285,8 @@ defmodule Plausible.Ingestion.Event do pathname: event.request.pathname, scroll_depth: event.request.scroll_depth, engagement_time: event.request.engagement_time, - interactive?: event.request.interactive? + interactive?: event.request.interactive?, + replay_session_id: event.request.replay_session_id }) end From 7cdee5b177c9e0849c6ed95f449bb8733a14fcd8 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 16:36:48 +0200 Subject: [PATCH 04/34] Put session-dependent salt for replayed event --- lib/plausible/ingestion/event.ex | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 7a1a7ec1b24b..5493683ed0c1 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -381,8 +381,21 @@ defmodule Plausible.Ingestion.Event do end end - defp put_salts(%__MODULE__{} = event, _context) do - %{event | salts: Plausible.Session.Salts.fetch()} + on_ee do + defp put_salts(%__MODULE__{} = event, _context) do + if session_id = event.request.replay_session_id do + computed_salt = :crypto.hash(:sha, [secret_key_base(), session_id]) |> binary_part(0, 16) + + salts = %{previous: nil, current: computed_salt} + %{event | salts: salts} + else + %{event | salts: Plausible.Session.Salts.fetch()} + end + end + else + defp put_salts(%__MODULE__{} = event, _context) do + %{event | salts: Plausible.Session.Salts.fetch()} + end end defp put_user_id(%__MODULE__{} = event, _context) do @@ -586,4 +599,11 @@ defmodule Plausible.Ingestion.Event do end defp spam_referrer?(_), do: false + + on_ee do + defp secret_key_base() do + Application.get_env(:plausible, PlausibleWeb.Endpoint) + |> Keyword.fetch!(:secret_key_base) + end + end end From a1a8b587ed620088f497df318a4592921ead8f25 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 19:45:09 +0200 Subject: [PATCH 05/34] Add a separate cache for replayed sessions --- config/runtime.exs | 4 +++- lib/plausible/application.ex | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 91f056353d27..c9a1d808b64b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1099,7 +1099,9 @@ unless s3_disabled? do imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET") end -config :plausible, Plausible.Cache.Adapter, sessions: [partitions: 100] +config :plausible, Plausible.Cache.Adapter, + sessions: [partitions: 100], + replay_sessions: [partitions: 20] config :plausible, Plausible.IP.Tools, allow_reserved_ips?: diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 20191fc70189..885e5becb5dc 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -58,6 +58,14 @@ defmodule Plausible.Application do n_lock_partitions: 1, ets_options: [read_concurrency: true, write_concurrency: true] ), + on_ee do + Plausible.Cache.Adapter.child_specs(:replay_sessions, :cache_replay_sessions, + ttl_check_interval: :timer.seconds(10), + global_ttl: :timer.minutes(30), + n_lock_partitions: 1, + ets_options: [read_concurrency: true, write_concurrency: true] + ) + end, {Plausible.Session.Transfer, base_path: Application.get_env(:plausible, :session_transfer_dir)}, warmed_cache(Plausible.Site.Cache, From 448eeed7c4a57c20eacf197f3dd19a16fb96683b Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 17 Jun 2026 19:46:00 +0200 Subject: [PATCH 06/34] Use different cache and key for replayed events in `CacheStore` --- lib/plausible/session/cache_store.ex | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index 05b3998e06ce..f1bb38b60259 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -70,7 +70,14 @@ defmodule Plausible.Session.CacheStore do defp find_session(_domain, nil), do: nil defp find_session(event, user_id) do - from_cache = Plausible.Cache.Adapter.get(:sessions, {event.site_id, user_id}) + {cache_name, key} = + if event.replay_session_id do + {:replay_sessions, {event.site_id, user_id, event.replay_session_id}} + else + {:sessions, {event.site_id, user_id}} + end + + from_cache = Plausible.Cache.Adapter.get(cache_name, key) case from_cache do nil -> @@ -84,8 +91,14 @@ defmodule Plausible.Session.CacheStore do end defp update_session_cache(session) do - key = {session.site_id, session.user_id} - Plausible.Cache.Adapter.put(:sessions, key, session, dirty?: true) + {cache_name, key} = + if session.replay_session_id do + {:replay_sessions, {session.site_id, session.user_id, session.replay_session_id}} + else + {:sessions, {session.site_id, session.user_id}} + end + + Plausible.Cache.Adapter.put(cache_name, key, session, dirty?: true) session end @@ -158,6 +171,7 @@ defmodule Plausible.Session.CacheStore do browser_version: Map.get(session_attributes, :browser_version), timestamp: event.timestamp, start: event.timestamp, + replay_session_id: event.replay_session_id, "entry_meta.key": Map.get(event, :"meta.key"), "entry_meta.value": Map.get(event, :"meta.value") } From fa09841cdf308678df46f577a04643885174c913 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 10:54:54 +0200 Subject: [PATCH 07/34] Deserialize and persist replay session ID as integer --- lib/plausible/clickhouse_event_v2.ex | 6 +++--- lib/plausible/clickhouse_session_v2.ex | 4 ++-- lib/plausible/ingestion/event.ex | 4 +++- lib/plausible/ingestion/request.ex | 18 ++++++++++++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 79b2b2571176..3624789c7c39 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -48,11 +48,11 @@ defmodule Plausible.ClickhouseEventV2 do field :acquisition_channel, Ch, type: "LowCardinality(String)", writable: :never + # Field used during event replay + field :replay_session_id, Ch, type: "UInt64" + # Virtual field used during event processing field :interactive?, :boolean, default: true, virtual: true, writable: :never - - # Virtual field used during event replay - field :replay_session_id, :string, default: nil, virtual: true, writable: :never end def new(attrs) do diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex index e7fc405b96b2..25d5fde1e1f9 100644 --- a/lib/plausible/clickhouse_session_v2.ex +++ b/lib/plausible/clickhouse_session_v2.ex @@ -76,8 +76,8 @@ defmodule Plausible.ClickhouseSessionV2 do field :acquisition_channel, Ch, type: "LowCardinality(String)", writable: :never - # Virtual field used during event replay - field :replay_session_id, :string, default: nil, virtual: true, writable: :never + # Field used during event replay + field :replay_session_id, Ch, type: "UInt64" end def random_uint64() do diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 5493683ed0c1..2c28a6cef3de 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -384,7 +384,9 @@ defmodule Plausible.Ingestion.Event do on_ee do defp put_salts(%__MODULE__{} = event, _context) do if session_id = event.request.replay_session_id do - computed_salt = :crypto.hash(:sha, [secret_key_base(), session_id]) |> binary_part(0, 16) + computed_salt = + :crypto.hash(:sha, [secret_key_base(), :binary.encode_unsigned(session_id)]) + |> binary_part(0, 16) salts = %{previous: nil, current: computed_salt} %{event | salts: salts} diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index 853ad899e3bf..3638a6bb7a66 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -61,8 +61,8 @@ defmodule Plausible.Ingestion.Request do field :revenue_source, :map # fields meant for replayed events only - field :replay_id, :string - field :replay_session_id, :string + field :replay_id, :integer + field :replay_session_id, :integer end field :query_params, :map @@ -146,12 +146,14 @@ defmodule Plausible.Ingestion.Request do conn |> Plug.Conn.get_req_header(@replay_id_header) |> List.first() + |> to_integer() if id do session_id = conn |> Plug.Conn.get_req_header(@replay_session_id_header) |> List.first() + |> to_integer() time = conn @@ -167,6 +169,18 @@ defmodule Plausible.Ingestion.Request do changeset end end + + defp to_integer(s) when is_binary(s) do + case Integer.parse(s) do + {n, ""} when n > 0 -> + n + + _ -> + nil + end + end + + defp to_integer(_), do: nil else defp put_replay_data(changeset, _conn), do: changeset end From 055c4deb1a2944baead92e9693efdb923acfe4ae Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 11:29:51 +0200 Subject: [PATCH 08/34] Add tests for Request --- test/plausible/ingestion/request_test.exs | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index 7426926a2546..5fb6129f1e77 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -173,6 +173,42 @@ defmodule Plausible.Ingestion.RequestTest do assert request.tracker_script_version == 137 end + @tag :ee_only + test "parses replay headers if present" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html" + } + + conn = + build_conn(:post, "/api/events", payload) + |> put_req_header("x-replay-event-id", "123") + |> put_req_header("x-replay-session-id", "456") + |> put_req_header("x-replay-time", "2026-06-02 12:43:00") + + assert {:ok, request, _conn} = Request.build(conn) + assert request.replay_id == 123 + assert request.replay_session_id == 456 + assert NaiveDateTime.compare(request.timestamp, ~N[2026-06-02 12:43:00]) == :eq + end + + @tag :ee_only + test "leaves replay fields empty and timestamp intact if headers not present" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html" + } + + conn = build_conn(:post, "/api/events", payload) + + assert {:ok, request, _conn} = Request.build(conn) + assert is_nil(request.replay_id) + assert is_nil(request.replay_session_id) + assert %NaiveDateTime{} = request.timestamp + end + @tag :ee_only test "parses revenue source field from a json string" do payload = %{ From 36c2881308b45b62bab1880017b5daeb779a5db1 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 15:31:46 +0200 Subject: [PATCH 09/34] Add ingest event tests --- test/plausible/ingestion/event_test.exs | 41 +++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 43a0a06e2d53..1f0ec94aff1b 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -475,6 +475,47 @@ defmodule Plausible.Ingestion.EventTest do assert Decimal.eq?(event.clickhouse_event.revenue_source_amount, Decimal.new("10.2")) end + @tag :ee_only + test "saves replay session id when passed in headers" do + site = new_site() + + payload = %{ + name: "pageview", + url: "http://#{site.domain}" + } + + conn = + build_conn(:post, "/api/events", payload) + |> Plug.Conn.put_req_header("x-replay-event-id", "123") + |> Plug.Conn.put_req_header("x-replay-session-id", "456") + |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:43:00") + + assert {:ok, request, _conn} = Request.build(conn) + + assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request) + assert event.clickhouse_event.replay_session_id == 456 + assert event.clickhouse_event.timestamp == ~N[2026-06-02 12:43:00] + end + + @tag :ee_only + test "leaves replay session id empty when not passed in headers" do + site = new_site() + + payload = %{ + name: "pageview", + url: "http://#{site.domain}" + } + + conn = + build_conn(:post, "/api/events", payload) + + assert {:ok, request, _conn} = Request.build(conn) + + assert {:ok, %{buffered: [event], dropped: []}} = Event.build_and_buffer(request) + assert is_nil(event.clickhouse_event.replay_session_id) + assert %NaiveDateTime{} = event.clickhouse_event.timestamp + end + test "does not save revenue amount when there is no revenue goal" do site = new_site() From 947a4b753660e545a77f0f6879a94c99af579f9d Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 17:12:56 +0200 Subject: [PATCH 10/34] Add test for session-dependent salt generation --- test/plausible/ingestion/event_test.exs | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 1f0ec94aff1b..8e6f95d43b45 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -516,6 +516,67 @@ defmodule Plausible.Ingestion.EventTest do assert %NaiveDateTime{} = event.clickhouse_event.timestamp end + @tag :ee_only + test "replayed events have a distinct user id due to reply session-dependent salt" do + site = new_site() + + payload = %{ + name: "pageview", + url: "http://#{site.domain}" + } + + conn1 = + build_conn(:post, "/api/events", payload) + |> Plug.Conn.put_req_header("user-agent", "Mozilla") + |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4") + + assert {:ok, request1, _conn} = Request.build(conn1) + assert {:ok, %{buffered: [event1], dropped: []}} = Event.build_and_buffer(request1) + + conn2 = + build_conn(:post, "/api/events", payload) + |> Plug.Conn.put_req_header("user-agent", "Mozilla") + |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4") + |> Plug.Conn.put_req_header("x-replay-event-id", "123") + |> Plug.Conn.put_req_header("x-replay-session-id", "456") + |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:43:00") + + assert {:ok, request2, _conn} = Request.build(conn2) + assert {:ok, %{buffered: [event2], dropped: []}} = Event.build_and_buffer(request2) + + conn3 = + build_conn(:post, "/api/events", payload) + |> Plug.Conn.put_req_header("user-agent", "Mozilla") + |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4") + |> Plug.Conn.put_req_header("x-replay-event-id", "124") + |> Plug.Conn.put_req_header("x-replay-session-id", "456") + |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:44:01") + + assert {:ok, request3, _conn} = Request.build(conn3) + assert {:ok, %{buffered: [event3], dropped: []}} = Event.build_and_buffer(request3) + + conn4 = + build_conn(:post, "/api/events", payload) + |> Plug.Conn.put_req_header("user-agent", "Mozilla") + |> Plug.Conn.put_req_header("x-plausible-ip", "1.2.3.4") + |> Plug.Conn.put_req_header("x-replay-event-id", "125") + |> Plug.Conn.put_req_header("x-replay-session-id", "789") + |> Plug.Conn.put_req_header("x-replay-time", "2026-06-02 12:44:01") + + assert {:ok, request4, _conn} = Request.build(conn4) + assert {:ok, %{buffered: [event4], dropped: []}} = Event.build_and_buffer(request4) + + # non-replayed event user id different from replayed event user despite + # identical fingerprint + assert event1.clickhouse_event.user_id != event2.clickhouse_event.user_id + # replayed events with the same session for the same fingerprint have the + # same useer id + assert event2.clickhouse_event.user_id == event3.clickhouse_event.user_id + # replayed event from different sessions for the same fingerprint + # have different users + assert event3.clickhouse_event.user_id != event4.clickhouse_event.user_id + end + test "does not save revenue amount when there is no revenue goal" do site = new_site() From 3805f8452789b9294a32dfebfb22253f33103d11 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 17:13:13 +0200 Subject: [PATCH 11/34] Add test for cache store --- test/plausible/session/cache_store_test.exs | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs index 8efb8c90cb28..1dad650df61d 100644 --- a/test/plausible/session/cache_store_test.exs +++ b/test/plausible/session/cache_store_test.exs @@ -246,6 +246,50 @@ defmodule Plausible.Session.CacheStoreTest do # assert Map.get(session, :"entry.meta.value") == ["true", "false"] end + @tag :ee_only + test "creates a session from a replayed event", %{buffer: buffer} do + event = + build(:event, + name: "pageview", + replay_session_id: 456, + "meta.key": ["logged_in", "darkmode"], + "meta.value": ["true", "false"] + ) + + CacheStore.on_event(event, @session_params, nil, buffer_insert: buffer) + + assert_receive({:buffer, :insert, [sessions]}) + assert [session] = sessions + assert session.hostname == event.hostname + + assert session.site_id == event.site_id + + assert session.replay_session_id == 456 + + assert session.user_id == event.user_id + assert session.entry_page == event.pathname + assert session.exit_page == event.pathname + assert session.is_bounce == true + assert session.duration == 0 + assert session.pageviews == 1 + assert session.events == 1 + assert session.referrer == Map.get(@session_params, :referrer) + assert session.referrer_source == Map.get(@session_params, :referrer_source) + assert session.utm_medium == Map.get(@session_params, :utm_medium) + assert session.utm_source == Map.get(@session_params, :utm_source) + assert session.utm_campaign == Map.get(@session_params, :utm_campaign) + assert session.utm_content == Map.get(@session_params, :utm_content) + assert session.utm_term == Map.get(@session_params, :utm_term) + assert session.country_code == Map.get(@session_params, :country_code) + assert session.screen_size == Map.get(@session_params, :screen_size) + assert session.operating_system == Map.get(@session_params, :operating_system) + assert session.operating_system_version == Map.get(@session_params, :operating_system_version) + assert session.browser == Map.get(@session_params, :browser) + assert session.browser_version == Map.get(@session_params, :browser_version) + assert session.timestamp == event.timestamp + assert session.start === event.timestamp + end + test "updates session counters", %{buffer: buffer} do timestamp = DateTime.utc_now() From ccb30bf37521feac4156d6f04b3abf0b584bfd77 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 17:19:38 +0200 Subject: [PATCH 12/34] Revert "Add a separate cache for replayed sessions" This reverts commit fd2df8b3bf2e1ef04a87f6e1705371c8f6ef5574. --- config/runtime.exs | 4 +--- lib/plausible/application.ex | 8 -------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index c9a1d808b64b..91f056353d27 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1099,9 +1099,7 @@ unless s3_disabled? do imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET") end -config :plausible, Plausible.Cache.Adapter, - sessions: [partitions: 100], - replay_sessions: [partitions: 20] +config :plausible, Plausible.Cache.Adapter, sessions: [partitions: 100] config :plausible, Plausible.IP.Tools, allow_reserved_ips?: diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 885e5becb5dc..20191fc70189 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -58,14 +58,6 @@ defmodule Plausible.Application do n_lock_partitions: 1, ets_options: [read_concurrency: true, write_concurrency: true] ), - on_ee do - Plausible.Cache.Adapter.child_specs(:replay_sessions, :cache_replay_sessions, - ttl_check_interval: :timer.seconds(10), - global_ttl: :timer.minutes(30), - n_lock_partitions: 1, - ets_options: [read_concurrency: true, write_concurrency: true] - ) - end, {Plausible.Session.Transfer, base_path: Application.get_env(:plausible, :session_transfer_dir)}, warmed_cache(Plausible.Site.Cache, From 024b36fcb24dd0dd68d435d905cc9fde9a3638d6 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 25 Jun 2026 17:19:43 +0200 Subject: [PATCH 13/34] Revert "Use different cache and key for replayed events in `CacheStore`" This reverts commit f72a8120cd19a7fbda5e94732539dd50079ee984. --- lib/plausible/session/cache_store.ex | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index f1bb38b60259..05b3998e06ce 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -70,14 +70,7 @@ defmodule Plausible.Session.CacheStore do defp find_session(_domain, nil), do: nil defp find_session(event, user_id) do - {cache_name, key} = - if event.replay_session_id do - {:replay_sessions, {event.site_id, user_id, event.replay_session_id}} - else - {:sessions, {event.site_id, user_id}} - end - - from_cache = Plausible.Cache.Adapter.get(cache_name, key) + from_cache = Plausible.Cache.Adapter.get(:sessions, {event.site_id, user_id}) case from_cache do nil -> @@ -91,14 +84,8 @@ defmodule Plausible.Session.CacheStore do end defp update_session_cache(session) do - {cache_name, key} = - if session.replay_session_id do - {:replay_sessions, {session.site_id, session.user_id, session.replay_session_id}} - else - {:sessions, {session.site_id, session.user_id}} - end - - Plausible.Cache.Adapter.put(cache_name, key, session, dirty?: true) + key = {session.site_id, session.user_id} + Plausible.Cache.Adapter.put(:sessions, key, session, dirty?: true) session end @@ -171,7 +158,6 @@ defmodule Plausible.Session.CacheStore do browser_version: Map.get(session_attributes, :browser_version), timestamp: event.timestamp, start: event.timestamp, - replay_session_id: event.replay_session_id, "entry_meta.key": Map.get(event, :"meta.key"), "entry_meta.value": Map.get(event, :"meta.value") } From 0e6ae782c0256e8452bdb7680d9615ea6fa5bf0b Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 7 Jul 2026 21:52:46 +0200 Subject: [PATCH 14/34] Don't replay events with timestamp in the future --- lib/plausible/ingestion/request.ex | 14 ++++++++++---- test/plausible/ingestion/request_test.exs | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index 3638a6bb7a66..21e9e6f7bf1e 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -142,6 +142,8 @@ defmodule Plausible.Ingestion.Request do @replay_time_header "x-replay-time" defp put_replay_data(changeset, conn) do + now = NaiveDateTime.utc_now(:second) + id = conn |> Plug.Conn.get_req_header(@replay_id_header) @@ -161,10 +163,14 @@ defmodule Plausible.Ingestion.Request do |> List.first() |> NaiveDateTime.from_iso8601!() - changeset - |> Changeset.put_change(:replay_id, id) - |> Changeset.put_change(:replay_session_id, session_id) - |> Changeset.put_change(:timestamp, time) + if NaiveDateTime.compare(time, now) in [:lt, :eq] do + changeset + |> Changeset.put_change(:replay_id, id) + |> Changeset.put_change(:replay_session_id, session_id) + |> Changeset.put_change(:timestamp, time) + else + changeset + end else changeset end diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index 5fb6129f1e77..bb84c51562a3 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -209,6 +209,29 @@ defmodule Plausible.Ingestion.RequestTest do assert %NaiveDateTime{} = request.timestamp end + @tag :ee_only + test "leaves replay fields empty and timestamp intact if replay time in the future" do + payload = %{ + name: "pageview", + domain: "dummy.site", + url: "http://dummy.site/index.html" + } + + now = NaiveDateTime.utc_now(:second) + time = NaiveDateTime.add(now, 10, :second) + + conn = + build_conn(:post, "/api/events", payload) + |> put_req_header("x-replay-event-id", "123") + |> put_req_header("x-replay-session-id", "456") + |> put_req_header("x-replay-time", NaiveDateTime.to_iso8601(time)) + + assert {:ok, request, _conn} = Request.build(conn) + assert is_nil(request.replay_id) + assert is_nil(request.replay_session_id) + assert NaiveDateTime.compare(request.timestamp, time) == :lt + end + @tag :ee_only test "parses revenue source field from a json string" do payload = %{ From 2a1bb1a27ad29879c643d8e15ba706a73ed25307 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 7 Jul 2026 21:56:11 +0200 Subject: [PATCH 15/34] Fix request encoding test --- test/plausible/ingestion/request_test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index bb84c51562a3..3e91a7441d93 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -662,7 +662,9 @@ defmodule Plausible.Ingestion.RequestTest do "scroll_depth" => nil, "engagement_time" => nil, "tracker_script_version" => 0, - "interactive?" => true + "interactive?" => true, + "replay_id" => nil, + "replay_session_id" => nil } assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"]) From 5dff699a7f54f503bf1c542ad68015517d823351 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 8 Jul 2026 10:14:07 +0200 Subject: [PATCH 16/34] Bring back logic using different cache key for replayed sessions --- lib/plausible/session/cache_store.ex | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index 05b3998e06ce..2d0101bff3ba 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -70,7 +70,14 @@ defmodule Plausible.Session.CacheStore do defp find_session(_domain, nil), do: nil defp find_session(event, user_id) do - from_cache = Plausible.Cache.Adapter.get(:sessions, {event.site_id, user_id}) + key = + if event.replay_session_id do + {event.site_id, user_id, event.replay_session_id} + else + {event.site_id, user_id} + end + + from_cache = Plausible.Cache.Adapter.get(:sessions, key) case from_cache do nil -> @@ -84,7 +91,13 @@ defmodule Plausible.Session.CacheStore do end defp update_session_cache(session) do - key = {session.site_id, session.user_id} + key = + if session.replay_session_id do + {session.site_id, session.user_id, session.replay_session_id} + else + {session.site_id, session.user_id} + end + Plausible.Cache.Adapter.put(:sessions, key, session, dirty?: true) session end @@ -158,6 +171,7 @@ defmodule Plausible.Session.CacheStore do browser_version: Map.get(session_attributes, :browser_version), timestamp: event.timestamp, start: event.timestamp, + replay_session_id: event.replay_session_id, "entry_meta.key": Map.get(event, :"meta.key"), "entry_meta.value": Map.get(event, :"meta.value") } From 9924aab291e0d4399714ec55917310b13a47ae8c Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 8 Jul 2026 10:25:41 +0200 Subject: [PATCH 17/34] Add more cache store tests --- test/plausible/session/cache_store_test.exs | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs index 1dad650df61d..0f0c85d9fb26 100644 --- a/test/plausible/session/cache_store_test.exs +++ b/test/plausible/session/cache_store_test.exs @@ -293,6 +293,31 @@ defmodule Plausible.Session.CacheStoreTest do test "updates session counters", %{buffer: buffer} do timestamp = DateTime.utc_now() + event1 = + build(:event, + name: "pageview", + replay_session_id: 456, + timestamp: timestamp |> NaiveDateTime.shift(second: -10) + ) + + event2 = %{ + event1 + | timestamp: timestamp + } + + CacheStore.on_event(event1, %{}, nil, buffer_insert: buffer) + CacheStore.on_event(event2, %{}, nil, buffer_insert: buffer) + assert_receive({:buffer, :insert, [[_negative_record, session]]}) + assert session.is_bounce == false + assert session.duration == 10 + assert session.pageviews == 2 + assert session.events == 2 + assert session.replay_session_id == 456 + end + + test "updates session counters for replayed event", %{buffer: buffer} do + timestamp = DateTime.utc_now() + event1 = build(:event, name: "pageview", timestamp: timestamp |> NaiveDateTime.shift(second: -10)) @@ -310,6 +335,32 @@ defmodule Plausible.Session.CacheStoreTest do assert session.events == 2 end + test "treats replayed and normal events as belonging to distinct sessions even if user_id and site_id match", + %{buffer: buffer} do + timestamp = DateTime.utc_now() + + event1 = + build(:event, + name: "pageview", + replay_session_id: 456, + timestamp: timestamp |> NaiveDateTime.shift(second: -10) + ) + + event2 = %{ + event1 + | replay_session_id: nil, + timestamp: timestamp + } + + CacheStore.on_event(event1, %{}, nil, buffer_insert: buffer) + assert_receive({:buffer, :insert, [[session1]]}) + CacheStore.on_event(event2, %{}, nil, buffer_insert: buffer) + assert_receive({:buffer, :insert, [[session2]]}) + assert session1.session_id != session2.session_id + assert session1.replay_session_id == 456 + assert is_nil(session2.replay_session_id) + end + test "does not update session counters on engagement event", %{buffer: buffer} do now = NaiveDateTime.utc_now(:second) pageview = build(:pageview, timestamp: NaiveDateTime.shift(now, second: -10)) From fb015852e06e4633da0f46ac9e2f528ee50bbe48 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 8 Jul 2026 11:52:41 +0200 Subject: [PATCH 18/34] Compute session salts more safely and cache them --- extra/lib/plausible/session/computed_salts.ex | 66 +++++++++++++++++++ lib/plausible/application.ex | 3 + lib/plausible/ingestion/event.ex | 16 +---- .../plausible/session/computed_salts_test.exs | 52 +++++++++++++++ 4 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 extra/lib/plausible/session/computed_salts.ex create mode 100644 test/plausible/session/computed_salts_test.exs diff --git a/extra/lib/plausible/session/computed_salts.ex b/extra/lib/plausible/session/computed_salts.ex new file mode 100644 index 000000000000..ed3835ed1ce6 --- /dev/null +++ b/extra/lib/plausible/session/computed_salts.ex @@ -0,0 +1,66 @@ +defmodule Plausible.Session.ComputedSalts do + @moduledoc """ + Cache for computed salts used for replayed events. + + It's not cleaned, however the range of possible values is limited + to the number of replayed sessions over the lifetime of the node + and sessions IDs can't be provided externally by users. + + Despite that, the cache is still purged every couple hours. + """ + + @purge_interval :timer.seconds(7200) + + use GenServer + + alias Plug.Crypto.KeyGenerator + + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name] || __MODULE__) + end + + @impl true + def init(opts) do + name = opts[:name] || __MODULE__ + + ^name = + :ets.new(name, [ + :named_table, + :set, + :public, + {:read_concurrency, true} + ]) + + schedule_purge() + + {:ok, name} + end + + @spec fetch(module() | atom(), pos_integer()) :: %{previous: nil, current: binary()} + def fetch(name \\ __MODULE__, session_id) do + computed_salt = + secret_key_base() + |> KeyGenerator.generate(:binary.encode_unsigned(session_id), cache: name) + |> binary_part(0, 16) + + %{previous: nil, current: computed_salt} + end + + @impl true + def handle_info(:purge, name) do + :ets.delete_all_objects(name) + schedule_purge() + + {:noreply, name} + end + + defp schedule_purge() do + Process.send_after(self(), :purge, @purge_interval) + end + + defp secret_key_base() do + Application.get_env(:plausible, PlausibleWeb.Endpoint) + |> Keyword.fetch!(:secret_key_base) + end +end diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 20191fc70189..4d67e729b75e 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -194,6 +194,9 @@ defmodule Plausible.Application do end, Plausible.Ingestion.Counters, Plausible.Session.Salts, + on_ee do + Plausible.Session.ComputedSalts + end, Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), ReferrerBlocklist, diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 2c28a6cef3de..d21e5cb4ae48 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -383,13 +383,8 @@ defmodule Plausible.Ingestion.Event do on_ee do defp put_salts(%__MODULE__{} = event, _context) do - if session_id = event.request.replay_session_id do - computed_salt = - :crypto.hash(:sha, [secret_key_base(), :binary.encode_unsigned(session_id)]) - |> binary_part(0, 16) - - salts = %{previous: nil, current: computed_salt} - %{event | salts: salts} + if session_id = event.clickhouse_event_attrs.replay_session_id do + %{event | salts: Plausible.Session.ComputedSalts.fetch(session_id)} else %{event | salts: Plausible.Session.Salts.fetch()} end @@ -601,11 +596,4 @@ defmodule Plausible.Ingestion.Event do end defp spam_referrer?(_), do: false - - on_ee do - defp secret_key_base() do - Application.get_env(:plausible, PlausibleWeb.Endpoint) - |> Keyword.fetch!(:secret_key_base) - end - end end diff --git a/test/plausible/session/computed_salts_test.exs b/test/plausible/session/computed_salts_test.exs new file mode 100644 index 000000000000..86bf303f62a9 --- /dev/null +++ b/test/plausible/session/computed_salts_test.exs @@ -0,0 +1,52 @@ +defmodule Plausible.Session.ComputedSaltsTest do + use Plausible.DataCase, async: false + use Plausible + + on_ee do + alias Plausible.Session.ComputedSalts + + test "agent starts and responds with salt based on provided session id" do + {:ok, _} = ComputedSalts.start_link(name: __MODULE__) + %{current: current, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + assert is_binary(current) + end + + test "agent starts and responds with the same salt for the same session id" do + {:ok, _} = ComputedSalts.start_link(name: __MODULE__) + %{current: current1, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + %{current: current2, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + assert is_binary(current1) + assert current1 == current2 + end + + test "agent starts and responds with a different salt for different session ids" do + {:ok, _} = ComputedSalts.start_link(name: __MODULE__) + %{current: current1, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + %{current: current2, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_124) + assert is_binary(current1) + assert is_binary(current2) + assert current1 != current2 + end + + test "salts are cached and cache can be purged" do + {:ok, _} = ComputedSalts.start_link(name: __MODULE__) + %{current: salt_before_purge, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + %{current: _, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_124) + + assert [_, _] = :ets.tab2list(__MODULE__) + + send(__MODULE__, :purge) + + assert eventually(fn -> + table = :ets.tab2list(__MODULE__) + {table == [], table} + end) + + %{current: salt_after_purge, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) + + assert [_] = :ets.tab2list(__MODULE__) + + assert salt_before_purge == salt_after_purge + end + end +end From a0864cae4f757938cc820894d9491a5b2fd47cb9 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 9 Jul 2026 11:34:13 +0200 Subject: [PATCH 19/34] Fix logic and tests for CE --- lib/plausible/clickhouse_event_v2.ex | 12 +++--- lib/plausible/clickhouse_session_v2.ex | 7 ++- lib/plausible/ingestion/event.ex | 13 ++++-- lib/plausible/session/cache_store.ex | 27 +++++++++--- test/plausible/ingestion/request_test.exs | 47 ++++++++++++--------- test/plausible/session/cache_store_test.exs | 3 ++ 6 files changed, 71 insertions(+), 38 deletions(-) diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 3624789c7c39..ee90a04b1c10 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -3,6 +3,7 @@ defmodule Plausible.ClickhouseEventV2 do Event schema for when NumericIDs migration is complete """ use Ecto.Schema + use Plausible import Ecto.Changeset @primary_key false @@ -48,8 +49,10 @@ defmodule Plausible.ClickhouseEventV2 do field :acquisition_channel, Ch, type: "LowCardinality(String)", writable: :never - # Field used during event replay - field :replay_session_id, Ch, type: "UInt64" + on_ee do + # Field used during event replay + field :replay_session_id, Ch, type: "UInt64" + end # Virtual field used during event processing field :interactive?, :boolean, default: true, virtual: true, writable: :never @@ -74,9 +77,8 @@ defmodule Plausible.ClickhouseEventV2 do :revenue_source_currency, :revenue_reporting_amount, :revenue_reporting_currency, - :interactive?, - :replay_session_id - ] + :interactive? + ] ++ on_ee(do: [:replay_session_id], else: []) ) |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp]) end diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex index 25d5fde1e1f9..379247dce362 100644 --- a/lib/plausible/clickhouse_session_v2.ex +++ b/lib/plausible/clickhouse_session_v2.ex @@ -3,6 +3,7 @@ defmodule Plausible.ClickhouseSessionV2 do Session schema for when NumericIDs migration is complete """ use Ecto.Schema + use Plausible defmodule BoolUInt8 do @moduledoc """ @@ -76,8 +77,10 @@ defmodule Plausible.ClickhouseSessionV2 do field :acquisition_channel, Ch, type: "LowCardinality(String)", writable: :never - # Field used during event replay - field :replay_session_id, Ch, type: "UInt64" + on_ee do + # Field used during event replay + field :replay_session_id, Ch, type: "UInt64" + end end def random_uint64() do diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index d21e5cb4ae48..04e16e51bce6 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -276,7 +276,7 @@ defmodule Plausible.Ingestion.Event do end defp put_basic_info(%__MODULE__{} = event, _context) do - update_event_attrs(event, %{ + attrs = %{ domain: event.domain, site_id: event.site.id, timestamp: event.request.timestamp, @@ -285,9 +285,14 @@ defmodule Plausible.Ingestion.Event do pathname: event.request.pathname, scroll_depth: event.request.scroll_depth, engagement_time: event.request.engagement_time, - interactive?: event.request.interactive?, - replay_session_id: event.request.replay_session_id - }) + interactive?: event.request.interactive? + } + + on_ee do + attrs = Map.put(attrs, :replay_session_id, event.request.replay_session_id) + end + + update_event_attrs(event, attrs) end defp put_source_info(%__MODULE__{} = event, _context) do diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index 2d0101bff3ba..a40814d4e7a0 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -3,6 +3,8 @@ defmodule Plausible.Session.CacheStore do Session management on the basis of incoming events. """ + use Plausible + alias Plausible.Session.WriteBuffer @lock_timeout 1000 @@ -71,8 +73,12 @@ defmodule Plausible.Session.CacheStore do defp find_session(event, user_id) do key = - if event.replay_session_id do - {event.site_id, user_id, event.replay_session_id} + on_ee do + if event.replay_session_id do + {event.site_id, user_id, event.replay_session_id} + else + {event.site_id, user_id} + end else {event.site_id, user_id} end @@ -92,8 +98,12 @@ defmodule Plausible.Session.CacheStore do defp update_session_cache(session) do key = - if session.replay_session_id do - {session.site_id, session.user_id, session.replay_session_id} + on_ee do + if session.replay_session_id do + {session.site_id, session.user_id, session.replay_session_id} + else + {session.site_id, session.user_id} + end else {session.site_id, session.user_id} end @@ -139,7 +149,7 @@ defmodule Plausible.Session.CacheStore do end defp new_session_from_event(event, session_attributes) do - %Plausible.ClickhouseSessionV2{ + new_session = %Plausible.ClickhouseSessionV2{ sign: 1, session_id: Plausible.ClickhouseSessionV2.random_uint64(), hostname: if(event.name == "pageview", do: event.hostname, else: ""), @@ -171,9 +181,14 @@ defmodule Plausible.Session.CacheStore do browser_version: Map.get(session_attributes, :browser_version), timestamp: event.timestamp, start: event.timestamp, - replay_session_id: event.replay_session_id, "entry_meta.key": Map.get(event, :"meta.key"), "entry_meta.value": Map.get(event, :"meta.value") } + + on_ee do + %{new_session | replay_session_id: event.replay_session_id} + else + new_session + end end end diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index 3e91a7441d93..e7a4d3e4d5cc 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -645,27 +645,32 @@ defmodule Plausible.Ingestion.RequestTest do request = request |> Jason.encode!() |> Jason.decode!() - assert Map.drop(request, ["timestamp"]) == %{ - "domains" => ["dummy.site"], - "event_name" => "pageview", - "hash_mode" => 1, - "hostname" => "dummy.site", - "pathname" => "/pictures/index.html", - "props" => %{"abc" => "qwerty", "hello" => "world"}, - "query_params" => %{"baz" => "bam", "foo" => "bar"}, - "referrer" => "https://example.com", - "remote_ip" => "127.0.0.1", - "revenue_source" => %{"amount" => "12.3", "currency" => "USD"}, - "uri" => "https://dummy.site/pictures/index.html?foo=bar&baz=bam", - "user_agent" => "Mozilla", - "ip_classification" => nil, - "scroll_depth" => nil, - "engagement_time" => nil, - "tracker_script_version" => 0, - "interactive?" => true, - "replay_id" => nil, - "replay_session_id" => nil - } + expected = %{ + "domains" => ["dummy.site"], + "event_name" => "pageview", + "hash_mode" => 1, + "hostname" => "dummy.site", + "pathname" => "/pictures/index.html", + "props" => %{"abc" => "qwerty", "hello" => "world"}, + "query_params" => %{"baz" => "bam", "foo" => "bar"}, + "referrer" => "https://example.com", + "remote_ip" => "127.0.0.1", + "revenue_source" => %{"amount" => "12.3", "currency" => "USD"}, + "uri" => "https://dummy.site/pictures/index.html?foo=bar&baz=bam", + "user_agent" => "Mozilla", + "ip_classification" => nil, + "scroll_depth" => nil, + "engagement_time" => nil, + "tracker_script_version" => 0, + "interactive?" => true, + "replay_id" => nil + } + + on_ee do + expected = Map.put(expected, "replay_session_id", nil) + end + + assert Map.drop(request, ["timestamp"]) == expected assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"]) end diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs index 0f0c85d9fb26..0d47f9384837 100644 --- a/test/plausible/session/cache_store_test.exs +++ b/test/plausible/session/cache_store_test.exs @@ -290,6 +290,7 @@ defmodule Plausible.Session.CacheStoreTest do assert session.start === event.timestamp end + @tag :ee_only test "updates session counters", %{buffer: buffer} do timestamp = DateTime.utc_now() @@ -315,6 +316,7 @@ defmodule Plausible.Session.CacheStoreTest do assert session.replay_session_id == 456 end + @tag :ee_only test "updates session counters for replayed event", %{buffer: buffer} do timestamp = DateTime.utc_now() @@ -335,6 +337,7 @@ defmodule Plausible.Session.CacheStoreTest do assert session.events == 2 end + @tag :ee_only test "treats replayed and normal events as belonging to distinct sessions even if user_id and site_id match", %{buffer: buffer} do timestamp = DateTime.utc_now() From 987642e8bc1d7e75a8a5127686a92ef30530f357 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 9 Jul 2026 11:41:51 +0200 Subject: [PATCH 20/34] Fix typo --- test/plausible/ingestion/event_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 8e6f95d43b45..93a70d8fc413 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -570,7 +570,7 @@ defmodule Plausible.Ingestion.EventTest do # identical fingerprint assert event1.clickhouse_event.user_id != event2.clickhouse_event.user_id # replayed events with the same session for the same fingerprint have the - # same useer id + # same user id assert event2.clickhouse_event.user_id == event3.clickhouse_event.user_id # replayed event from different sessions for the same fingerprint # have different users From 39104efddff032c0ab0b403f5b980ee954c2ce9f Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 9 Jul 2026 13:14:30 +0200 Subject: [PATCH 21/34] Reduce confusion in naming between replay session ID and session ID --- extra/lib/plausible/session/computed_salts.ex | 4 ++-- lib/plausible/ingestion/event.ex | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/lib/plausible/session/computed_salts.ex b/extra/lib/plausible/session/computed_salts.ex index ed3835ed1ce6..29e0a6b99d2e 100644 --- a/extra/lib/plausible/session/computed_salts.ex +++ b/extra/lib/plausible/session/computed_salts.ex @@ -38,10 +38,10 @@ defmodule Plausible.Session.ComputedSalts do end @spec fetch(module() | atom(), pos_integer()) :: %{previous: nil, current: binary()} - def fetch(name \\ __MODULE__, session_id) do + def fetch(name \\ __MODULE__, replay_session_id) do computed_salt = secret_key_base() - |> KeyGenerator.generate(:binary.encode_unsigned(session_id), cache: name) + |> KeyGenerator.generate(:binary.encode_unsigned(replay_session_id), cache: name) |> binary_part(0, 16) %{previous: nil, current: computed_salt} diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 04e16e51bce6..c9c58e075827 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -388,8 +388,8 @@ defmodule Plausible.Ingestion.Event do on_ee do defp put_salts(%__MODULE__{} = event, _context) do - if session_id = event.clickhouse_event_attrs.replay_session_id do - %{event | salts: Plausible.Session.ComputedSalts.fetch(session_id)} + if replay_session_id = event.clickhouse_event_attrs.replay_session_id do + %{event | salts: Plausible.Session.ComputedSalts.fetch(replay_session_id)} else %{event | salts: Plausible.Session.Salts.fetch()} end From 0b13e0c4d7d8561dc53c349f10f25e88c2c7b146 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 9 Jul 2026 13:14:52 +0200 Subject: [PATCH 22/34] Do not parse `X-Replay-Event-ID` as it's not used now --- lib/plausible/ingestion/request.ex | 19 +++++-------------- test/plausible/ingestion/request_test.exs | 6 +----- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index 21e9e6f7bf1e..c9c51a4d5c9c 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -60,8 +60,7 @@ defmodule Plausible.Ingestion.Request do on_ee do field :revenue_source, :map - # fields meant for replayed events only - field :replay_id, :integer + # field for replayed events field :replay_session_id, :integer end @@ -137,26 +136,19 @@ defmodule Plausible.Ingestion.Request do end on_ee do - @replay_id_header "x-replay-event-id" @replay_session_id_header "x-replay-session-id" @replay_time_header "x-replay-time" defp put_replay_data(changeset, conn) do now = NaiveDateTime.utc_now(:second) - id = + replay_session_id = conn - |> Plug.Conn.get_req_header(@replay_id_header) + |> Plug.Conn.get_req_header(@replay_session_id_header) |> List.first() |> to_integer() - if id do - session_id = - conn - |> Plug.Conn.get_req_header(@replay_session_id_header) - |> List.first() - |> to_integer() - + if replay_session_id do time = conn |> Plug.Conn.get_req_header(@replay_time_header) @@ -165,8 +157,7 @@ defmodule Plausible.Ingestion.Request do if NaiveDateTime.compare(time, now) in [:lt, :eq] do changeset - |> Changeset.put_change(:replay_id, id) - |> Changeset.put_change(:replay_session_id, session_id) + |> Changeset.put_change(:replay_session_id, replay_session_id) |> Changeset.put_change(:timestamp, time) else changeset diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs index e7a4d3e4d5cc..5b7d58f103cd 100644 --- a/test/plausible/ingestion/request_test.exs +++ b/test/plausible/ingestion/request_test.exs @@ -188,7 +188,6 @@ defmodule Plausible.Ingestion.RequestTest do |> put_req_header("x-replay-time", "2026-06-02 12:43:00") assert {:ok, request, _conn} = Request.build(conn) - assert request.replay_id == 123 assert request.replay_session_id == 456 assert NaiveDateTime.compare(request.timestamp, ~N[2026-06-02 12:43:00]) == :eq end @@ -204,7 +203,6 @@ defmodule Plausible.Ingestion.RequestTest do conn = build_conn(:post, "/api/events", payload) assert {:ok, request, _conn} = Request.build(conn) - assert is_nil(request.replay_id) assert is_nil(request.replay_session_id) assert %NaiveDateTime{} = request.timestamp end @@ -227,7 +225,6 @@ defmodule Plausible.Ingestion.RequestTest do |> put_req_header("x-replay-time", NaiveDateTime.to_iso8601(time)) assert {:ok, request, _conn} = Request.build(conn) - assert is_nil(request.replay_id) assert is_nil(request.replay_session_id) assert NaiveDateTime.compare(request.timestamp, time) == :lt end @@ -662,8 +659,7 @@ defmodule Plausible.Ingestion.RequestTest do "scroll_depth" => nil, "engagement_time" => nil, "tracker_script_version" => 0, - "interactive?" => true, - "replay_id" => nil + "interactive?" => true } on_ee do From 02903713170b29a16937b4b6315a15b6d55be7c8 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 15 Jul 2026 10:56:43 +0200 Subject: [PATCH 23/34] Do not clear `Site.domain_changed_at` on rename expiration --- lib/plausible/site/domain.ex | 6 ++---- lib/plausible/sites.ex | 1 - test/plausible/site/domain_test.exs | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/plausible/site/domain.ex b/lib/plausible/site/domain.ex index 9767b09b076c..547ae01e0c09 100644 --- a/lib/plausible/site/domain.ex +++ b/lib/plausible/site/domain.ex @@ -30,12 +30,10 @@ defmodule Plausible.Site.Domain do {updated, _} = Repo.update_all( from(s in Site.regular(), + where: not is_nil(s.domain_changed_from), where: s.domain_changed_at < ago(^expire_threshold_hours, "hour") ), - set: [ - domain_changed_from: nil, - domain_changed_at: nil - ] + set: [domain_changed_from: nil] ) {:ok, updated} diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 3b9e16dd8b84..4107b66ca1c6 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -325,7 +325,6 @@ defmodule Plausible.Sites do site |> Ecto.Changeset.change() |> Ecto.Changeset.put_change(:domain_changed_from, nil) - |> Ecto.Changeset.put_change(:domain_changed_at, nil) |> Repo.update() _ -> diff --git a/test/plausible/site/domain_test.exs b/test/plausible/site/domain_test.exs index 98a0135738d5..fe654cfc53e9 100644 --- a/test/plausible/site/domain_test.exs +++ b/test/plausible/site/domain_test.exs @@ -83,8 +83,9 @@ defmodule Plausible.Site.DomainTest do assert site.domain_changed_at assert {:ok, _} = Domain.expire_change_transitions(-1) - refute Repo.reload!(site).domain_changed_from - refute Repo.reload!(site).domain_changed_at + expired_site = Repo.reload!(site) + refute expired_site.domain_changed_from + assert expired_site.domain_changed_at end test "expire changes overdue" do @@ -99,14 +100,19 @@ defmodule Plausible.Site.DomainTest do assert {:ok, 1} = Domain.expire_change_transitions() - assert is_nil(Repo.reload!(s3).domain_changed_from) - assert is_nil(Repo.reload!(s3).domain_changed_at) + s3_after = Repo.reload!(s3) + assert is_nil(s3_after.domain_changed_from) + assert s3_after.domain_changed_at assert {:ok, 1} = Domain.expire_change_transitions(24) - assert is_nil(Repo.reload!(s2).domain_changed_at) + s2_after = Repo.reload!(s2) + assert is_nil(s2_after.domain_changed_from) + assert s2_after.domain_changed_at assert {:ok, 0} = Domain.expire_change_transitions() - assert Repo.reload!(s1).domain_changed_at + s1_after = Repo.reload!(s1) + assert s1_after.domain_changed_from + assert s1_after.domain_changed_at end test "new domain gets validated" do From 4bb676934464cc3301a8acd120d731760fd4414e Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 15 Jul 2026 11:02:00 +0200 Subject: [PATCH 24/34] Load `domain_changed_at` and `native_stats_start_at` into site cache --- lib/plausible/site/cache.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex index 2c559f66ca22..d3c822f9388d 100644 --- a/lib/plausible/site/cache.ex +++ b/lib/plausible/site/cache.ex @@ -31,6 +31,8 @@ defmodule Plausible.Site.Cache do domain_changed_from ingest_rate_limit_scale_seconds ingest_rate_limit_threshold + domain_changed_at + native_stats_start_at )a @impl true From 6d661f8214fa3c3daf2074050a7752034b3f551c Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 15 Jul 2026 11:11:26 +0200 Subject: [PATCH 25/34] Reject replayed events with timestamp preceeding matching site's domain --- lib/plausible/ingestion/event.ex | 27 +++++++++++++++++++++++++ test/plausible/ingestion/event_test.exs | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index c9c58e075827..0e51c71351e1 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -129,6 +129,7 @@ defmodule Plausible.Ingestion.Event do defp pipeline() do [ + drop_rogue_replay: &drop_rogue_replay/2, drop_verification_agent: &drop_verification_agent/2, drop_datacenter_ip: &drop_datacenter_ip/2, drop_threat_ip: &drop_threat_ip/2, @@ -193,6 +194,32 @@ defmodule Plausible.Ingestion.Event do struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs)) end + on_ee do + defp drop_rogue_replay(%__MODULE__{request: %{replay_session_id: nil}} = event, _context) do + event + end + + defp drop_rogue_replay(%__MODULE__{} = event, _context) do + before_native_stats? = + NaiveDateTime.compare(event.request.timestamp, event.site.native_stats_start_at) == :lt + + before_last_domain_change? = + if event.site.domain_changed_at do + NaiveDateTime.compare(event.request.timestamp, event.site.domain_changed_at) == :lt + else + false + end + + if before_native_stats? or before_last_domain_change? do + drop(event, :rogue_replay) + else + event + end + end + else + defp drop_rogue_replay(%__MODULE__{} = event, _context), do: event + end + on_ee do @verification_user_agent Plausible.InstallationSupport.user_agent() diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index 93a70d8fc413..fd001fb7e15f 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -518,7 +518,7 @@ defmodule Plausible.Ingestion.EventTest do @tag :ee_only test "replayed events have a distinct user id due to reply session-dependent salt" do - site = new_site() + site = new_site(native_stats_start_at: ~N[2026-06-01 12:00:00]) payload = %{ name: "pageview", From 7461760c09d6066ad0130f70941e9534cf6cb89c Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 16 Jul 2026 12:15:17 +0200 Subject: [PATCH 26/34] Revert "Reject replayed events with timestamp preceeding matching site's domain" This reverts commit d5d24944b6d6a8578d9daae099b382bd9bb781cb. --- lib/plausible/ingestion/event.ex | 27 ------------------------- test/plausible/ingestion/event_test.exs | 2 +- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 0e51c71351e1..c9c58e075827 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -129,7 +129,6 @@ defmodule Plausible.Ingestion.Event do defp pipeline() do [ - drop_rogue_replay: &drop_rogue_replay/2, drop_verification_agent: &drop_verification_agent/2, drop_datacenter_ip: &drop_datacenter_ip/2, drop_threat_ip: &drop_threat_ip/2, @@ -194,32 +193,6 @@ defmodule Plausible.Ingestion.Event do struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs)) end - on_ee do - defp drop_rogue_replay(%__MODULE__{request: %{replay_session_id: nil}} = event, _context) do - event - end - - defp drop_rogue_replay(%__MODULE__{} = event, _context) do - before_native_stats? = - NaiveDateTime.compare(event.request.timestamp, event.site.native_stats_start_at) == :lt - - before_last_domain_change? = - if event.site.domain_changed_at do - NaiveDateTime.compare(event.request.timestamp, event.site.domain_changed_at) == :lt - else - false - end - - if before_native_stats? or before_last_domain_change? do - drop(event, :rogue_replay) - else - event - end - end - else - defp drop_rogue_replay(%__MODULE__{} = event, _context), do: event - end - on_ee do @verification_user_agent Plausible.InstallationSupport.user_agent() diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index fd001fb7e15f..93a70d8fc413 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -518,7 +518,7 @@ defmodule Plausible.Ingestion.EventTest do @tag :ee_only test "replayed events have a distinct user id due to reply session-dependent salt" do - site = new_site(native_stats_start_at: ~N[2026-06-01 12:00:00]) + site = new_site() payload = %{ name: "pageview", From 12693f1bac048ef7ceeb9211ef18cc7542d8eea9 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 16 Jul 2026 12:15:21 +0200 Subject: [PATCH 27/34] Revert "Load `domain_changed_at` and `native_stats_start_at` into site cache" This reverts commit 06d25f5e1bb74d0fbce348254bce5e44f17c700f. --- lib/plausible/site/cache.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex index d3c822f9388d..2c559f66ca22 100644 --- a/lib/plausible/site/cache.ex +++ b/lib/plausible/site/cache.ex @@ -31,8 +31,6 @@ defmodule Plausible.Site.Cache do domain_changed_from ingest_rate_limit_scale_seconds ingest_rate_limit_threshold - domain_changed_at - native_stats_start_at )a @impl true From be2a4440e87d0dfbdb6cbfaca89af617aa9b3154 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 16 Jul 2026 12:15:22 +0200 Subject: [PATCH 28/34] Revert "Do not clear `Site.domain_changed_at` on rename expiration" This reverts commit cae6d95c46fb6dd600d316a0f2534fa56c9b66f5. --- lib/plausible/site/domain.ex | 6 ++++-- lib/plausible/sites.ex | 1 + test/plausible/site/domain_test.exs | 18 ++++++------------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/plausible/site/domain.ex b/lib/plausible/site/domain.ex index 547ae01e0c09..9767b09b076c 100644 --- a/lib/plausible/site/domain.ex +++ b/lib/plausible/site/domain.ex @@ -30,10 +30,12 @@ defmodule Plausible.Site.Domain do {updated, _} = Repo.update_all( from(s in Site.regular(), - where: not is_nil(s.domain_changed_from), where: s.domain_changed_at < ago(^expire_threshold_hours, "hour") ), - set: [domain_changed_from: nil] + set: [ + domain_changed_from: nil, + domain_changed_at: nil + ] ) {:ok, updated} diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 4107b66ca1c6..3b9e16dd8b84 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -325,6 +325,7 @@ defmodule Plausible.Sites do site |> Ecto.Changeset.change() |> Ecto.Changeset.put_change(:domain_changed_from, nil) + |> Ecto.Changeset.put_change(:domain_changed_at, nil) |> Repo.update() _ -> diff --git a/test/plausible/site/domain_test.exs b/test/plausible/site/domain_test.exs index fe654cfc53e9..98a0135738d5 100644 --- a/test/plausible/site/domain_test.exs +++ b/test/plausible/site/domain_test.exs @@ -83,9 +83,8 @@ defmodule Plausible.Site.DomainTest do assert site.domain_changed_at assert {:ok, _} = Domain.expire_change_transitions(-1) - expired_site = Repo.reload!(site) - refute expired_site.domain_changed_from - assert expired_site.domain_changed_at + refute Repo.reload!(site).domain_changed_from + refute Repo.reload!(site).domain_changed_at end test "expire changes overdue" do @@ -100,19 +99,14 @@ defmodule Plausible.Site.DomainTest do assert {:ok, 1} = Domain.expire_change_transitions() - s3_after = Repo.reload!(s3) - assert is_nil(s3_after.domain_changed_from) - assert s3_after.domain_changed_at + assert is_nil(Repo.reload!(s3).domain_changed_from) + assert is_nil(Repo.reload!(s3).domain_changed_at) assert {:ok, 1} = Domain.expire_change_transitions(24) - s2_after = Repo.reload!(s2) - assert is_nil(s2_after.domain_changed_from) - assert s2_after.domain_changed_at + assert is_nil(Repo.reload!(s2).domain_changed_at) assert {:ok, 0} = Domain.expire_change_transitions() - s1_after = Repo.reload!(s1) - assert s1_after.domain_changed_from - assert s1_after.domain_changed_at + assert Repo.reload!(s1).domain_changed_at end test "new domain gets validated" do From 26e8189fe8285df7bd3cb39f68e538c3da213c6c Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 16 Jul 2026 12:20:41 +0200 Subject: [PATCH 29/34] Simplify integer casting used for parsing replay headers --- lib/plausible/ingestion/request.ex | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex index c9c51a4d5c9c..e0a758fe5ab5 100644 --- a/lib/plausible/ingestion/request.ex +++ b/lib/plausible/ingestion/request.ex @@ -167,16 +167,7 @@ defmodule Plausible.Ingestion.Request do end end - defp to_integer(s) when is_binary(s) do - case Integer.parse(s) do - {n, ""} when n > 0 -> - n - - _ -> - nil - end - end - + defp to_integer(s) when is_binary(s), do: String.to_integer(s) defp to_integer(_), do: nil else defp put_replay_data(changeset, _conn), do: changeset From 496b50fc3dd711ff347acd30b221bbfb4ce56112 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Thu, 16 Jul 2026 12:26:49 +0200 Subject: [PATCH 30/34] Rename `ComputedSalts` to `ReplaySalts` --- .../{computed_salts.ex => replay_salts.ex} | 8 +-- lib/plausible/application.ex | 2 +- lib/plausible/ingestion/event.ex | 2 +- .../plausible/session/computed_salts_test.exs | 52 ------------------- test/plausible/session/replay_salts_test.exs | 52 +++++++++++++++++++ 5 files changed, 58 insertions(+), 58 deletions(-) rename extra/lib/plausible/session/{computed_salts.ex => replay_salts.ex} (89%) delete mode 100644 test/plausible/session/computed_salts_test.exs create mode 100644 test/plausible/session/replay_salts_test.exs diff --git a/extra/lib/plausible/session/computed_salts.ex b/extra/lib/plausible/session/replay_salts.ex similarity index 89% rename from extra/lib/plausible/session/computed_salts.ex rename to extra/lib/plausible/session/replay_salts.ex index 29e0a6b99d2e..5bf8f2247ecf 100644 --- a/extra/lib/plausible/session/computed_salts.ex +++ b/extra/lib/plausible/session/replay_salts.ex @@ -1,6 +1,6 @@ -defmodule Plausible.Session.ComputedSalts do +defmodule Plausible.Session.ReplaySalts do @moduledoc """ - Cache for computed salts used for replayed events. + Cache for salts used for replayed events. It's not cleaned, however the range of possible values is limited to the number of replayed sessions over the lifetime of the node @@ -39,12 +39,12 @@ defmodule Plausible.Session.ComputedSalts do @spec fetch(module() | atom(), pos_integer()) :: %{previous: nil, current: binary()} def fetch(name \\ __MODULE__, replay_session_id) do - computed_salt = + replay_salt = secret_key_base() |> KeyGenerator.generate(:binary.encode_unsigned(replay_session_id), cache: name) |> binary_part(0, 16) - %{previous: nil, current: computed_salt} + %{previous: nil, current: replay_salt} end @impl true diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 4d67e729b75e..d408dc7ee5e8 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -195,7 +195,7 @@ defmodule Plausible.Application do Plausible.Ingestion.Counters, Plausible.Session.Salts, on_ee do - Plausible.Session.ComputedSalts + Plausible.Session.ReplaySalts end, Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index c9c58e075827..f42e1afa2c5c 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -389,7 +389,7 @@ defmodule Plausible.Ingestion.Event do on_ee do defp put_salts(%__MODULE__{} = event, _context) do if replay_session_id = event.clickhouse_event_attrs.replay_session_id do - %{event | salts: Plausible.Session.ComputedSalts.fetch(replay_session_id)} + %{event | salts: Plausible.Session.ReplaySalts.fetch(replay_session_id)} else %{event | salts: Plausible.Session.Salts.fetch()} end diff --git a/test/plausible/session/computed_salts_test.exs b/test/plausible/session/computed_salts_test.exs deleted file mode 100644 index 86bf303f62a9..000000000000 --- a/test/plausible/session/computed_salts_test.exs +++ /dev/null @@ -1,52 +0,0 @@ -defmodule Plausible.Session.ComputedSaltsTest do - use Plausible.DataCase, async: false - use Plausible - - on_ee do - alias Plausible.Session.ComputedSalts - - test "agent starts and responds with salt based on provided session id" do - {:ok, _} = ComputedSalts.start_link(name: __MODULE__) - %{current: current, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - assert is_binary(current) - end - - test "agent starts and responds with the same salt for the same session id" do - {:ok, _} = ComputedSalts.start_link(name: __MODULE__) - %{current: current1, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - %{current: current2, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - assert is_binary(current1) - assert current1 == current2 - end - - test "agent starts and responds with a different salt for different session ids" do - {:ok, _} = ComputedSalts.start_link(name: __MODULE__) - %{current: current1, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - %{current: current2, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_124) - assert is_binary(current1) - assert is_binary(current2) - assert current1 != current2 - end - - test "salts are cached and cache can be purged" do - {:ok, _} = ComputedSalts.start_link(name: __MODULE__) - %{current: salt_before_purge, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - %{current: _, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_124) - - assert [_, _] = :ets.tab2list(__MODULE__) - - send(__MODULE__, :purge) - - assert eventually(fn -> - table = :ets.tab2list(__MODULE__) - {table == [], table} - end) - - %{current: salt_after_purge, previous: nil} = ComputedSalts.fetch(__MODULE__, 123_123_123) - - assert [_] = :ets.tab2list(__MODULE__) - - assert salt_before_purge == salt_after_purge - end - end -end diff --git a/test/plausible/session/replay_salts_test.exs b/test/plausible/session/replay_salts_test.exs new file mode 100644 index 000000000000..43cf6ec6cc63 --- /dev/null +++ b/test/plausible/session/replay_salts_test.exs @@ -0,0 +1,52 @@ +defmodule Plausible.Session.ReplaySaltsTest do + use Plausible.DataCase, async: false + use Plausible + + on_ee do + alias Plausible.Session.ReplaySalts + + test "agent starts and responds with salt based on provided session id" do + {:ok, _} = ReplaySalts.start_link(name: __MODULE__) + %{current: current, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + assert is_binary(current) + end + + test "agent starts and responds with the same salt for the same session id" do + {:ok, _} = ReplaySalts.start_link(name: __MODULE__) + %{current: current1, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + %{current: current2, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + assert is_binary(current1) + assert current1 == current2 + end + + test "agent starts and responds with a different salt for different session ids" do + {:ok, _} = ReplaySalts.start_link(name: __MODULE__) + %{current: current1, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + %{current: current2, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_124) + assert is_binary(current1) + assert is_binary(current2) + assert current1 != current2 + end + + test "salts are cached and cache can be purged" do + {:ok, _} = ReplaySalts.start_link(name: __MODULE__) + %{current: salt_before_purge, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + %{current: _, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_124) + + assert [_, _] = :ets.tab2list(__MODULE__) + + send(__MODULE__, :purge) + + assert eventually(fn -> + table = :ets.tab2list(__MODULE__) + {table == [], table} + end) + + %{current: salt_after_purge, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) + + assert [_] = :ets.tab2list(__MODULE__) + + assert salt_before_purge == salt_after_purge + end + end +end From 714be1e630874e172dd0db2d148c852de820d69d Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 21 Jul 2026 11:23:41 +0200 Subject: [PATCH 31/34] Fix mixup in test names --- test/plausible/session/cache_store_test.exs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs index 0d47f9384837..e07fb28d7a2a 100644 --- a/test/plausible/session/cache_store_test.exs +++ b/test/plausible/session/cache_store_test.exs @@ -291,7 +291,7 @@ defmodule Plausible.Session.CacheStoreTest do end @tag :ee_only - test "updates session counters", %{buffer: buffer} do + test "updates session counters on replayed event", %{buffer: buffer} do timestamp = DateTime.utc_now() event1 = @@ -316,8 +316,7 @@ defmodule Plausible.Session.CacheStoreTest do assert session.replay_session_id == 456 end - @tag :ee_only - test "updates session counters for replayed event", %{buffer: buffer} do + test "updates session counters", %{buffer: buffer} do timestamp = DateTime.utc_now() event1 = From 2dfe04468ac4200d136a9acd95b139e873b8e809 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 22 Jul 2026 13:02:32 +0200 Subject: [PATCH 32/34] Add replay session id to hash value instead of generating salt off it --- lib/plausible/ingestion/event.ex | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index f42e1afa2c5c..d8a13fd1742c 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -386,18 +386,8 @@ defmodule Plausible.Ingestion.Event do end end - on_ee do - defp put_salts(%__MODULE__{} = event, _context) do - if replay_session_id = event.clickhouse_event_attrs.replay_session_id do - %{event | salts: Plausible.Session.ReplaySalts.fetch(replay_session_id)} - else - %{event | salts: Plausible.Session.Salts.fetch()} - end - end - else - defp put_salts(%__MODULE__{} = event, _context) do - %{event | salts: Plausible.Session.Salts.fetch()} - end + defp put_salts(%__MODULE__{} = event, _context) do + %{event | salts: Plausible.Session.Salts.fetch()} end defp put_user_id(%__MODULE__{} = event, _context) do @@ -578,7 +568,17 @@ defmodule Plausible.Ingestion.Event do user_agent = request.user_agent || "" root_domain = get_root_domain(hostname) - SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain) + replay_session_id = + on_ee do + to_string(request.replay_session_id) + else + "" + end + + SipHash.hash!( + salt, + user_agent <> request.remote_ip <> domain <> root_domain <> replay_session_id + ) end end From f10982400b1925f17fd8238fa76d1760ccde8ad3 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 22 Jul 2026 13:05:44 +0200 Subject: [PATCH 33/34] Remove no longer needed `ReplaySalts` --- extra/lib/plausible/session/replay_salts.ex | 66 -------------------- lib/plausible/application.ex | 3 - test/plausible/session/replay_salts_test.exs | 52 --------------- 3 files changed, 121 deletions(-) delete mode 100644 extra/lib/plausible/session/replay_salts.ex delete mode 100644 test/plausible/session/replay_salts_test.exs diff --git a/extra/lib/plausible/session/replay_salts.ex b/extra/lib/plausible/session/replay_salts.ex deleted file mode 100644 index 5bf8f2247ecf..000000000000 --- a/extra/lib/plausible/session/replay_salts.ex +++ /dev/null @@ -1,66 +0,0 @@ -defmodule Plausible.Session.ReplaySalts do - @moduledoc """ - Cache for salts used for replayed events. - - It's not cleaned, however the range of possible values is limited - to the number of replayed sessions over the lifetime of the node - and sessions IDs can't be provided externally by users. - - Despite that, the cache is still purged every couple hours. - """ - - @purge_interval :timer.seconds(7200) - - use GenServer - - alias Plug.Crypto.KeyGenerator - - @spec start_link(Keyword.t()) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: opts[:name] || __MODULE__) - end - - @impl true - def init(opts) do - name = opts[:name] || __MODULE__ - - ^name = - :ets.new(name, [ - :named_table, - :set, - :public, - {:read_concurrency, true} - ]) - - schedule_purge() - - {:ok, name} - end - - @spec fetch(module() | atom(), pos_integer()) :: %{previous: nil, current: binary()} - def fetch(name \\ __MODULE__, replay_session_id) do - replay_salt = - secret_key_base() - |> KeyGenerator.generate(:binary.encode_unsigned(replay_session_id), cache: name) - |> binary_part(0, 16) - - %{previous: nil, current: replay_salt} - end - - @impl true - def handle_info(:purge, name) do - :ets.delete_all_objects(name) - schedule_purge() - - {:noreply, name} - end - - defp schedule_purge() do - Process.send_after(self(), :purge, @purge_interval) - end - - defp secret_key_base() do - Application.get_env(:plausible, PlausibleWeb.Endpoint) - |> Keyword.fetch!(:secret_key_base) - end -end diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index d408dc7ee5e8..20191fc70189 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -194,9 +194,6 @@ defmodule Plausible.Application do end, Plausible.Ingestion.Counters, Plausible.Session.Salts, - on_ee do - Plausible.Session.ReplaySalts - end, Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer), Supervisor.child_spec(Plausible.Session.WriteBuffer, id: Plausible.Session.WriteBuffer), ReferrerBlocklist, diff --git a/test/plausible/session/replay_salts_test.exs b/test/plausible/session/replay_salts_test.exs deleted file mode 100644 index 43cf6ec6cc63..000000000000 --- a/test/plausible/session/replay_salts_test.exs +++ /dev/null @@ -1,52 +0,0 @@ -defmodule Plausible.Session.ReplaySaltsTest do - use Plausible.DataCase, async: false - use Plausible - - on_ee do - alias Plausible.Session.ReplaySalts - - test "agent starts and responds with salt based on provided session id" do - {:ok, _} = ReplaySalts.start_link(name: __MODULE__) - %{current: current, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - assert is_binary(current) - end - - test "agent starts and responds with the same salt for the same session id" do - {:ok, _} = ReplaySalts.start_link(name: __MODULE__) - %{current: current1, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - %{current: current2, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - assert is_binary(current1) - assert current1 == current2 - end - - test "agent starts and responds with a different salt for different session ids" do - {:ok, _} = ReplaySalts.start_link(name: __MODULE__) - %{current: current1, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - %{current: current2, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_124) - assert is_binary(current1) - assert is_binary(current2) - assert current1 != current2 - end - - test "salts are cached and cache can be purged" do - {:ok, _} = ReplaySalts.start_link(name: __MODULE__) - %{current: salt_before_purge, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - %{current: _, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_124) - - assert [_, _] = :ets.tab2list(__MODULE__) - - send(__MODULE__, :purge) - - assert eventually(fn -> - table = :ets.tab2list(__MODULE__) - {table == [], table} - end) - - %{current: salt_after_purge, previous: nil} = ReplaySalts.fetch(__MODULE__, 123_123_123) - - assert [_] = :ets.tab2list(__MODULE__) - - assert salt_before_purge == salt_after_purge - end - end -end From 2c60dc289cceb18343fd9267ed10a1a03e53947a Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Wed, 22 Jul 2026 14:37:19 +0200 Subject: [PATCH 34/34] Skip rate limiting for replayed events --- lib/plausible/ingestion/event.ex | 9 ++++++++- lib/plausible/site/gate_keeper.ex | 15 ++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index d8a13fd1742c..ff140b3f93b1 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -54,13 +54,20 @@ defmodule Plausible.Ingestion.Event do @spec build_and_buffer(Request.t(), Keyword.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} def build_and_buffer(%Request{domains: domains} = request, context \\ []) do + skip_rate_limit? = + on_ee do + not is_nil(request.replay_session_id) + else + false + end + processed_events = if spam_referrer?(request) do for domain <- domains, do: drop(new(domain, request), :spam_referrer) else Enum.reduce(domains, [], fn domain, acc -> # credo:disable-for-next-line Credo.Check.Refactor.Nesting - case GateKeeper.check(domain) do + case GateKeeper.check(domain, skip_rate_limit?: skip_rate_limit?) do {:allow, site} -> processed = domain diff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.ex index 67f79d2790bc..0cac4c1b2706 100644 --- a/lib/plausible/site/gate_keeper.ex +++ b/lib/plausible/site/gate_keeper.ex @@ -44,11 +44,16 @@ defmodule Plausible.Site.GateKeeper do with %Site{team: %{accept_traffic_until: accept_traffic_until}} = site <- Cache.get(domain, Keyword.get(opts, :cache_opts, [])), true <- Plausible.Sites.regular?(site) do - if not is_nil(accept_traffic_until) and - Date.after?(Date.utc_today(), accept_traffic_until) do - :payment_required - else - check_rate_limit(site, opts) + cond do + not is_nil(accept_traffic_until) and + Date.after?(Date.utc_today(), accept_traffic_until) -> + :payment_required + + opts[:skip_rate_limit?] -> + {:allow, site} + + true -> + check_rate_limit(site, opts) end else _ ->