From 230db959130e3fea1017f67a326ac47735d213f4 Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Fri, 8 May 2026 17:31:21 -0700 Subject: [PATCH] [Perf] GetUsers triangle rewrite + partial album index for GetTracks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent fixes for the API's two most-called queries (GetUsers ~287M calls, GetTracks ~268M calls in pg_stat_statements). == GetUsers: rewrite current_user_followee_follow_count Signed-in GetUsers was 700x slower than unsigned (2-3s vs 4ms for 20 users). Drilling in: the entire delta was one personalization subquery — "of the people I follow, how many also follow this user" — at 2,246ms for 20 users. Every other personalization subquery (does_current_user_follow, does_current_user_subscribe, does_follow_current_user, artist_coin_badge) was sub-millisecond. The old shape let Postgres pick a Merge Join that walked the full follower list of the target — 492k-1.9M rows for popular users like @audius — just to intersect with my ~1,752 followees. Rewrite drives the loop from "my followees" (always small) and probes whether each follows the target. The LIMIT 1 OFFSET 0 inside the EXISTS is the same optimization fence used by the feed query (#798): it pins the planner to nested-loop semantics so the plan never flips back to merge join. Verified on the prod read replica (full GetUsers, 20 popular target users, three warm runs each): myId=0 (unsigned) : 4ms -> 2ms (unchanged, sanity) myId=20 (1752 follows) : 2-3s -> 127-155ms (15-20x) myId=755516 (1816 follows) : 2.5s -> 142-157ms (15-18x) End-to-end via local server, /v1/full/users/handle/audius (1.95M followers) with ?user_id=Wem1e: 60-85ms warm. Response shape unchanged; current_user_followee_follow_count returns the same count as before. == GetTracks album_backlink: partial album index The album_backlink subquery does ~200 random playlists_pkey lookups per popular track to filter for `is_album AND is_delete=false AND is_current=true`. ~99.98% of those lookups end up rejected (because most playlists aren't albums). For 50 popular tracks that's 10k heap probes returning 1-2 actual matches. A partial index covering only published-album playlists lets non- album lookups skip the heap entirely — the planner sees no row at the index level and moves on without ever fetching the page. Index size: ~55k rows x ~12 bytes ≈ 700 KB. Built CONCURRENTLY (no ACCESS EXCLUSIVE lock). Expected GetTracks album_backlink portion drops from ~38ms (50 popular tracks, warm) to ~10-15ms — most of GetTracks's "always-on" cost. --- api/dbv1/get_users.sql.go | 32 +++++++++++++------ api/dbv1/queries/get_users.sql | 32 +++++++++++++------ .../0197_playlists_albums_partial_idx.sql | 21 ++++++++++++ 3 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 ddl/migrations/0197_playlists_albums_partial_idx.sql diff --git a/api/dbv1/get_users.sql.go b/api/dbv1/get_users.sql.go index 2f9dc4e2..6b7acd60 100644 --- a/api/dbv1/get_users.sql.go +++ b/api/dbv1/get_users.sql.go @@ -99,19 +99,31 @@ SELECT is_storage_v2, creator_node_endpoint, + -- "Of the people I follow, how many also follow this user?" + -- + -- Drive the loop from my followees (always small — a few thousand at + -- most) and probe whether each follows the target user. The previous + -- shape let Postgres pick a Merge Join that walked the full follower + -- list of the target — for popular target users that's millions of + -- rows. The LIMIT 1 OFFSET 0 inside the EXISTS is the same + -- optimization fence used by the feed query: it pins the planner to + -- nested-loop semantics so the plan never flips to merge join. ( SELECT count(*) - FROM follows f - JOIN ( - SELECT followee_user_id - FROM follows mf - WHERE mf.follower_user_id = $1 - AND mf.is_delete = false - ) mf ON f.follower_user_id = mf.followee_user_id + FROM follows mf WHERE $1 > 0 - AND $1 != u.user_id -- don't compute when viewing own profile - AND f.followee_user_id = u.user_id - AND f.is_delete = false + AND $1 != u.user_id -- don't compute when viewing own profile + AND mf.follower_user_id = $1 + AND mf.is_delete = false + AND EXISTS ( + SELECT 1 FROM ( + SELECT 1 FROM follows f + WHERE f.follower_user_id = mf.followee_user_id + AND f.followee_user_id = u.user_id + AND f.is_delete = false + LIMIT 1 OFFSET 0 + ) x + ) ) AS current_user_followee_follow_count, ( diff --git a/api/dbv1/queries/get_users.sql b/api/dbv1/queries/get_users.sql index e0e709a7..d4bc41a4 100644 --- a/api/dbv1/queries/get_users.sql +++ b/api/dbv1/queries/get_users.sql @@ -83,19 +83,31 @@ SELECT is_storage_v2, creator_node_endpoint, + -- "Of the people I follow, how many also follow this user?" + -- + -- Drive the loop from my followees (always small — a few thousand at + -- most) and probe whether each follows the target user. The previous + -- shape let Postgres pick a Merge Join that walked the full follower + -- list of the target — for popular target users that's millions of + -- rows. The LIMIT 1 OFFSET 0 inside the EXISTS is the same + -- optimization fence used by the feed query: it pins the planner to + -- nested-loop semantics so the plan never flips to merge join. ( SELECT count(*) - FROM follows f - JOIN ( - SELECT followee_user_id - FROM follows mf - WHERE mf.follower_user_id = @my_id - AND mf.is_delete = false - ) mf ON f.follower_user_id = mf.followee_user_id + FROM follows mf WHERE @my_id > 0 - AND @my_id != u.user_id -- don't compute when viewing own profile - AND f.followee_user_id = u.user_id - AND f.is_delete = false + AND @my_id != u.user_id -- don't compute when viewing own profile + AND mf.follower_user_id = @my_id + AND mf.is_delete = false + AND EXISTS ( + SELECT 1 FROM ( + SELECT 1 FROM follows f + WHERE f.follower_user_id = mf.followee_user_id + AND f.followee_user_id = u.user_id + AND f.is_delete = false + LIMIT 1 OFFSET 0 + ) x + ) ) AS current_user_followee_follow_count, ( diff --git a/ddl/migrations/0197_playlists_albums_partial_idx.sql b/ddl/migrations/0197_playlists_albums_partial_idx.sql new file mode 100644 index 00000000..ecb0ab35 --- /dev/null +++ b/ddl/migrations/0197_playlists_albums_partial_idx.sql @@ -0,0 +1,21 @@ +-- Partial index covering only public, undeleted, current album playlists. +-- +-- Speeds up the album_backlink subquery in GetTracks: that subquery does +-- ~200 random `playlists_pkey` lookups per popular track to filter for +-- (is_album=true AND is_delete=false AND is_current=true) — and ~99.98% +-- of those lookups end up rejected by the filter. With this partial index +-- the planner can resolve "is this an album playlist?" without ever +-- touching the playlists heap for non-album rows. +-- +-- Size budget: ~55k matching rows × ~12 bytes ≈ 700 KB. +-- +-- NOTE: intentionally NOT wrapped in BEGIN/COMMIT so that +-- CREATE INDEX CONCURRENTLY can run without holding an ACCESS EXCLUSIVE +-- lock on playlists. IF NOT EXISTS makes the migration idempotent. + +create index concurrently if not exists idx_playlists_albums_published + on playlists (playlist_id) + where is_album = true and is_delete = false and is_current = true; + +comment on index idx_playlists_albums_published is + 'Partial index for GetTracks album_backlink subquery; lets non-album lookups skip the heap entirely.';