Bot-tournaments: nofollow

front_end/src/app/(main)/notebooks/[id]/[[...slug]]/page.tsx

@@ -7,6 +7,7 @@ import { defaultDescription } from "@/constants/metadata";
 import ServerPostsApi from "@/services/api/posts/posts.server";
 import { getValidString } from "@/utils/formatters/string";
 import { getPostLink } from "@/utils/navigation";
+import { getPostSeoMetadata } from "@/utils/questions/metadata";
 
 import IndividualNotebookPage from "./page_compotent";
 
@@ -26,6 +27,7 @@ export async function generateMetadata(props: Props): Promise<Metadata> {
   const parsedDescription = String(file).split("\n")[0];
 
   return {
+    ...getPostSeoMetadata(postData),
     title:
       getValidString(postData.html_metadata_json?.title) ??
       getValidString(postData.short_title) ??

front_end/src/app/(main)/questions/[id]/[[...slug]]/page.tsx

@@ -4,6 +4,7 @@ import { defaultDescription } from "@/constants/metadata";
 import { SearchParams } from "@/types/navigation";
 import { getValidString } from "@/utils/formatters/string";
 import { getPostTitle } from "@/utils/questions/helpers";
+import { getPostSeoMetadata } from "@/utils/questions/metadata";
 
 import IndividualQuestionPage from "./page_component";
 import { cachedGetPostForMetadata } from "./utils/get_post";
@@ -22,6 +23,7 @@ export async function generateMetadata(props: Props): Promise<Metadata> {
   }
   const questionTitle = getPostTitle(postData);
   return {
+    ...getPostSeoMetadata(postData),
     title:
       getValidString(postData.html_metadata_json?.title) ??
       getValidString(postData.short_title) ??

front_end/src/types/post.ts

@@ -166,6 +166,7 @@ type BasePost = {
     title: string;
     description: string;
     image_url: string;
+    canonical_url: string;
   };
 };
 

front_end/src/utils/questions/metadata.ts

@@ -0,0 +1,53 @@
+import "server-only";
+import { Metadata } from "next";
+
+import { Post } from "@/types/post";
+import { BotLeaderboardStatus } from "@/types/projects";
+import { getValidString } from "@/utils/formatters/string";
+import { getPostLink } from "@/utils/navigation";
+import { getPublicSettings } from "@/utils/public_settings.server";
+
+type SeoPost = Parameters<typeof getPostLink>[0] &
+  Pick<Post, "projects" | "html_metadata_json">;
+
+/**
+ * Bot-only posts duplicate their human counterparts, which lets Google pick the
+ * bot version as canonical. Keyed off default_project to match how bot-only
+ * posts are identified elsewhere (see comments/services/feed.py).
+ */
+function isBotsOnlyPost(post: SeoPost) {
+  return (
+    post.projects?.default_project?.bot_leaderboard_status ===
+    BotLeaderboardStatus.BotsOnly
+  );
+}
+
+/**
+ * A canonical pointing at another URL combined with noindex is contradictory:
+ * Google may apply the noindex to the canonical target, deindexing the very
+ * page we're pointing at. So an explicit override always suppresses the
+ * bot-only noindex — never emit both.
+ *
+ * Without an override the canonical is self-referencing: any slug resolves
+ * under the /questions/[id]/[[...slug]] catch-all, so each variant would
+ * otherwise be a separate indexable URL for the same post.
+ */
+export function getPostSeoMetadata(post: SeoPost): Metadata {
+  const canonicalOverride = getValidString(
+    post.html_metadata_json?.canonical_url
+  );
+  if (canonicalOverride) {
+    return { alternates: { canonical: canonicalOverride } };
+  }
+
+  const { PUBLIC_APP_URL } = getPublicSettings();
+  const metadata: Metadata = {
+    alternates: { canonical: `${PUBLIC_APP_URL}${getPostLink(post)}` },
+  };
+
+  if (isBotsOnlyPost(post)) {
+    metadata.robots = { index: false, follow: true };
+  }
+
+  return metadata;
+}

posts/migrations/0019_post_html_metadata_json.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("posts", "0018_notebook_markdown_zh_tw_post_short_title_zh_tw_and_more"),
     ]
@@ -16,7 +15,7 @@ class Migration(migrations.Migration):
             field=models.JSONField(
                 blank=True,
                 default=None,
-                help_text="Custom JSON for HTML meta tags. Supported fields are: title, description, image_url",
+                help_text="Custom JSON for HTML meta tags. Supported fields are: title, description, image_url, canonical_url.<br>canonical_url: absolute URL to index instead of this post. Also disables the automatic noindex on bots-only posts.",
                 null=True,
             ),
         ),

posts/models.py

@@ -657,7 +657,12 @@ class PostStatusChange(models.TextChoices):
     # Whether we should display Post/Notebook on the homepage
     show_on_homepage = models.BooleanField(default=False, db_index=True)
     html_metadata_json = models.JSONField(
-        help_text="Custom JSON for HTML meta tags. Supported fields are: title, description, image_url",
+        help_text=(
+            "Custom JSON for HTML meta tags. Supported fields are: title, description, "
+            "image_url, canonical_url.<br>"
+            "canonical_url: absolute URL to index instead of this post. "
+            "Also disables the automatic noindex on bots-only posts."
+        ),
         null=True,
         blank=True,
         default=None,