Move the use-case index onto Learn so the pilot stories stay reachable, and make share images resolve on the host that serves the page.

Co-authored-by: Cursor <cursoragent@cursor.com>

tests/unit/siteMetadata.test.ts

@@ -146,13 +146,27 @@ describe("originFromHost", () => {
 });
 
 describe("buildRobots", () => {
-  it("disallows crawlers on staging and unknown hosts", () => {
-    expect(buildRobots("staging.communityrule.info")).toEqual({
-      rules: { userAgent: "*", disallow: "/" },
-    });
-    expect(buildRobots("localhost:3000")).toEqual({
-      rules: { userAgent: "*", disallow: "/" },
+  it("disallows crawlers on staging and unknown hosts, and still lets link unfurlers fetch the page", () => {
+    const staging = buildRobots("staging.communityrule.info");
+    expect(staging).toEqual({
+      rules: [
+        { userAgent: "*", disallow: "/" },
+        {
+          userAgent: [
+            "Twitterbot",
+            "facebookexternalhit",
+            "Slackbot",
+            "Discordbot",
+            "LinkedInBot",
+            "TelegramBot",
+            "WhatsApp",
+            "Bluesky",
+          ],
+          allow: "/",
+        },
+      ],
     });
+    expect(buildRobots("localhost:3000")).toEqual(staging);
   });
 
   it("allows production hosts and advertises the sitemap", () => {
@@ -168,10 +182,9 @@ describe("buildRobots", () => {
 
   it("honors SITE_INDEXING overrides", () => {
     process.env.SITE_INDEXING = "false";
-    expect(buildRobots("communityrule.info").rules).toEqual({
-      userAgent: "*",
-      disallow: "/",
-    });
+    expect(buildRobots("communityrule.info").rules).toEqual(
+      buildRobots("staging.communityrule.info").rules,
+    );
 
     process.env.SITE_INDEXING = "true";
     expect(buildRobots("staging.communityrule.info").sitemap).toBe(