feature: Phase 23 Track A — env-gated /robots.txt + /sitemap.xml public crawl endpoints

2026-06-23 07:23:42 -04:00
parent 9a4b79d377
commit 5f4807cc4a
6 changed files with 432 additions and 0 deletions
@@ -0,0 +1,34 @@
+namespace DeepDrftPublic.Seo;
+
+/// <summary>
+/// Pure composition of the <c>robots.txt</c> body (Phase 23 wave 23.1). The environment gate is the
+/// caller's: the endpoint reads <see cref="Microsoft.AspNetCore.Hosting.IWebHostEnvironment.IsProduction"/>
+/// server-side and passes the boolean here, so the production-vs-beta branch lives in one testable place.
+/// Fail-safe is closed — anything that is not Production yields <c>Disallow: /</c> (Invariant E1).
+/// </summary>
+public static class RobotsTxt
+{
+    /// <summary>
+    /// Builds the directive body. In Production: allow everything except the embed shell and the proxy API
+    /// paths, plus a <c>Sitemap:</c> pointer (OQ-R2). In any non-production environment: a closed door
+    /// (<c>Disallow: /</c>) with no sitemap pointer, so a crawl of beta sees nothing and the sitemap is
+    /// never advertised.
+    /// </summary>
+    /// <param name="isProduction">The server-side <c>IsProduction()</c> result — the single gate.</param>
+    /// <param name="baseUrl">Canonical origin (no trailing slash) for the <c>Sitemap:</c> line; Production only.</param>
+    public static string Build(bool isProduction, string baseUrl)
+    {
+        if (!isProduction)
+        {
+            return "User-agent: *\n" +
+                   "Disallow: /\n";
+        }
+
+        var origin = baseUrl.TrimEnd('/');
+        return "User-agent: *\n" +
+               "Allow: /\n" +
+               "Disallow: /FramePlayer\n" +
+               "Disallow: /api/\n" +
+               $"Sitemap: {origin}/sitemap.xml\n";
+    }
+}