feat: block crawlers from production webapp

backend/bot/app/web/webapp/_runtime.py

@@ -107,6 +107,33 @@ WEBAPP_CSRF_COOKIE_NAME = "rw_webapp_csrf"
 WEBAPP_TELEGRAM_OAUTH_STATE_COOKIE_NAME = "rw_tg_oauth_state"
 WEBAPP_CSRF_HEADER_NAME = "X-CSRF-Token"
 WEBAPP_STATE_CHANGING_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
+ROBOTS_TX = """User-agent: *
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+"""
 _APP_VERSION_CACHE: Optional[str] = None
 WEBAPP_CSRF_EXEMPT_PATHS = {
     "/api/auth/telegram/nonce",

backend/bot/app/web/webapp/assets.py

@@ -23,6 +23,12 @@ async def health_route(request: web.Request) -> web.Response:
     return web.json_response({"ok": True})
 
 
+async def robots_txt_route(request: web.Request) -> web.Response:
+    response = web.Response(text=ROBOTS_TX, content_type="text/plain")
+    response.headers["Cache-Control"] = "public, max-age=3600"
+    return response
+
+
 async def css_asset_route(request: web.Request) -> web.Response:
     return await _css_asset_route(request, base_name="subscription_webapp")
 
@@ -869,6 +875,7 @@ async def _security_headers_middleware(request: web.Request, handler):
     )
     response.headers.setdefault("Referrer-Policy", "no-referrer")
     response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("X-Robots-Tag", "noindex, nofollow, noarchive")
     response.headers.setdefault(
         "Permissions-Policy",
         (

backend/bot/app/web/webapp/routes.py

@@ -3,6 +3,7 @@ from ._runtime import *  # noqa: F403,F405
 
 
 def setup_subscription_webapp_routes(app: web.Application) -> None:
+    app.router.add_get("/robots.txt", robots_txt_route)
     app.router.add_get("/", index_route)
     app.router.add_get("/login/password", index_route)
     app.router.add_get("/home", index_route)