feat: block crawlers from production webapp

backend/bot/app/web/webapp/_runtime.py

@@ -107,6 +107,33 @@ WEBAPP_CSRF_COOKIE_NAME = "rw_webapp_csrf"
 WEBAPP_TELEGRAM_OAUTH_STATE_COOKIE_NAME = "rw_tg_oauth_state"
 WEBAPP_CSRF_HEADER_NAME = "X-CSRF-Token"
 WEBAPP_STATE_CHANGING_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
+ROBOTS_TX = """User-agent: *
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+"""
 _APP_VERSION_CACHE: Optional[str] = None
 WEBAPP_CSRF_EXEMPT_PATHS = {
     "/api/auth/telegram/nonce",

backend/bot/app/web/webapp/assets.py

@@ -23,6 +23,12 @@ async def health_route(request: web.Request) -> web.Response:
     return web.json_response({"ok": True})
 
 
+async def robots_txt_route(request: web.Request) -> web.Response:
+    response = web.Response(text=ROBOTS_TX, content_type="text/plain")
+    response.headers["Cache-Control"] = "public, max-age=3600"
+    return response
+
+
 async def css_asset_route(request: web.Request) -> web.Response:
     return await _css_asset_route(request, base_name="subscription_webapp")
 
@@ -869,6 +875,7 @@ async def _security_headers_middleware(request: web.Request, handler):
     )
     response.headers.setdefault("Referrer-Policy", "no-referrer")
     response.headers.setdefault("X-Content-Type-Options", "nosniff")
+    response.headers.setdefault("X-Robots-Tag", "noindex, nofollow, noarchive")
     response.headers.setdefault(
         "Permissions-Policy",
         (

backend/bot/app/web/webapp/routes.py

@@ -3,6 +3,7 @@ from ._runtime import *  # noqa: F403,F405
 
 
 def setup_subscription_webapp_routes(app: web.Application) -> None:
+    app.router.add_get("/robots.txt", robots_txt_route)
     app.router.add_get("/", index_route)
     app.router.add_get("/login/password", index_route)
     app.router.add_get("/home", index_route)

deploy/docker/Dockerfile

@@ -130,6 +130,7 @@ LABEL org.opencontainers.image.source="https://github.com/3252a8/remnawave-minis
 
 COPY deploy/docker/frontend/nginx.conf /etc/nginx/conf.d/default.conf
 COPY deploy/docker/frontend/00-startup-banner.sh /docker-entrypoint.d/00-startup-banner.sh
+COPY deploy/docker/frontend/robots.txt /usr/share/nginx/html/robots.txt
 COPY backend/bot/app/web/templates/subscription_webapp.html /usr/share/nginx/html/index.html
 COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.css /usr/share/nginx/html/subscription_webapp.css
 COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.*.css /usr/share/nginx/html/

deploy/docker/frontend/nginx.conf

@@ -23,6 +23,13 @@ server {
         return 200 "ok\n";
     }
 
+    location = /robots.txt {
+        expires 1h;
+        add_header Cache-Control "public, max-age=3600";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
+        try_files /robots.txt =404;
+    }
+
     location /api/ {
         proxy_pass http://backend:8081;
         proxy_http_version 1.1;
@@ -89,23 +96,27 @@ server {
     location ~* ^/subscription_webapp(_admin)?\.(min\.)?[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]\.(css|js)$ {
         expires off;
         add_header Cache-Control "public, max-age=31536000, immutable";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location ~* ^/subscription_webapp(_admin)?\.(css|js)$ {
         expires off;
         add_header Cache-Control "no-cache";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg|webp)$ {
         expires 30d;
         add_header Cache-Control "public";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location / {
         add_header Cache-Control "no-cache";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri /index.html;
     }
 }

deploy/docker/frontend/robots.txt

@@ -0,0 +1,26 @@
+User-agent: *
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /

tests/test_security.py

@@ -661,6 +661,7 @@ class WebAppSecurityTests(unittest.IsolatedAsyncioTestCase):
         self.assertNotIn("'unsafe-eval'", csp)
         self.assertIn("img-src 'self' data: blob: https:;", csp)
         self.assertNotIn("img-src 'self' data: https: http:;", csp)
+        self.assertEqual(response.headers["X-Robots-Tag"], "noindex, nofollow, noarchive")
 
 
 class AdminSettingsSecurityTests(unittest.IsolatedAsyncioTestCase):

tests/test_webapp_route_contract.py

@@ -87,6 +87,7 @@ class WebAppRouteContractTests(unittest.TestCase):
             ("GET", "/auth/telegram/start"): "telegram_oauth_start_route",
             ("GET", "/auth/telegram/callback"): "telegram_oauth_callback_route",
             ("GET", "/health"): "health_route",
+            ("GET", "/robots.txt"): "robots_txt_route",
             ("GET", "/webapp-logo"): "webapp_logo_route",
             ("GET", "/webapp-uploaded-logo/{filename}"): "webapp_uploaded_logo_route",
             ("GET", "/webapp-emoji/{codepoints}/512.{ext}"): "webapp_animated_emoji_route",
@@ -133,6 +134,17 @@ class WebAppRouteContractTests(unittest.TestCase):
         for key, handler_name in expected.items():
             self.assertEqual(routes.get(key), handler_name, key)
 
+    def test_robots_txt_disallows_crawling_webapp(self):
+        response = asyncio.run(subscription_webapp.robots_txt_route(_Request()))
+
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(response.headers["Cache-Control"], "public, max-age=3600")
+        self.assertIn("User-agent: *", response.text)
+        self.assertIn("User-agent: OAI-SearchBot", response.text)
+        self.assertIn("User-agent: GPTBot", response.text)
+        self.assertIn("Disallow: /", response.text)
+
     def test_admin_api_registers_expected_routes(self):
         app = web.Application()