From 3541f2f78b7aca098f80786c92a77db13113686a Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Sat, 30 May 2026 21:25:03 +0300 Subject: [PATCH] feat: block crawlers from production webapp --- backend/bot/app/web/webapp/_runtime.py | 27 ++++++++++++++++++++++++++ backend/bot/app/web/webapp/assets.py | 7 +++++++ backend/bot/app/web/webapp/routes.py | 1 + deploy/docker/Dockerfile | 1 + deploy/docker/frontend/nginx.conf | 11 +++++++++++ deploy/docker/frontend/robots.txt | 26 +++++++++++++++++++++++++ tests/test_security.py | 1 + tests/test_webapp_route_contract.py | 12 ++++++++++++ 8 files changed, 86 insertions(+) create mode 100644 deploy/docker/frontend/robots.txt diff --git a/backend/bot/app/web/webapp/_runtime.py b/backend/bot/app/web/webapp/_runtime.py index aad3b35..e223de2 100644 --- a/backend/bot/app/web/webapp/_runtime.py +++ b/backend/bot/app/web/webapp/_runtime.py @@ -107,6 +107,33 @@ WEBAPP_CSRF_COOKIE_NAME = "rw_webapp_csrf" WEBAPP_TELEGRAM_OAUTH_STATE_COOKIE_NAME = "rw_tg_oauth_state" WEBAPP_CSRF_HEADER_NAME = "X-CSRF-Token" WEBAPP_STATE_CHANGING_METHODS = {"POST", "PUT", "PATCH", "DELETE"} +ROBOTS_TX = """User-agent: * +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: OAI-SearchBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: PerplexityBot +Disallow: / + +User-agent: Applebot-Extended +Disallow: / +""" _APP_VERSION_CACHE: Optional[str] = None WEBAPP_CSRF_EXEMPT_PATHS = { "/api/auth/telegram/nonce", diff --git a/backend/bot/app/web/webapp/assets.py b/backend/bot/app/web/webapp/assets.py index 8ce316b..3a656d4 100644 --- a/backend/bot/app/web/webapp/assets.py +++ b/backend/bot/app/web/webapp/assets.py @@ -23,6 +23,12 @@ async def health_route(request: web.Request) -> web.Response: return web.json_response({"ok": True}) +async def robots_txt_route(request: web.Request) -> web.Response: + response = web.Response(text=ROBOTS_TX, content_type="text/plain") + response.headers["Cache-Control"] = "public, max-age=3600" + return response + + async def css_asset_route(request: web.Request) -> web.Response: return await _css_asset_route(request, base_name="subscription_webapp") @@ -869,6 +875,7 @@ async def _security_headers_middleware(request: web.Request, handler): ) response.headers.setdefault("Referrer-Policy", "no-referrer") response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("X-Robots-Tag", "noindex, nofollow, noarchive") response.headers.setdefault( "Permissions-Policy", ( diff --git a/backend/bot/app/web/webapp/routes.py b/backend/bot/app/web/webapp/routes.py index 93b5a3b..c9b7c13 100644 --- a/backend/bot/app/web/webapp/routes.py +++ b/backend/bot/app/web/webapp/routes.py @@ -3,6 +3,7 @@ from ._runtime import * # noqa: F403,F405 def setup_subscription_webapp_routes(app: web.Application) -> None: + app.router.add_get("/robots.txt", robots_txt_route) app.router.add_get("/", index_route) app.router.add_get("/login/password", index_route) app.router.add_get("/home", index_route) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 841df25..21d5085 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -130,6 +130,7 @@ LABEL org.opencontainers.image.source="https://github.com/3252a8/remnawave-minis COPY deploy/docker/frontend/nginx.conf /etc/nginx/conf.d/default.conf COPY deploy/docker/frontend/00-startup-banner.sh /docker-entrypoint.d/00-startup-banner.sh +COPY deploy/docker/frontend/robots.txt /usr/share/nginx/html/robots.txt COPY backend/bot/app/web/templates/subscription_webapp.html /usr/share/nginx/html/index.html COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.css /usr/share/nginx/html/subscription_webapp.css COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.*.css /usr/share/nginx/html/ diff --git a/deploy/docker/frontend/nginx.conf b/deploy/docker/frontend/nginx.conf index 19332a1..00bcfab 100644 --- a/deploy/docker/frontend/nginx.conf +++ b/deploy/docker/frontend/nginx.conf @@ -23,6 +23,13 @@ server { return 200 "ok\n"; } + location = /robots.txt { + expires 1h; + add_header Cache-Control "public, max-age=3600"; + add_header X-Robots-Tag "noindex, nofollow, noarchive"; + try_files /robots.txt =404; + } + location /api/ { proxy_pass http://backend:8081; proxy_http_version 1.1; @@ -89,23 +96,27 @@ server { location ~* ^/subscription_webapp(_admin)?\.(min\.)?[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]\.(css|js)$ { expires off; add_header Cache-Control "public, max-age=31536000, immutable"; + add_header X-Robots-Tag "noindex, nofollow, noarchive"; try_files $uri =404; } location ~* ^/subscription_webapp(_admin)?\.(css|js)$ { expires off; add_header Cache-Control "no-cache"; + add_header X-Robots-Tag "noindex, nofollow, noarchive"; try_files $uri =404; } location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg|webp)$ { expires 30d; add_header Cache-Control "public"; + add_header X-Robots-Tag "noindex, nofollow, noarchive"; try_files $uri =404; } location / { add_header Cache-Control "no-cache"; + add_header X-Robots-Tag "noindex, nofollow, noarchive"; try_files $uri /index.html; } } diff --git a/deploy/docker/frontend/robots.txt b/deploy/docker/frontend/robots.txt new file mode 100644 index 0000000..c993fd3 --- /dev/null +++ b/deploy/docker/frontend/robots.txt @@ -0,0 +1,26 @@ +User-agent: * +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: OAI-SearchBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: PerplexityBot +Disallow: / + +User-agent: Applebot-Extended +Disallow: / diff --git a/tests/test_security.py b/tests/test_security.py index a3548d3..95808ea 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -661,6 +661,7 @@ class WebAppSecurityTests(unittest.IsolatedAsyncioTestCase): self.assertNotIn("'unsafe-eval'", csp) self.assertIn("img-src 'self' data: blob: https:;", csp) self.assertNotIn("img-src 'self' data: https: http:;", csp) + self.assertEqual(response.headers["X-Robots-Tag"], "noindex, nofollow, noarchive") class AdminSettingsSecurityTests(unittest.IsolatedAsyncioTestCase): diff --git a/tests/test_webapp_route_contract.py b/tests/test_webapp_route_contract.py index c8f41ce..3384482 100644 --- a/tests/test_webapp_route_contract.py +++ b/tests/test_webapp_route_contract.py @@ -87,6 +87,7 @@ class WebAppRouteContractTests(unittest.TestCase): ("GET", "/auth/telegram/start"): "telegram_oauth_start_route", ("GET", "/auth/telegram/callback"): "telegram_oauth_callback_route", ("GET", "/health"): "health_route", + ("GET", "/robots.txt"): "robots_txt_route", ("GET", "/webapp-logo"): "webapp_logo_route", ("GET", "/webapp-uploaded-logo/{filename}"): "webapp_uploaded_logo_route", ("GET", "/webapp-emoji/{codepoints}/512.{ext}"): "webapp_animated_emoji_route", @@ -133,6 +134,17 @@ class WebAppRouteContractTests(unittest.TestCase): for key, handler_name in expected.items(): self.assertEqual(routes.get(key), handler_name, key) + def test_robots_txt_disallows_crawling_webapp(self): + response = asyncio.run(subscription_webapp.robots_txt_route(_Request())) + + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual(response.headers["Cache-Control"], "public, max-age=3600") + self.assertIn("User-agent: *", response.text) + self.assertIn("User-agent: OAI-SearchBot", response.text) + self.assertIn("User-agent: GPTBot", response.text) + self.assertIn("Disallow: /", response.text) + def test_admin_api_registers_expected_routes(self): app = web.Application()