feat: block crawlers from production webapp

deploy/docker/Dockerfile

@@ -130,6 +130,7 @@ LABEL org.opencontainers.image.source="https://github.com/3252a8/remnawave-minis
 
 COPY deploy/docker/frontend/nginx.conf /etc/nginx/conf.d/default.conf
 COPY deploy/docker/frontend/00-startup-banner.sh /docker-entrypoint.d/00-startup-banner.sh
+COPY deploy/docker/frontend/robots.txt /usr/share/nginx/html/robots.txt
 COPY backend/bot/app/web/templates/subscription_webapp.html /usr/share/nginx/html/index.html
 COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.css /usr/share/nginx/html/subscription_webapp.css
 COPY --from=frontend-builder /app/backend/bot/app/web/templates/subscription_webapp.*.css /usr/share/nginx/html/

deploy/docker/frontend/nginx.conf

@@ -23,6 +23,13 @@ server {
         return 200 "ok\n";
     }
 
+    location = /robots.txt {
+        expires 1h;
+        add_header Cache-Control "public, max-age=3600";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
+        try_files /robots.txt =404;
+    }
+
     location /api/ {
         proxy_pass http://backend:8081;
         proxy_http_version 1.1;
@@ -89,23 +96,27 @@ server {
     location ~* ^/subscription_webapp(_admin)?\.(min\.)?[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]\.(css|js)$ {
         expires off;
         add_header Cache-Control "public, max-age=31536000, immutable";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location ~* ^/subscription_webapp(_admin)?\.(css|js)$ {
         expires off;
         add_header Cache-Control "no-cache";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg|webp)$ {
         expires 30d;
         add_header Cache-Control "public";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri =404;
     }
 
     location / {
         add_header Cache-Control "no-cache";
+        add_header X-Robots-Tag "noindex, nofollow, noarchive";
         try_files $uri /index.html;
     }
 }

deploy/docker/frontend/robots.txt

@@ -0,0 +1,26 @@
+User-agent: *
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /