add robots.txt

llm_server/workers/inferencer.py

@@ -115,7 +115,7 @@ def worker(backend_url):
         if not selected_model:
             selected_model = backend_info['model']
 
-        logger.debug(f"Starting using {backend_url} and {selected_model}. Online: {backend_info['online']}")
+        logger.debug(f"Starting using {backend_url} and {selected_model}. Online: {backend_info['online']}. Streaming: {do_stream}")
 
         try:
             stream_redis.delete(get_stream_name(worker_id))  # clean up any old streams

server.py

@@ -10,7 +10,7 @@ import sys
 from pathlib import Path
 
 import simplejson as json
-from flask import Flask, jsonify, render_template, request
+from flask import Flask, jsonify, render_template, request, Response
 
 import config
 from llm_server import opts
@@ -168,6 +168,17 @@ def home():
                            )
 
 
+@app.route('/robots.txt')
+def robots():
+    # TODO: have config value to deny all
+    # TODO: https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt
+    t = """User-agent: *
+Allow: /"""
+    r = Response(t)
+    r.headers['Content-Type'] = 'text/plain'
+    return r
+
+
 @app.route('/<first>')
 @app.route('/<first>/<path:rest>')
 def fallback(first=None, rest=None):