Remove robots.txt

In practice, bots ignore this anyway. Removing it might help with avatar image federation

daemon.py

@@ -984,17 +984,6 @@ class PubServer(BaseHTTPRequestHandler):
             tries += 1
         return False
 
-    def _robotsTxt(self) -> bool:
-        if not self.path.lower().startswith('/robot'):
-            return False
-        msg = 'User-agent: *\nDisallow: /'
-        msg = msg.encode('utf-8')
-        msglen = len(msg)
-        self._set_headers('text/plain; charset=utf-8', msglen,
-                          None, self.server.domainFull, True)
-        self._write(msg)
-        return True
-
     def _hasAccept(self, callingDomain: str) -> bool:
         """Do the http headers have an Accept field?
         """
@@ -13300,14 +13289,6 @@ class PubServer(BaseHTTPRequestHandler):
                            '_GET', 'show about screen done',
                            self.server.debug)
 
-        # send robots.txt if asked
-        if self._robotsTxt():
-            return
-
-        fitnessPerformance(GETstartTime, self.server.fitness,
-                           '_GET', 'robots txt',
-                           self.server.debug)
-
         # the initial welcome screen after first logging in
         if htmlGET and authorized and \
            '/users/' in self.path and self.path.endswith('/welcome'):