indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Remove robots.txt 

In practice, bots ignore this anyway. Removing it might help with avatar image federation
merge-requests/30/head
Bob Mottram 2021-10-28 23:22:18 +01:00
parent 1f073d4745
commit 65d75b2e71
1 changed files with 0 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
daemon.py
View File

@ -984,17 +984,6 @@ class PubServer(BaseHTTPRequestHandler):
tries += 1 tries += 1
return False return False
def _robotsTxt(self) -> bool:
if not self.path.lower().startswith('/robot'):
return False
msg = 'User-agent: *\nDisallow: /'
msg = msg.encode('utf-8')
msglen = len(msg)
self._set_headers('text/plain; charset=utf-8', msglen,
None, self.server.domainFull, True)
self._write(msg)
return True
def _hasAccept(self, callingDomain: str) -> bool: def _hasAccept(self, callingDomain: str) -> bool:
"""Do the http headers have an Accept field? """Do the http headers have an Accept field?
""" """
@ -13300,14 +13289,6 @@ class PubServer(BaseHTTPRequestHandler):
'_GET', 'show about screen done', '_GET', 'show about screen done',
self.server.debug) self.server.debug)
# send robots.txt if asked
if self._robotsTxt():
return
fitnessPerformance(GETstartTime, self.server.fitness,
'_GET', 'robots txt',
self.server.debug)
# the initial welcome screen after first logging in # the initial welcome screen after first logging in
if htmlGET and authorized and \ if htmlGET and authorized and \
'/users/' in self.path and self.path.endswith('/welcome'): '/users/' in self.path and self.path.endswith('/welcome'):