indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Reject header used by LLM scrapers

merge-requests/30/head
Bob Mottram 2024-07-21 10:52:03 +01:00
parent a88aecf564
commit 47ec4dfccb
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
daemon_get.py
View File

@ -256,6 +256,12 @@ def daemon_http_get(self) -> None:
http_402(self) http_402(self)
return return
# headers used by LLM scrapers
if 'oai-host-hash' in self.headers:
print('GET HTTP LLM scraper bounced: ' + str(self.headers))
http_402(self)
return
if contains_invalid_chars(str(self.headers)): if contains_invalid_chars(str(self.headers)):
print('GET HTTP headers contain invalid characters ' + print('GET HTTP headers contain invalid characters ' +
str(self.headers)) str(self.headers))

6
daemon_post.py
View File

@ -92,6 +92,12 @@ def daemon_http_post(self) -> None:
self.server.headers_catalog, self.server.headers_catalog,
self.headers) self.headers)
# headers used by LLM scrapers
if 'oai-host-hash' in self.headers:
print('POST HTTP LLM scraper bounced: ' + str(self.headers))
http_402(self)
return
calling_domain = self.server.domain_full calling_domain = self.server.domain_full
if self.headers.get('Host'): if self.headers.get('Host'):
calling_domain = decoded_host(self.headers['Host']) calling_domain = decoded_host(self.headers['Host'])