indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Comment

main
Bob Mottram 2022-03-06 13:26:06 +00:00
parent 73be47e80f
commit 885a2b82c9
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawlers.py
View File

@ -73,7 +73,8 @@ def blocked_user_agent(calling_domain: str, agent_str: str,
agent_domain = None agent_domain = None
if agent_str: if agent_str:
# is this a web crawler? If so the block it # is this a web crawler? If so then block it by default
# unless this is a news instance or if it is in the allowed list
if 'bot/' in agent_str_lower or 'bot-' in agent_str_lower: if 'bot/' in agent_str_lower or 'bot-' in agent_str_lower:
# if this is a news instance then we want it # if this is a news instance then we want it
# to be indexed by search engines # to be indexed by search engines