diff --git a/scripts/invalid b/scripts/invalid index 0adbab1fe..7f3572592 100755 --- a/scripts/invalid +++ b/scripts/invalid @@ -10,4 +10,4 @@ if [ ! -d /etc/${dir_name} ]; then fi fi -journalctl -u ${dir_name} | grep "invalid expanded\|invalid nickname\|invalid header\|invalid characters\|leech bounced\|LLM scraper\|suspicious\|contradictory browsers\|bad path\|not wordpress\|attempt to access passwords\|GET Unknown request\|no actor domain\|no actor nickname\|should be a string\|has no actor\|no valid context\|fields should be strings\|fields should be lists\|should be a list\|which are too long\|does not look like a url\|actor contains local network address" +journalctl -u ${dir_name} | grep "invalid expanded\|invalid nickname\|invalid header\|invalid characters\|leech bounced\|LLM scraper\|suspicious\|contradictory browsers\|bad path\|not wordpress\|attempt to access passwords\|GET Unknown request\|no actor domain\|no actor nickname\|should be a string\|has no actor\|no valid context\|fields should be strings\|fields should be lists\|should be a list\|which are too long\|does not look like a url\|actor contains local network address\|GET no user agent\|POST no user agent" diff --git a/src/daemon_get.py b/src/daemon_get.py index 844f93133..a01a646ad 100644 --- a/src/daemon_get.py +++ b/src/daemon_get.py @@ -389,10 +389,21 @@ def daemon_http_get(self) -> None: ua_str = get_user_agent(self) - if ua_str: - if 'Epicyon/' in ua_str: - log_epicyon_instances(self.server.base_dir, ua_str, - self.server.known_epicyon_instances) + invalid_user_agent: bool = False + if not ua_str: + invalid_user_agent = True + else: + # if user agent is just spaces + if not ua_str.strip(): + invalid_user_agent = True + if invalid_user_agent: + print('GET no user agent ' + str(self.headers).replace('\n', ' ')) + http_400(self) + return + + if 'Epicyon/' in ua_str: + log_epicyon_instances(self.server.base_dir, ua_str, + self.server.known_epicyon_instances) if not _permitted_crawler_path(self.path): block, self.server.blocked_cache_last_updated, llm = \ diff --git a/src/daemon_post.py b/src/daemon_post.py index adcd54bcd..d3ea0e26f 100644 --- a/src/daemon_post.py +++ b/src/daemon_post.py @@ -283,10 +283,21 @@ def daemon_http_post(self) -> None: ua_str = get_user_agent(self) - if ua_str: - if 'Epicyon/' in ua_str: - log_epicyon_instances(self.server.base_dir, ua_str, - self.server.known_epicyon_instances) + invalid_user_agent: bool = False + if not ua_str: + invalid_user_agent = True + else: + # if user agent is just spaces + if not ua_str.strip(): + invalid_user_agent = True + if invalid_user_agent: + print('POST no user agent ' + str(self.headers).replace('\n', ' ')) + http_400(self) + return + + if 'Epicyon/' in ua_str: + log_epicyon_instances(self.server.base_dir, ua_str, + self.server.known_epicyon_instances) block, self.server.blocked_cache_last_updated, _ = \ blocked_user_agent(calling_domain, ua_str,