refac/enh: web search domain allow/block filter

This commit is contained in:
Timothy Jaeryang Baek 2025-11-16 13:52:09 -05:00
parent 21c0dd93e2
commit ee10f372a0
2 changed files with 21 additions and 2 deletions

View file

@ -2840,6 +2840,7 @@ WEB_SEARCH_DOMAIN_FILTER_LIST = PersistentConfig(
# "wikipedia.com",
# "wikimedia.org",
# "wikidata.org",
# "!stackoverflow.com",
],
)

View file

@ -9,14 +9,32 @@ from pydantic import BaseModel
def get_filtered_results(results, filter_list):
if not filter_list:
return results
# Domains starting without "!" → allowed
allow_list = [d for d in filter_list if not d.startswith("!")]
# Domains starting with "!" → blocked
block_list = [d[1:] for d in filter_list if d.startswith("!")]
filtered_results = []
for result in results:
url = result.get("url") or result.get("link", "") or result.get("href", "")
if not validators.url(url):
continue
domain = urlparse(url).netloc
if any(domain.endswith(filtered_domain) for filtered_domain in filter_list):
# If allow list is non-empty, require domain to match one of them
if allow_list:
if not any(domain.endswith(allowed) for allowed in allow_list):
continue
# Block list always removes matches
if any(domain.endswith(blocked) for blocked in block_list):
continue
filtered_results.append(result)
return filtered_results