From b210723de1db3b7eb9fe832e337bd225e5595c1f Mon Sep 17 00:00:00 2001 From: Deven Patel Date: Fri, 26 Jan 2024 11:04:25 +0530 Subject: [PATCH] [Improvement] add default user-agent header in webpage loader (#1219) --- embedchain/loaders/web_page.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/embedchain/loaders/web_page.py b/embedchain/loaders/web_page.py index e69bcbb523..71ad4e1410 100644 --- a/embedchain/loaders/web_page.py +++ b/embedchain/loaders/web_page.py @@ -22,7 +22,10 @@ class WebPageLoader(BaseLoader): def load_data(self, url): """Load data from a web page using a shared requests' session.""" - response = self._session.get(url, timeout=30) + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36", # noqa:E501 + } + response = self._session.get(url, headers=headers, timeout=30) response.raise_for_status() data = response.content content = self._get_clean_content(data, url)