From b8b58d821ddc654ecea1ce24b19850dd39ad9581 Mon Sep 17 00:00:00 2001 From: derIckeBrln Date: Thu, 23 May 2024 21:08:52 +0200 Subject: [PATCH] Debug information on nist requests (#290) * Update api_base_class.py Debug Logging * Update nvd_nist_api.py Debug Information Logging * Update api_base_class.py Log-Level for debugging information * Update api_base_class.py Black formatting * Update nvd_nist_api.py Loglevel for debug information * Update api_base_class.py Typofix --- CveXplore/core/api_base_class.py | 33 +++++++++---- CveXplore/core/nvd_nist/nvd_nist_api.py | 61 +++++++++++++++++++------ 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/CveXplore/core/api_base_class.py b/CveXplore/core/api_base_class.py index c5618983..43835d66 100644 --- a/CveXplore/core/api_base_class.py +++ b/CveXplore/core/api_base_class.py @@ -89,17 +89,25 @@ def _connect( request_api_resource.update(self.kwargs) + full_url = self._build_url(resource) + self.logger.debug(f"Making {method} request to URL: {full_url}") + self.logger.debug(f"Request headers: {request_api_resource['headers']}") + self.logger.debug(f"Request data: {data}") + try: if method == self.methods.POST: - r = session.post(self._build_url(resource), **request_api_resource) + r = session.post(full_url, **request_api_resource) elif method == self.methods.PUT: - r = session.put(self._build_url(resource), **request_api_resource) + r = session.put(full_url, **request_api_resource) elif method == self.methods.PATCH: - r = session.patch(self._build_url(resource), **request_api_resource) + r = session.patch(full_url, **request_api_resource) elif method == self.methods.DELETE: - r = session.delete(self._build_url(resource), **request_api_resource) + r = session.delete(full_url, **request_api_resource) else: - r = session.get(self._build_url(resource), **request_api_resource) + r = session.get(full_url, **request_api_resource) + + self.logger.debug(f"Received response: {r.status_code}") + self.logger.debug(f"Response content: {r.text}") try: if isinstance(r, Response): @@ -118,9 +126,11 @@ def _connect( return the_response except requests.exceptions.ConnectionError as err: - raise requests.exceptions.ConnectionError(err) + self.logger.error(f"Connection error occurred: {err}") + raise except Exception as err: - raise Exception(err) + self.logger.error(f"An error occurred: {err}") + raise def get_session( self, @@ -156,8 +166,8 @@ def call( ) -> dict: """ Method for requesting free format api resources - """ + self.logger.debug(f"API call with method: {method}, RESOURCE: {resource}") try: with self.get_session() as session: result = self._connect( @@ -168,10 +178,13 @@ def call( timeout=timeout, return_response_object=return_response_object, ) + self.logger.debug(f"Received API response: {result}") return result - except requests.ConnectionError: + except requests.ConnectionError as conn_err: + self.logger.error(f"Connection error occurred: {conn_err}") raise - except Exception: + except Exception as e: + self.logger.error(f"Error during API call: {e}") raise @property diff --git a/CveXplore/core/nvd_nist/nvd_nist_api.py b/CveXplore/core/nvd_nist/nvd_nist_api.py index 0a05c6d2..dc76b26f 100644 --- a/CveXplore/core/nvd_nist/nvd_nist_api.py +++ b/CveXplore/core/nvd_nist/nvd_nist_api.py @@ -10,6 +10,8 @@ from json import JSONDecodeError from urllib.parse import urlencode +from requests.exceptions import RequestException, HTTPError + import aiohttp as aiohttp import requests from aiohttp import ContentTypeError @@ -81,19 +83,22 @@ def _build_url(self, resource: dict = None, data: int = 1) -> str: resource = urlencode(resource) if data == self.datasource.CVE: if self.filter_rejected: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}" else: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}" else: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}" else: if data == self.datasource.CVE: if self.filter_rejected: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected" else: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/" else: - return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/" + url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/" + + self.logger.debug(f"Built URL: {url}") + return url def _connect( self, @@ -113,11 +118,27 @@ def _connect( "proxies": self.proxies, } + full_url = self._build_url(resource, data=data) + self.logger.debug(f"Sending {method} request to URL: {full_url}") + self.logger.debug(f"Request headers: {request_api_resource['headers']}") + self.logger.debug(f"Request data: {data}") + try: - self.logger.debug(f"Sending request: resource={resource}, data={data}") - r = session.get( - self._build_url(resource, data=data), **request_api_resource - ) + if method == "GET": + r = session.get(full_url, **request_api_resource) + elif method == "POST": + r = session.post(full_url, data=data, **request_api_resource) + elif method == "PUT": + r = session.put(full_url, data=data, **request_api_resource) + elif method == "PATCH": + r = session.patch(full_url, data=data, **request_api_resource) + elif method == "DELETE": + r = session.delete(full_url, **request_api_resource) + else: + raise ValueError(f"Unsupported method: {method}") + + self.logger.debug(f"Received response: {r.status_code}") + self.logger.debug(f"Response content: {r.text}") try: if isinstance(r, Response): @@ -146,8 +167,10 @@ def _connect( return the_response except requests.exceptions.ConnectionError as err: + self.logger.error(f"Connection error occurred: {err}") raise requests.exceptions.ConnectionError(err) except Exception as err: + self.logger.error(f"An error occurred: {err}") raise Exception(err) def __repr__(self): @@ -244,17 +267,29 @@ def get_count( last_mod_end_date=last_mod_end_date, ) - self.logger.debug(f"Getting count for datasource: {datasource}") + self.logger.info(f"Getting count for datasource: {datasource}") try: ret_data = self.call(self.methods.GET, resource=resource, data=datasource) if not isinstance(ret_data, Response): + self.logger.debug(f"API response data: {ret_data}") return ret_data["totalResults"] else: + self.logger.error(f"Unexpected response type: {ret_data}") raise ApiDataRetrievalFailed(resource) - except Exception: - raise ApiMaxRetryError + except HTTPError as http_err: + self.logger.error(f"HTTP error occurred: {http_err}") + raise ApiMaxRetryError from http_err + except RequestException as req_err: + self.logger.error(f"Request exception occurred: {req_err}") + raise ApiMaxRetryError from req_err + except ApiDataRetrievalFailed as data_err: + self.logger.error(f"Data retrieval error: {data_err}") + raise + except Exception as err: + self.logger.error(f"An unexpected error occurred: {err}") + raise ApiMaxRetryError from err def get_all_data( self,