Skip to content

Commit

Permalink
Debug information on nist requests (#290)
Browse files Browse the repository at this point in the history 
* Update api_base_class.py

Debug Logging

* Update nvd_nist_api.py

Debug Information Logging

* Update api_base_class.py

Log-Level for debugging information

* Update api_base_class.py

Black formatting

* Update nvd_nist_api.py

Loglevel for debug information

* Update api_base_class.py

Typofix
  • Loading branch information
@derIckeBrln
derIckeBrln authored May 23, 2024
1 parent 5a2c7ae commit b8b58d8
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 23 deletions.
33 changes: 23 additions & 10 deletions CveXplore/core/api_base_class.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,25 @@ def _connect(

request_api_resource.update(self.kwargs)

full_url = self._build_url(resource)
self.logger.debug(f"Making {method} request to URL: {full_url}")
self.logger.debug(f"Request headers: {request_api_resource['headers']}")
self.logger.debug(f"Request data: {data}")

try:
if method == self.methods.POST:
r = session.post(self._build_url(resource), **request_api_resource)
r = session.post(full_url, **request_api_resource)
elif method == self.methods.PUT:
r = session.put(self._build_url(resource), **request_api_resource)
r = session.put(full_url, **request_api_resource)
elif method == self.methods.PATCH:
r = session.patch(self._build_url(resource), **request_api_resource)
r = session.patch(full_url, **request_api_resource)
elif method == self.methods.DELETE:
r = session.delete(self._build_url(resource), **request_api_resource)
r = session.delete(full_url, **request_api_resource)
else:
r = session.get(self._build_url(resource), **request_api_resource)
r = session.get(full_url, **request_api_resource)

self.logger.debug(f"Received response: {r.status_code}")
self.logger.debug(f"Response content: {r.text}")

try:
if isinstance(r, Response):
Expand All @@ -118,9 +126,11 @@ def _connect(

return the_response
except requests.exceptions.ConnectionError as err:
raise requests.exceptions.ConnectionError(err)
self.logger.error(f"Connection error occurred: {err}")
raise
except Exception as err:
raise Exception(err)
self.logger.error(f"An error occurred: {err}")
raise

def get_session(
self,
Expand Down Expand Up @@ -156,8 +166,8 @@ def call(
) -> dict:
"""
Method for requesting free format api resources
"""
self.logger.debug(f"API call with method: {method}, RESOURCE: {resource}")
try:
with self.get_session() as session:
result = self._connect(
Expand All @@ -168,10 +178,13 @@ def call(
timeout=timeout,
return_response_object=return_response_object,
)
self.logger.debug(f"Received API response: {result}")
return result
except requests.ConnectionError:
except requests.ConnectionError as conn_err:
self.logger.error(f"Connection error occurred: {conn_err}")
raise
except Exception:
except Exception as e:
self.logger.error(f"Error during API call: {e}")
raise

@property
Expand Down
61 changes: 48 additions & 13 deletions CveXplore/core/nvd_nist/nvd_nist_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from json import JSONDecodeError
from urllib.parse import urlencode

from requests.exceptions import RequestException, HTTPError

import aiohttp as aiohttp
import requests
from aiohttp import ContentTypeError
Expand Down Expand Up @@ -81,19 +83,22 @@ def _build_url(self, resource: dict = None, data: int = 1) -> str:
resource = urlencode(resource)
if data == self.datasource.CVE:
if self.filter_rejected:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected&{resource}"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?{resource}"
else:
if data == self.datasource.CVE:
if self.filter_rejected:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/?noRejected"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
else:
return f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"
url = f"{self.baseurl}/rest/json/{self.datasource_mapping[data]}/{self.api_path}/"

self.logger.debug(f"Built URL: {url}")
return url

def _connect(
self,
Expand All @@ -113,11 +118,27 @@ def _connect(
"proxies": self.proxies,
}

full_url = self._build_url(resource, data=data)
self.logger.debug(f"Sending {method} request to URL: {full_url}")
self.logger.debug(f"Request headers: {request_api_resource['headers']}")
self.logger.debug(f"Request data: {data}")

try:
self.logger.debug(f"Sending request: resource={resource}, data={data}")
r = session.get(
self._build_url(resource, data=data), **request_api_resource
)
if method == "GET":
r = session.get(full_url, **request_api_resource)
elif method == "POST":
r = session.post(full_url, data=data, **request_api_resource)
elif method == "PUT":
r = session.put(full_url, data=data, **request_api_resource)
elif method == "PATCH":
r = session.patch(full_url, data=data, **request_api_resource)
elif method == "DELETE":
r = session.delete(full_url, **request_api_resource)
else:
raise ValueError(f"Unsupported method: {method}")

self.logger.debug(f"Received response: {r.status_code}")
self.logger.debug(f"Response content: {r.text}")

try:
if isinstance(r, Response):
Expand Down Expand Up @@ -146,8 +167,10 @@ def _connect(
return the_response

except requests.exceptions.ConnectionError as err:
self.logger.error(f"Connection error occurred: {err}")
raise requests.exceptions.ConnectionError(err)
except Exception as err:
self.logger.error(f"An error occurred: {err}")
raise Exception(err)

def __repr__(self):
Expand Down Expand Up @@ -244,17 +267,29 @@ def get_count(
last_mod_end_date=last_mod_end_date,
)

self.logger.debug(f"Getting count for datasource: {datasource}")
self.logger.info(f"Getting count for datasource: {datasource}")

try:
ret_data = self.call(self.methods.GET, resource=resource, data=datasource)

if not isinstance(ret_data, Response):
self.logger.debug(f"API response data: {ret_data}")
return ret_data["totalResults"]
else:
self.logger.error(f"Unexpected response type: {ret_data}")
raise ApiDataRetrievalFailed(resource)
except Exception:
raise ApiMaxRetryError
except HTTPError as http_err:
self.logger.error(f"HTTP error occurred: {http_err}")
raise ApiMaxRetryError from http_err
except RequestException as req_err:
self.logger.error(f"Request exception occurred: {req_err}")
raise ApiMaxRetryError from req_err
except ApiDataRetrievalFailed as data_err:
self.logger.error(f"Data retrieval error: {data_err}")
raise
except Exception as err:
self.logger.error(f"An unexpected error occurred: {err}")
raise ApiMaxRetryError from err

def get_all_data(
self,
Expand Down

0 comments on commit b8b58d8

Please sign in to comment.