Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix fails.csv generation - missing eol #15

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix fails.csv generation - missing eol #15

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d127314
fix fails.csv generation - missing eol
jefimm Jan 25, 2020
18b7319
split files to directories
jefimm Jan 27, 2020
87e12c8
handle rate limiting
jefimm Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions axeman/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import argparse
import asyncio
import time
from collections import deque

import uvloop
Expand All @@ -25,7 +26,7 @@

from . import certlib

DOWNLOAD_CONCURRENCY = 50
DOWNLOAD_CONCURRENCY = 5
MAX_QUEUE_SIZE = 1000

async def download_worker(session, log_info, work_deque, download_queue):
Expand All @@ -37,17 +38,29 @@ async def download_worker(session, log_info, work_deque, download_queue):

logging.debug("[{}] Queueing up blocks {}-{}...".format(log_info['url'], start, end))

for x in range(3):
for x in range(30):
try:
async with session.get(certlib.DOWNLOAD.format(log_info['url'], start, end)) as response:
if response.status == 429:
logging.info("got http status {}-{}".format(x, response.status))
time.sleep(min(x + 1, 10))
continue
if response.content_type != 'application/json':
text = await response.text()
logging.error("got response {}-{}-{}".format(x, response.content_type, text))
entry_list = await response.json()
logging.debug("[{}] Retrieved blocks {}-{}...".format(log_info['url'], start, end))
if entry_list.get('error_code', '') == 'rate_limited':
logging.info("{}-{}".format(x, entry_list))
time.sleep(min(x + 1, 10))
continue
break
except Exception as e:
logging.error("Exception getting block {}-{}! {}".format(start, end, e))
time.sleep(min(x+1, 10))
else: # Notorious for else, if we didn't encounter a break our request failed 3 times D:
with open('/tmp/fails.csv', 'a') as f:
f.write(",".join([log_info['url'], str(start), str(end)]))
f.write(",".join([log_info['url'], str(start), str(end)])+"\n")
return

for index, entry in zip(range(start, end + 1), entry_list['entries']):
Expand Down Expand Up @@ -143,9 +156,14 @@ async def processing_coro(download_results_queue, output_dir="/tmp"):

logging.debug("Got a chunk of {}. Mapping into process pool".format(process_pool.pool_workers))


for entry in entries_iter:
csv_storage = '{}/certificates/{}'.format(output_dir, entry['log_info']['url'].replace('/', '_'))
dir_index = 0
for x in entry['entries']:
dir_index = max(dir_index, int(x['cert_index']/1000000))

csv_storage = '{}/certificates/{}/{}'.format(output_dir,
entry['log_info']['url'].replace('/', '_'),
str(dir_index))
if not os.path.exists(csv_storage):
print("[{}] Making dir...".format(os.getpid()))
os.makedirs(csv_storage)
Expand Down Expand Up @@ -272,7 +290,8 @@ def main():

parser.add_argument('-v', dest="verbose", action="store_true", help="Print out verbose/debug info")

parser.add_argument('-c', dest='concurrency_count', action='store', default=50, type=int, help="The number of concurrent downloads to run at a time")
parser.add_argument('-c', dest='concurrency_count', action='store', default=DOWNLOAD_CONCURRENCY, type=int,
help="The number of concurrent downloads to run at a time, default "+str(DOWNLOAD_CONCURRENCY))

args = parser.parse_args()

Expand Down