diff --git a/.gitignore b/.gitignore index 7723488..1f973c1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ test.py backups/* !backups/.gitkeep wizard.pyc +__pycache__/ diff --git a/README.md b/README.md index bd24e5d..5b8bd6f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ There are shell and bash scripts out there, which were created in order to downl but most of them are not maintained and throwing errors. So, this project is aiming for full backup automation, and therefore this is the features road map: :white_check_mark: Create a script in python -:white_check_mark: Support creating config.json from user input ('wizard') +:white_check_mark: Support creating config.yaml from user input ('wizard') :white_check_mark: Download backup file locally :white_check_mark: Add an option to stream backup file to S3 :white_check_mark: Check how to manually create a cron task on OS X / Linux @@ -16,7 +16,7 @@ but most of them are not maintained and throwing errors. So, this project is aim # Installation ## Prerequisite: -:heavy_plus_sign: python 2.7.x or python 3.x.x +:heavy_plus_sign: python 3 :heavy_plus_sign: [virtualenv](https://pypi.org/project/virtualenv/) installed globally (pip install virtualenv) ## Instructions: diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backup.py b/backup.py index a43add1..e24803a 100644 --- a/backup.py +++ b/backup.py @@ -4,10 +4,9 @@ import os import argparse import requests -import boto -from boto.s3.key import Key +import boto3 +from boto3.s3.transfer import TransferConfig import wizard -from time import gmtime, strftime def read_config(): @@ -38,8 +37,8 @@ def create_confluence_backup(self): time.sleep(self.wait) while 'fileName' not in self.backup_status.keys(): self.backup_status = json.loads(self.session.get(confluence_backup_status).text) - print('Current status: {progress}; {description}'.format( - progress=self.backup_status['alternativePercentage'], + print('Current status: {progress}; {description}'.format( + progress=self.backup_status['alternativePercentage'], description=self.backup_status['currentStatus'])) time.sleep(self.wait) return 'https://{url}/wiki/download/{file_name}'.format( @@ -58,8 +57,8 @@ def create_jira_backup(self): while 'result' not in self.backup_status.keys(): self.backup_status = json.loads(self.session.get(jira_backup_status).text) print('Current status: {status} {progress}; {description}'.format( - status=self.backup_status['status'], - progress=self.backup_status['progress'], + status=self.backup_status['status'], + progress=self.backup_status['progress'], description=self.backup_status['description'])) time.sleep(self.wait) return '{prefix}/{result_id}'.format( @@ -79,22 +78,40 @@ def stream_to_s3(self, url, remote_filename): print('-> Streaming to S3') if self.config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'] == '': - connect = boto.connect_s3() + s3_client = boto3.client('s3') else: - connect = boto.connect_s3( - aws_access_key_id=self.config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'], - aws_secret_access_key=self.config['UPLOAD_TO_S3']['AWS_SECRET_KEY'] - ) - - bucket = connect.get_bucket(self.config['UPLOAD_TO_S3']['S3_BUCKET']) + s3_client = boto3.client( + 's3', + aws_access_key_id=self.config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'], + aws_secret_access_key=self.config['UPLOAD_TO_S3']['AWS_SECRET_KEY'], + region_name=self.config['UPLOAD_TO_S3']['AWS_REGION'], + endpoint_url=self.config['UPLOAD_TO_S3']['AWS_ENDPOINT_URL'], + use_ssl=self.config['UPLOAD_TO_S3']['AWS_IS_SECURE'] + ) + + bucket_name = self.config['UPLOAD_TO_S3']['S3_BUCKET'] r = self.session.get(url, stream=True) if r.status_code == 200: - k = Key(bucket) - k.key = remote_filename - k.content_type = r.headers['content-type'] - k.set_contents_from_string(r.content) - return - + key = "{s3_bucket}{s3_filename}".format( + s3_bucket=self.config['UPLOAD_TO_S3']['S3_DIR'], + s3_filename=remote_filename + ) + + content_length = int(r.headers.get('Content-Length', 0)) + + config = TransferConfig( + multipart_threshold=content_length + 1, + max_concurrency=1, + use_threads=False + ) + + s3_client.upload_fileobj( + Fileobj=r.raw, + Bucket=bucket_name, + Key=key, + ExtraArgs={'ContentType': r.headers['content-type']}, + Config=config + ) if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/config.yaml b/config.yaml index 961fa7a..c47b396 100644 --- a/config.yaml +++ b/config.yaml @@ -4,7 +4,11 @@ USER_EMAIL: "email address for the Atlassian account you're using to create the API_TOKEN: "token ID generated at https://id.atlassian.com/manage/api-tokens" INCLUDE_ATTACHMENTS: "include attachments? this will make the backup size bigger - true / false" DOWNLOAD_LOCALLY: "download the backup file to backups folder? true / false" -UPLOAD_TO_S3: - S3_BUCKET: "S3 bucket name (empty value will skip this step)" +UPLOAD_TO_S3: + AWS_ENDPOINT_URL: "amazon S3 endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html" + AWS_REGION: "amazon S3 region" + S3_BUCKET: "S3 bucket name (empty value will skip this step)" + S3_DIR: "S3 directory for upload (example Atlassian/)" AWS_ACCESS_KEY: "not mandatory if already set on the machine with AWS CLI" - AWS_SECRET_KEY: "not mandatory if already set on the machine with AWS CLI" \ No newline at end of file + AWS_SECRET_KEY: "not mandatory if already set on the machine with AWS CLI" + AWS_IS_SECURE: True \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cbe0492..2a1659f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -boto==2.48.0 -certifi==2018.4.16 -chardet==3.0.4 -idna==2.6 -requests==2.20.0 -urllib3==1.24.2 -PyYAML==5.3.1 \ No newline at end of file +boto3==1.35.26 +PyYAML==6.0.2 +Requests==2.32.3 diff --git a/wizard.py b/wizard.py index 5b4780d..f0eb6e4 100644 --- a/wizard.py +++ b/wizard.py @@ -1,31 +1,45 @@ import os -import json +import yaml def create_config(): - jira_host = raw_input("What is your Jira host name? ") - user = raw_input("What is your Jira account email address? ") - password = raw_input("Paste your Jira API token: ") - attachments = raw_input("Do you want to include attachments? (true / false) ") - download_locally = raw_input("Do you want to download the backup file locally? (true / false) ") + jira_host = input("What is your Jira host name? ") + user = input("What is your Jira account email address? ") + password = input("Paste your Jira API token: ") + attachments = input("Do you want to include attachments? (true / false) ") + download_locally = input("Do you want to download the backup file locally? (true / false) ") + custom_config = { - 'JIRA_HOST': jira_host, - 'INCLUDE_ATTACHMENTS': attachments.lower(), - 'JIRA_EMAIL': user, + 'HOST_URL': jira_host, + 'USER_EMAIL': user, 'API_TOKEN': password, + 'INCLUDE_ATTACHMENTS': attachments.lower(), 'DOWNLOAD_LOCALLY': download_locally.lower(), 'UPLOAD_TO_S3': { + 'AWS_ENDPOINT_URL': "", + 'AWS_REGION': "", 'S3_BUCKET': "", + 'S3_DIR': "", 'AWS_ACCESS_KEY': "", - 'AWS_SECRET_KEY': "" + 'AWS_SECRET_KEY': "", + 'AWS_IS_SECURE': True } } - upload_backup = raw_input("Do you want to upload the backup file to S3? (true / false) ") + + upload_backup = input("Do you want to upload the backup file to S3? (true / false) ") if upload_backup.lower() == 'true': - custom_config['UPLOAD_TO_S3']['S3_BUCKET'] = raw_input("What is the S3 bucket name? ") - custom_config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'] = raw_input("What is your AWS access key? ") - custom_config['UPLOAD_TO_S3']['AWS_SECRET_KEY'] = raw_input("What is your AWS secret key? ") + custom_config['UPLOAD_TO_S3']['AWS_ENDPOINT_URL'] = input("What is your AWS endpoint url? ") + custom_config['UPLOAD_TO_S3']['AWS_REGION'] = input("What is your AWS region? ") + custom_config['UPLOAD_TO_S3']['S3_BUCKET'] = input("What is the S3 bucket name? ") + custom_config['UPLOAD_TO_S3']['S3_DIR'] = input("What is the S3 directory for upload? (example Atlassian/) ") + custom_config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'] = input("What is your AWS access key? ") + custom_config['UPLOAD_TO_S3']['AWS_SECRET_KEY'] = input("What is your AWS secret key? ") + custom_config['UPLOAD_TO_S3']['AWS_IS_SECURE'] = input("Do you want to use SSL? (true / false) ") - config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json') + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.yaml') with open(config_path, 'w+') as config_file: - json.dump(custom_config, config_file) + yaml.dump(custom_config, config_file, default_flow_style=False) + + +if __name__ == "__main__": + create_config()