From 6f31be87fb7183009a49ba7712ef70f910791cd0 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 15:42:28 +0200 Subject: [PATCH 01/13] Fix repo path in semgrep scan, run scan for all the branches --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6bb27aa..afa7df6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,10 +5,7 @@ jobs: parameters: repo_path: type: string - default: aflorithmic/aflr_python_client - default_branch: - type: string - default: main + default: aflorithmic/apiaudio-python semgrep_deployment_id: type: integer default: 283 @@ -27,6 +24,8 @@ jobs: --publish-deployment << parameters.semgrep_deployment_id >> \ --publish-token $SEMGREP_APP_TOKEN + + workflows: semgrep-scan: jobs: From cc883a7f12ed3b662607b53387f2e9aa9317cab7 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 15:43:10 +0200 Subject: [PATCH 02/13] Apply githook to all the files --- README.md | 4 ++-- apiaudio/api_request.py | 6 +++--- apiaudio/api_resources/mastering.py | 3 +-- apiaudio/api_resources/orchestrator.py | 15 ++++++--------- apiaudio/helper_classes.py | 1 + tests/test_config.py | 4 +--- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4f9a78e..0fb2dbe 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ Script methods are: - `scriptText` \* [Required] (string) - Text for your script. A script can contain multiple sections and SSML tags. Learn more about scriptText details [here](https://docs.api.audio/docs/script-2) - `projectName` (string) - The name of your project. Default value is "default" - `moduleName` (string) - The name of your module. Default value is "default" - - `scriptName` (string) - The name of your script. Default value is "default" (max 60 characters) + - `scriptName` (string) - The name of your script. Default value is "default" (max 60 characters) - `scriptId` (string) - Custom identifier for your script. If scriptId parameter is used, then projectName, moduleName and scriptName are required parameters. - `versions` (dictionary) - A dictionary containing different versions of your script text, whereby the key is the version name, and its value is the associated `scriptText`. Version name `v0` is reserved as the default `scriptText`. Default value is "{}" - Example: @@ -788,7 +788,7 @@ Orchestrator methods are: - `scriptText` \* [Required] (str) - Text to synthesize (TTS). - `soundTemplate` (str) - Sound template to use. - `voice` \* [Required] (str) - Name of voice to use. - + - `create_three_sections()` Creates a TTS speech request with 3 sections and adds a sound template to it through mastering. diff --git a/apiaudio/api_request.py b/apiaudio/api_request.py index 38474db..14eb1b4 100644 --- a/apiaudio/api_request.py +++ b/apiaudio/api_request.py @@ -195,9 +195,9 @@ def _expanded_raise_for_status(self, res): def _options_request(cls, url=None): url = url or f"{apiaudio.api_base}{cls.resource_path}" headers = cls._build_header() - + r = requests.options(url=url, headers=headers) - + cls._expanded_raise_for_status(r) - + return r.json() diff --git a/apiaudio/api_resources/mastering.py b/apiaudio/api_resources/mastering.py index 4bec1e7..556b96c 100644 --- a/apiaudio/api_resources/mastering.py +++ b/apiaudio/api_resources/mastering.py @@ -12,7 +12,6 @@ class Mastering(CreatableResource, RetrievableResource, DownloadableResource): loop_status_code = 202 - @classmethod def list_presets(cls): - return cls._get_request(path_param=cls.mastering_preset_list_path) \ No newline at end of file + return cls._get_request(path_param=cls.mastering_preset_list_path) diff --git a/apiaudio/api_resources/orchestrator.py b/apiaudio/api_resources/orchestrator.py index 22d60a4..d3c1fd8 100644 --- a/apiaudio/api_resources/orchestrator.py +++ b/apiaudio/api_resources/orchestrator.py @@ -1,16 +1,16 @@ -from apiaudio.helper_classes import ( - CreatableResource, - HelpResource -) +from apiaudio.helper_classes import CreatableResource, HelpResource + class CreateMediaWithSound(CreatableResource): OBJECT_NAME = "orchestrator" resource_path = "/orchestrator/media_with_sound" + class CreateAudio(CreatableResource): OBJECT_NAME = "orchestrator" resource_path = "/orchestrator/create_audio" + class CreateThreeSections(CreatableResource): OBJECT_NAME = "orchestrator" resource_path = "/orchestrator/create_three_sections" @@ -31,10 +31,7 @@ def create_audio(cls, **args): @classmethod def create_three_sections(cls, **args): return cls.download(cls, CreateThreeSections.create(**args)) - def download(cls, response): - local_filename = cls._download_request( - url=response.get("url"), destination="." - ) - return local_filename \ No newline at end of file + local_filename = cls._download_request(url=response.get("url"), destination=".") + return local_filename diff --git a/apiaudio/helper_classes.py b/apiaudio/helper_classes.py index 44b82e5..5b62753 100644 --- a/apiaudio/helper_classes.py +++ b/apiaudio/helper_classes.py @@ -17,6 +17,7 @@ class HelpResource(APIRequest): def help(cls, **params): return cls._options_request() + class CreatableResource(APIRequest): @classmethod def create(cls, **params): diff --git a/tests/test_config.py b/tests/test_config.py index 082edea..86d6354 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -60,9 +60,7 @@ def test_script_versions(): from .assets import test_versions script = apiaudio.Script.create( - scriptId="test_sdk", - scriptText="hello...", - versions=test_versions + scriptId="test_sdk", scriptText="hello...", versions=test_versions ) assert script.get("availableVersions") == ["v0"] + [ From 2e1fec6ba49df3534f195e2a26f1055f02094093 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 18:22:13 +0200 Subject: [PATCH 03/13] New version-update workflow in CI: test config --- .circleci/config.yml | 49 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index afa7df6..0627f58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,16 @@ version: 2.1 +config_blocks: + only_tags: &only_tags + filters: + branches: + ignore: /.*/ + tags: + only: /^v[0-9]+(\.[0-9]+)*$/ + +orbs: + aws-cli: circleci/aws-cli@3.1 + jobs: semgrep-scan: parameters: @@ -24,10 +35,48 @@ jobs: --publish-deployment << parameters.semgrep_deployment_id >> \ --publish-token $SEMGREP_APP_TOKEN + pip-publish: + docker: + - image: cimg/python:3.8 + environment: + TRIGGER_TAG: << pipeline.git.tag >> + steps: + - run: + name: clone main repo, publish tag that triggered the pipeline + command: | + git clone git@github.com:aflorithmic/apiaudio-python.git + cd apiaudio-python + git checkout $TRIGGER_TAG + pip3 install twine wheel + python3 setup.py sdist bdist_wheel +# python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* + update-ssm: + environment: + TRIGGER_TAG: << pipeline.git.tag >> + executor: aws-cli/default + steps: + - aws-cli/setup: + profile-name: versionUpdate + - run: + name: update SSM parameter with latest SDK version + command: | + aws ssm get-parameter --name /sdk/latest_versions > ssm.json +# NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') +# aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite workflows: semgrep-scan: jobs: - semgrep-scan: context: aws + + version-update: + jobs: + - pip-publish: + <<: *only_tags + + - update-ssm: + context: aws + requires: + - pip-publish From e96bab3c82ce388ea5738c245cfea602406c9dcf Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 18:28:57 +0200 Subject: [PATCH 04/13] Add github.com to known ssh hosts --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0627f58..c818909 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,8 @@ jobs: - run: name: clone main repo, publish tag that triggered the pipeline command: | + mkdir -p ~/.ssh + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts git clone git@github.com:aflorithmic/apiaudio-python.git cd apiaudio-python git checkout $TRIGGER_TAG From c255b4bf45c0813dac7fd15a7c1b8a71bf92dd46 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 18:32:37 +0200 Subject: [PATCH 05/13] Add tag trigger to update-ssm job --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c818909..6fddefd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,7 @@ workflows: <<: *only_tags - update-ssm: + <<: *only_tags context: aws requires: - pip-publish From c1a647eca5408067c9f571897c82b94b4096e6c9 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 18:34:40 +0200 Subject: [PATCH 06/13] Update test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fddefd..334def4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,6 +64,7 @@ jobs: name: update SSM parameter with latest SDK version command: | aws ssm get-parameter --name /sdk/latest_versions > ssm.json + cat ssm.json # NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') # aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite From ae32dd528d4652b3ce766b987fe2b05970341643 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Wed, 4 May 2022 18:38:20 +0200 Subject: [PATCH 07/13] Finalise CI config --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 334def4..7ff584f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,7 @@ jobs: git checkout $TRIGGER_TAG pip3 install twine wheel python3 setup.py sdist bdist_wheel -# python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* + python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* update-ssm: environment: @@ -65,8 +65,8 @@ jobs: command: | aws ssm get-parameter --name /sdk/latest_versions > ssm.json cat ssm.json -# NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') -# aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite + NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') + aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite workflows: semgrep-scan: From e19cb5d67f2d99b9163e6b03912adb666770a81e Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 5 May 2022 10:39:56 +0200 Subject: [PATCH 08/13] add more yaml code --- .circleci/config.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ff584f..9aa0f45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,9 +7,31 @@ config_blocks: ignore: /.*/ tags: only: /^v[0-9]+(\.[0-9]+)*$/ +# ---------------- SLACK CONFIG BLOCK ---------------- + slack_channel: &slack_channel deployments #slack channel for notifications + + slack_basic_fail: &slack_basic_fail #on fail + - slack/notify: + channel: *slack_channel + event: fail + template: basic_fail_1 + + slack_basic_success: &slack_basic_success #on success + - slack/notify: + channel: *slack_channel + event: pass + template: basic_success_1 + + slack_success_deploy: &slack_success_deploy #success tag deploy + - slack/notify: + channel: *slack_channel + event: pass + template: success_tagged_deploy_1 +# ----------------------------------------------------------------- orbs: aws-cli: circleci/aws-cli@3.1 + slack: circleci/slack@4.9.3 jobs: semgrep-scan: @@ -34,6 +56,8 @@ jobs: python -m semgrep_agent \ --publish-deployment << parameters.semgrep_deployment_id >> \ --publish-token $SEMGREP_APP_TOKEN + - <<: *slack_basic_fail + - <<: *slack_success_deploy pip-publish: docker: @@ -52,6 +76,8 @@ jobs: pip3 install twine wheel python3 setup.py sdist bdist_wheel python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* + - <<: *slack_basic_fail + - <<: *slack_success_deploy update-ssm: environment: @@ -67,6 +93,8 @@ jobs: cat ssm.json NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite + - <<: *slack_basic_fail + - <<: *slack_success_deploy workflows: semgrep-scan: From 0d4985a563829f22aafe30fdc2751d5ea1294a2e Mon Sep 17 00:00:00 2001 From: Lars Date: Thu, 5 May 2022 10:43:21 +0200 Subject: [PATCH 09/13] comment one not needed job --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9aa0f45..8ccb9aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,8 +56,9 @@ jobs: python -m semgrep_agent \ --publish-deployment << parameters.semgrep_deployment_id >> \ --publish-token $SEMGREP_APP_TOKEN - - <<: *slack_basic_fail - - <<: *slack_success_deploy + # It runs on every commit and probably will annoy us too much. + # - <<: *slack_basic_fail + # - <<: *slack_success_deploy pip-publish: docker: From 8e5573e38e1da0f5b3ffad2fe6ba44c5d5c3846a Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 5 May 2022 11:18:47 +0200 Subject: [PATCH 10/13] Test: use checkout and run pre-commit --- .circleci/config.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ccb9aa..5813191 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,20 +8,20 @@ config_blocks: tags: only: /^v[0-9]+(\.[0-9]+)*$/ # ---------------- SLACK CONFIG BLOCK ---------------- - slack_channel: &slack_channel deployments #slack channel for notifications + slack_channel: &slack_channel deployments #slack channel for notifications - slack_basic_fail: &slack_basic_fail #on fail + slack_basic_fail: &slack_basic_fail #on fail - slack/notify: channel: *slack_channel event: fail template: basic_fail_1 - slack_basic_success: &slack_basic_success #on success + slack_basic_success: &slack_basic_success #on success - slack/notify: channel: *slack_channel event: pass template: basic_success_1 - + slack_success_deploy: &slack_success_deploy #success tag deploy - slack/notify: channel: *slack_channel @@ -56,7 +56,7 @@ jobs: python -m semgrep_agent \ --publish-deployment << parameters.semgrep_deployment_id >> \ --publish-token $SEMGREP_APP_TOKEN - # It runs on every commit and probably will annoy us too much. + # It runs on every commit and probably will annoy us too much. # - <<: *slack_basic_fail # - <<: *slack_success_deploy @@ -66,17 +66,15 @@ jobs: environment: TRIGGER_TAG: << pipeline.git.tag >> steps: + - checkout - run: name: clone main repo, publish tag that triggered the pipeline command: | - mkdir -p ~/.ssh - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - git clone git@github.com:aflorithmic/apiaudio-python.git - cd apiaudio-python - git checkout $TRIGGER_TAG + make + pre-commit run --all-files pip3 install twine wheel python3 setup.py sdist bdist_wheel - python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* +# python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* - <<: *slack_basic_fail - <<: *slack_success_deploy @@ -91,9 +89,9 @@ jobs: name: update SSM parameter with latest SDK version command: | aws ssm get-parameter --name /sdk/latest_versions > ssm.json - cat ssm.json NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') - aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite + echo $NEW_VALUE +# aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite - <<: *slack_basic_fail - <<: *slack_success_deploy From 08c0f246e69f48d23da63d08e56d9f20c1884861 Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 5 May 2022 11:24:51 +0200 Subject: [PATCH 11/13] Add AWS context, fix precommit hook --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5813191..2bafc40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,7 @@ jobs: name: clone main repo, publish tag that triggered the pipeline command: | make - pre-commit run --all-files + git commit pip3 install twine wheel python3 setup.py sdist bdist_wheel # python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* @@ -104,6 +104,7 @@ workflows: version-update: jobs: - pip-publish: + context: aws <<: *only_tags - update-ssm: From 3e1ccec6d12b2b2b8681e7dc34ce03f18e49839b Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 5 May 2022 11:30:54 +0200 Subject: [PATCH 12/13] No precommit --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bafc40..305aaac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,10 +68,8 @@ jobs: steps: - checkout - run: - name: clone main repo, publish tag that triggered the pipeline + name: publish tag that triggered the pipeline command: | - make - git commit pip3 install twine wheel python3 setup.py sdist bdist_wheel # python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* From 015288efd6d8095bbbf13d813805c0ba78ca7c7a Mon Sep 17 00:00:00 2001 From: Marcin Martinez Date: Thu, 5 May 2022 11:38:01 +0200 Subject: [PATCH 13/13] Slack & jobs working --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 305aaac..6f52904 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ jobs: command: | pip3 install twine wheel python3 setup.py sdist bdist_wheel -# python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* + python3 -m twine upload --username $PIPY_USERNAME --password $PIPY_PASSWORD dist/* - <<: *slack_basic_fail - <<: *slack_success_deploy @@ -89,7 +89,7 @@ jobs: aws ssm get-parameter --name /sdk/latest_versions > ssm.json NEW_VALUE=$(cat ssm.json | python3 -c 'import json,sys,os;obj=json.loads(sys.stdin.read());obj=json.loads(obj["Parameter"]["Value"]);obj["python"]=os.environ["TRIGGER_TAG"];print(obj)') echo $NEW_VALUE -# aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite + aws ssm put-parameter --name /sdk/latest_versions --type "String" --value $NEW_VALUE --overwrite - <<: *slack_basic_fail - <<: *slack_success_deploy