Data Extraction, Training, and Modeling #6
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LSTM Pipeline | |
| on: | |
| push: | |
| branches: | |
| - production | |
| tags: | |
| - '*' | |
| schedule: | |
| - cron: "0 9 * * *" | |
| # 16 - 7 = 9 | |
| jobs: | |
| extraction_train_modeling: | |
| name: Data Extraction, Training, and Modeling | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set global directory | |
| run: git config --global --add safe.directory /github/workspace | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Read pipeline schedule date | |
| id: read_schedule | |
| run: | | |
| SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl) | |
| echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV | |
| - name: Get current date | |
| id: get_date | |
| run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV | |
| - name: Check if dates match | |
| id: date_check | |
| run: | | |
| if [ "$schedule_date" = "$current_date" ]; then | |
| echo "match=true" >> $GITHUB_ENV | |
| else | |
| echo "match=false" >> $GITHUB_ENV | |
| fi | |
| - name: Scraping Yahoo Finance | |
| if: env.match != 'true' | |
| run: | | |
| mkdir datasets | |
| wget https://raw.githubusercontent.com/tebakaja/tebakaja_cryptocurrency/production/postman/symbols.json \ | |
| -O postman/symbols.json | |
| go run scraper.go | |
| - name: Install Libraries | |
| if: env.match != 'true' | |
| run: pip install -r requirements.txt | |
| - name: Modeling and Training | |
| if: env.match != 'true' | |
| run: | | |
| mkdir models | |
| mkdir pickles | |
| mkdir posttrained | |
| # python training.py --algorithm=LSTM | |
| - name: Set Pipeline Schedule | |
| if: env.match != 'true' | |
| run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl | |
| - name: Zip Posttrained, Models, and Pickles | |
| if: env.match != 'true' | |
| run: | | |
| zip -r models.zip models | |
| zip -r pickles.zip pickles | |
| zip -r datasets.zip datasets | |
| zip -r posttrained.zip posttrained | |
| - name: Store Datasets to Google Drive | |
| if: env.match != 'true' | |
| uses: adityak74/google-drive-upload-git-action@main | |
| with: | |
| credentials: ${{ secrets.GDRIVE_CRED }} | |
| filename: datasets.zip | |
| folderId: ${{ secrets.GDRIVE_LSTM_ID }} | |
| name: datasets.zip | |
| overwrite: "true" | |
| - name: Store Models to Google Drive | |
| if: env.match != 'true' | |
| uses: adityak74/google-drive-upload-git-action@main | |
| with: | |
| credentials: ${{ secrets.GDRIVE_CRED }} | |
| filename: models.zip | |
| folderId: ${{ secrets.GDRIVE_LSTM_ID }} | |
| name: models.zip | |
| overwrite: "true" | |
| - name: Store Pickles to Google Drive | |
| if: env.match != 'true' | |
| uses: adityak74/google-drive-upload-git-action@main | |
| with: | |
| credentials: ${{ secrets.GDRIVE_CRED }} | |
| filename: pickles.zip | |
| folderId: ${{ secrets.GDRIVE_LSTM_ID }} | |
| name: pickles.zip | |
| overwrite: "true" | |
| - name: Store Posttrained to Google Drive | |
| if: env.match != 'true' | |
| uses: adityak74/google-drive-upload-git-action@main | |
| with: | |
| credentials: ${{ secrets.GDRIVE_CRED }} | |
| filename: posttrained.zip | |
| folderId: ${{ secrets.GDRIVE_LSTM_ID }} | |
| name: posttrained.zip | |
| overwrite: "true" | |
| - name: Remove Temporarary Files and Directories | |
| if: env.match != 'true' | |
| run: | | |
| rm models.zip | |
| rm pickles.zip | |
| rm datasets.zip | |
| rm posttrained.zip | |
| rm -rf models | |
| rm -rf pickles | |
| rm -rf datasets | |
| rm -rf posttrained | |
| - name: Commit changes | |
| if: env.match != 'true' | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "belajarqywok" | |
| git add -A | |
| git commit -m "Data Extraction, Training, and Modeling" | |
| - name: Push changes | |
| if: env.match != 'true' | |
| uses: ad-m/github-push-action@master | |
| with: | |
| github_token: ${{ secrets.GH_TOKEN }} | |
| branch: production | |
| # model_deployment: | |
| # name: Model Deployment | |
| # runs-on: ubuntu-latest | |
| # needs: extraction_train_modeling | |
| # environment: Production | |
| # env: | |
| # HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| # SPACE_NAME: cryptocurrency_prediction | |
| # HF_USERNAME: qywok | |
| # steps: | |
| # - name: Set global directory | |
| # run: git config --global --add safe.directory /github/workspace | |
| # - uses: actions/checkout@v3 | |
| # with: | |
| # persist-credentials: false | |
| # fetch-depth: 1000 | |
| # - name: Check git status | |
| # run: git status | |
| # - name: Configure git | |
| # run: | | |
| # git config --local user.email "[email protected]" | |
| # git config --local user.name "qywok" | |
| # - name: Pull changes from remote | |
| # run: | | |
| # git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \ | |
| # (git merge --strategy-option theirs) | |
| # - name: Add and commit changes | |
| # run: | | |
| # git add -A | |
| # git diff-index --quiet HEAD || git commit -m "Model Deployment" | |
| # - name: Push to Hugging Face | |
| # run: | | |
| # git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force |