Skip to content
LSTM Pipeline

update: add functional testing and repair pipeline #48

Sign in to view logs

update: add functional testing and repair pipeline

update: add functional testing and repair pipeline #48

Workflow file for this run

.github/workflows/lstm_pipeline.yaml at 455424b
name: LSTM Pipeline
on:
push:
branches:
- main
tags:
- '*'
schedule:
- cron: "0 9 * * *"
# 16 - 7 = 9
jobs:
scraping_extracting:
name: Scraping and Extracting
runs-on: ubuntu-latest
environment: Scraping and Extraction
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
lfs: true
persist-credentials: false
fetch-depth: 1
- name: Read pipeline schedule date
id: read_schedule
run: |
SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
- name: Get current date
id: get_date
run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
- name: Check if dates match
id: date_check
run: |
if [ "$schedule_date" = "$current_date" ]; then
echo "match=true" >> $GITHUB_ENV
else
echo "match=false" >> $GITHUB_ENV
fi
- name: Scraping Yahoo Finance
if: env.match != 'true'
run: |
mkdir datasets
go run scraper.go \
--symbols-file=./postman/symbols_test.json
- name: Zip Datasets
if: env.match != 'true'
run: zip -r datasets.zip datasets
- name: Store Datasets to Google Drive
if: env.match != 'true'
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: datasets.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: datasets.zip
overwrite: "true"
- name: Upload Artifact (datasets)
if: env.match != 'true'
uses: actions/upload-artifact@v3
with:
name: datasets
path: datasets.zip
- name: Remove Temporarary Files and Directories
if: env.match != 'true'
run: |
rm datasets.zip
rm -rf datasets
preprocessing_training:
name: Preprocessing and Training
runs-on: ubuntu-latest
needs: scraping_extracting
environment: Preprocessing and Training
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
lfs: true
persist-credentials: false
fetch-depth: 1
- name: Read pipeline schedule date
id: read_schedule
run: |
SCHEDULE_DATE=$(cat schedulers/lstm_schedule.ctl)
echo "schedule_date=${SCHEDULE_DATE}" >> $GITHUB_ENV
- name: Get current date
id: get_date
run: echo "current_date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
- name: Check if dates match
id: date_check
run: |
if [ "$schedule_date" = "$current_date" ]; then
echo "match=true" >> $GITHUB_ENV
else
echo "match=false" >> $GITHUB_ENV
fi
- name: Install Libraries
if: env.match != 'true'
run: pip install -r requirements.txt
- name: Download Artifact (datasets)
if: env.match != 'true'
uses: actions/download-artifact@v3
with:
name: datasets
- name: Modeling and Training
if: env.match != 'true'
run: |
unzip datasets.zip
mkdir models
mkdir pickles
mkdir posttrained
python training.py \
--epochs=200 \
--batchs=32 \
--sequences=5 \
--algorithm=LSTM
- name: Set Pipeline Schedule
if: env.match != 'true'
run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
- name: Zip Posttrained, Models, and Pickles
if: env.match != 'true'
run: |
zip -r models.zip models
zip -r pickles.zip pickles
zip -r posttrained.zip posttrained
- name: Store Models to Google Drive
if: env.match != 'true'
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: models.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: models.zip
overwrite: "true"
- name: Store Pickles to Google Drive
if: env.match != 'true'
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: pickles.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: pickles.zip
overwrite: "true"
- name: Store Posttrained to Google Drive
if: env.match != 'true'
uses: adityak74/google-drive-upload-git-action@main
with:
credentials: ${{ secrets.GDRIVE_LSTM_CRED }}
filename: posttrained.zip
folderId: ${{ secrets.GDRIVE_LSTM_ID }}
name: posttrained.zip
overwrite: "true"
- name: Remove Temporarary Files and Directories
if: env.match != 'true'
run: |
rm models.zip
rm pickles.zip
rm posttrained.zip
rm -rf models
rm -rf pickles
rm -rf posttrained
- name: Commit changes
if: env.match != 'true'
run: |
git config --local user.email "[email protected]"
git config --local user.name "belajarqywok"
git add -A
git commit -m "Data Extraction, Training, and Modeling"
- name: Push changes
if: env.match != 'true'
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GH_TOKEN }}
branch: main
tebakaja_crypto_space-0:
name: crypto-forecast-svc-0
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-0
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "[email protected]"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-1:
name: crypto-forecast-svc-1
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-1
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "[email protected]"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-2:
name: crypto-forecast-svc-2
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-2
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "[email protected]"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-3:
name: crypto-forecast-svc-3
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-3
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "[email protected]"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
tebakaja_crypto_space-4:
name: crypto-forecast-svc-4
runs-on: ubuntu-latest
needs: preprocessing_training
environment: Deployment
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
SPACE_NAME: tebakaja_cryptocurrency_space-4
HF_USERNAME: tebakaja
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1000
- name: Check git status
run: git status
- name: Configure git
run: |
git config --local user.email "[email protected]"
git config --local user.name "qywok"
- name: Pull changes from remote
run: |
git pull https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main || \
(git merge --strategy-option theirs)
- name: Add and commit changes
run: |
git add -A
git diff-index --quiet HEAD || git commit -m "Model Deployment"
- name: Push to Hugging Face
run: |
git push https://$HF_USERNAME:[email protected]/spaces/$HF_USERNAME/$SPACE_NAME main --force
functional_testing:
name: Functional Testing
runs-on: ubuntu-latest
needs:
- tebakaja_crypto_space-0
- tebakaja_crypto_space-1
- tebakaja_crypto_space-2
- tebakaja_crypto_space-3
- tebakaja_crypto_space-4
steps:
- name: Set global directory
run: git config --global --add safe.directory /github/workspace
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 1
- name: Testing Proxy Endpoints
run: |
sleep 60
chmod +x endpoints_test.sh && ./endpoints_test.sh