Skip to content
CI

'File partition' option and 'document' directory specification #1031

Sign in to view logs

'File partition' option and 'document' directory specification

'File partition' option and 'document' directory specification #1031

Workflow file for this run

.github/workflows/CI.yml at 865b250
name: CI
on:
push:
branches:
- main
- master
tags:
- "*"
pull_request:
branches:
- main
- master
workflow_dispatch:
permissions:
contents: read
env:
DOLMA_TESTS_SKIP_AWS: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'true' || 'false' }}
DOLMA_TEST_S3_PREFIX: s3://dolma-tests
RUST_CHANNEL: stable
jobs:
info:
name: Run info
runs-on: ubuntu-latest
steps:
- name: Echo environment variables
run: |
echo "reference: ${{ github.ref }}"
echo "event name: ${{ github.event_name }}"
echo "run tests: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}"
echo "is main: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }}"
echo "is release: ${{ startsWith(github.ref, 'refs/tags/') }}"
echo "skip AWS: ${{ env.DOLMA_TESTS_SKIP_AWS }}"
echo "commit: ${{ github.sha }}"
echo "PR base repo: ${{ github.event.pull_request.base.repo.full_name }}/tree/${{ github.event.pull_request.base.ref }}"
echo "PR head repo: ${{ github.event.pull_request.head.repo.full_name }}/tree/${{ github.event.pull_request.head.ref }}"
should_build:
name: "Check if build"
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.ref }}
- name: List branches and tags
run: |
git branch -a
git tag -l
git log | head -n 1000
- id: check_version
run: |
set +e
has_updated=$(git diff --name-only '${{ github.event.pull_request.base.sha }}' | grep -E 'pyproject.toml|Cargo.toml')
is_main_or_release='${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}'
if [[ -n "${has_updated}" ]] || [[ "${is_main_or_release}" == 'true' ]]; then
echo "should_build=true" >> $GITHUB_OUTPUT
else
echo "should_build=false" >> $GITHUB_OUTPUT
fi
shell: bash
outputs:
should_build: ${{ steps.check_version.outputs.should_build }}
prepare-venv:
runs-on: ubuntu-latest
name: "Prepare Virtual Env"
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Cache Virtual Env
uses: actions/cache@v3
# name for referring later
id: cache-venv
with:
# what we cache: the virtualenv
path: ./.venv/
# The cache key depends on pyproject.toml and Cargo.toml
key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
- name: Setup system libraries
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
sudo apt-get update
sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source musl-tools
- name: Install Rust toolchain
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
rustup update ${{ env.RUST_CHANNEL }}
rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src
rustup default ${{ env.RUST_CHANNEL }}
- name: Install Python
if: steps.cache-venv.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "3.8"
architecture: "x64"
- name: Create a new Python environment & install maturin
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv .venv
source .venv/bin/activate
pip install -U pip
pip install maturin==1.7.1
- name: Install dolma wheels
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
source .venv/bin/activate
maturin build --release -i $(which python) --out dist
wheel_path=$(ls dist/*.whl)
pip install "${wheel_path}[all]"
tests:
runs-on: ubuntu-latest
needs: prepare-venv
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
name: "${{ matrix.task.name }}"
strategy:
fail-fast: true
matrix:
task:
- name: Check Python style
run: |
set -e
isort --check --verbose .
black --check --verbose .
- name: Check Rust style
run: |
rustfmt --edition 2021 src/*.rs --check
- name: Lint Python
run: |
flake8 tests/python/ && flake8 python/
- name: Types Python
run: |
mypy tests/python/ && mypy python/
- name: Run Python tests
run: |
pytest -vs --color=yes tests/python/
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Cache Virtual Env
uses: actions/cache@v3
# name for referring later
id: cache-venv
with:
# what we cache: the virtualenv
path: ./.venv/
# The cache key depends on pyproject.toml and Cargo.toml
key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
- name: ${{ matrix.task.name }}
run: |
source .venv/bin/activate
${{ matrix.task.run }}
build-linux:
needs: should_build
if: ${{ needs.should_build.outputs.should_build == 'true' }}
runs-on: ubuntu-latest
strategy:
matrix:
# fails to build on x86, so removing for now; low use anyway.
# target: [x86_64, x86, aarch64, armv7]
target: [x86_64, aarch64, armv7]
name: "Build Linux (${{ matrix.target }})"
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install 32bit version of libc
if: ${{ matrix.target == 'x86' || contains(matrix.target, 'i686') }}
run: |
sudo apt-get update
sudo apt-get install --yes --upgrade libc6-dev-i386
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
args: --release --out dist --find-interpreter
manylinux: manylinux_2_28
container: "ghcr.io/rust-cross/manylinux_2_28-cross:${{ matrix.target }}"
before-script-linux: |
sudo apt-get update
sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source musl-tools
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
build-windows:
needs: should_build
if: ${{ needs.should_build.outputs.should_build == 'true' }}
runs-on: windows-latest
name: "Build Windows (${{ matrix.target }})"
strategy:
matrix:
target: [x64, x86]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
architecture: ${{ matrix.target }}
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
args: --release --out dist --find-interpreter
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
build-macos:
needs: should_build
if: ${{ needs.should_build.outputs.should_build == 'true' }}
runs-on: macos-latest
name: "Build macOS (${{ matrix.target }})"
strategy:
matrix:
target: [x86_64, aarch64]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
args: --release --out dist --find-interpreter
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
sdist:
runs-on: ubuntu-latest
needs: should_build
if: ${{ needs.should_build.outputs.should_build == 'true' }}
steps:
- uses: actions/checkout@v3
- name: Build sdist
uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
- name: Upload sdist
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
release:
name: Release
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
needs: [build-linux, build-windows, build-macos, sdist]
steps:
- uses: actions/download-artifact@v3
with:
name: wheels
- name: Publish to PyPI
uses: PyO3/maturin-action@v1
env:
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
with:
command: upload
maturin-version: 1.7.1
args: --skip-existing *