Improve ci (#458) #18
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: backend checks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - mwp_v1 | |
| pull_request: | |
| paths: | |
| - "backend/**" | |
| - ".github/**" | |
| - "docker/**" | |
| - "tools/**" | |
| jobs: | |
| backend-checks: | |
| runs-on: ubuntu-latest | |
| env: | |
| API_PRODUCTION_WORKERS: 1 | |
| RAY_ENABLED: True | |
| OLLAMA_ENABLED: False | |
| COMPOSE_PROFILES: "ray,background,backend" | |
| RAY_CONFIG: "config_cpu.yaml" | |
| JWT_SECRET: ${{ secrets.JWT_SECRET }} | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: false | |
| - name: Set Swap Space to 10GB | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Cache models | |
| id: cache-models | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| docker/spacy_models | |
| docker/models_cache | |
| docker/numba_cache | |
| key: models-${{ hashFiles('backend/src/app/preprocessing/**') }} | |
| - name: Check 0 - Build & Start Docker Containers | |
| working-directory: docker | |
| run: | | |
| ./setup-folders.sh | |
| chmod -R a+rwx backend_repo/ models_cache/ spacy_models/ numba_cache/ ollama_cache/ | |
| cp .env.example .env | |
| python monkey_patch_docker_compose.py --disable_ollama | |
| export GID=$(id -g) | |
| docker compose -f docker-compose-patched.yml up -d --build --quiet-pull | |
| - name: Check 1 - Database migrates without errors database | |
| working-directory: docker | |
| run: | | |
| docker compose -f docker-compose-patched.yml run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py | |
| - name: Check 2 - Database schema is up-to-date after migration | |
| working-directory: docker | |
| run: | | |
| docker compose -f docker-compose-patched.yml run dats-backend-api /opt/envs/dats/bin/alembic check | |
| - name: Check 3 - Test End-2-End importer script | |
| working-directory: tools/importer | |
| env: | |
| TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }} | |
| run: | | |
| pip install -r requirements.txt | |
| wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip | |
| unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip | |
| python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text | |
| python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image | |
| - name: Check 4 - pytest runs without errors | |
| working-directory: docker | |
| run: | | |
| docker compose -f docker-compose-patched.yml down -v | |
| rm -r backend_repo/ | |
| ./setup-folders.sh | |
| chmod -R a+rwx backend_repo/ | |
| export COMPOSE_PROFILES="ray,background" | |
| docker compose -f docker-compose-patched.yml up -d | |
| docker compose -f docker-compose-patched.yml run -i dats-backend-api /opt/envs/dats/bin/python -m pytest |