Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jupyter notebook cherry pick #600

Merged
merged 4 commits into from
Apr 23, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
uncomment validation and test notebook breakage
  • Loading branch information
chiayi committed Apr 22, 2024
commit f6fc20a4f22f6b1e2b2c358756137a310bd4c9b5
32 changes: 23 additions & 9 deletions cloudbuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -256,11 +256,18 @@ steps:
python3 test_frontend.py "127.0.0.1:8081"
echo "pass" > /workspace/rag_frontend_result.txt

# Upload locally stored netflix dataset to GCS bucket mounted as /data
gsutil cp ./netflix_titles.csv gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/netflix-shows/netflix_titles.csv
ray job submit --working-dir . --address=http://127.0.0.1:8265 -- python ray_job.py
cd /workspace/
sed -i "s/<username>/$$KAGGLE_USERNAME/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb
sed -i "s/<token>/$$KAGGLE_KEY/g" ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb
gsutil cp ./applications/rag/example_notebooks/rag-kaggle-ray-sql-interactive.ipynb gs://gke-aieco-rag-$SHORT_SHA-$_BUILD_ID/
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID $(kubectl get pod -l app=jupyterhub,component=hub -n rag-$SHORT_SHA-$_BUILD_ID -o jsonpath="{.items[0].metadata.name}") -- jupyterhub token admin --log-level=CRITICAL | xargs python3 ./applications/rag/notebook_starter.py
# Wait for jupyterhub to trigger notebook pod startup
sleep 5s
kubectl wait --for=condition=Ready pod/jupyter-admin -n rag-$SHORT_SHA-$_BUILD_ID --timeout=300s
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- jupyter nbconvert --to script /data/rag-kaggle-ray-sql-interactive.ipynb
kubectl exec -it -n rag-$SHORT_SHA-$_BUILD_ID jupyter-admin -c notebook -- ipython /data/rag-kaggle-ray-sql-interactive.py

python3 test_rag.py "http://127.0.0.1:8081/prompt"
python3 ./applications/rag/tests/test_rag.py "http://127.0.0.1:8081/prompt"
echo "pass" > /workspace/rag_prompt_result.txt

allowFailure: true
Expand Down Expand Up @@ -365,11 +372,11 @@ steps:
exit 1
fi

# TODO: re-add this check once generating embeddings in CI is stable
# if [[ $(cat /workspace/rag_prompt_result.txt) != "pass" ]]; then
# echo "rag frontend test failed"
# exit 1
# fi
TODO: re-add this check once generating embeddings in CI is stable
if [[ $(cat /workspace/rag_prompt_result.txt) != "pass" ]]; then
echo "rag frontend test failed"
exit 1
fi

waitFor: ['cleanup gke cluster']

Expand All @@ -382,3 +389,10 @@ options:
substitutionOption: 'ALLOW_LOOSE'
machineType: 'E2_HIGHCPU_8'
timeout: 5400s

availableSecrets:
secretManager:
- versionName: projects/gke-ai-eco-dev/secrets/cloudbuild-kaggle-username/versions/latest
env: 'KAGGLE_USERNAME'
- versionName: projects/gke-ai-eco-dev/secrets/cloudbuild-kaggle-key/versions/latest
env: 'KAGGLE_KEY'