Merge branch 'main' into expo_51

wcjordan · Aug 27, 2024 · 1bff69b · 1bff69b
2 parents 3c17284 + 856ba12
commit 1bff69b
Show file tree

Hide file tree

Showing 35 changed files with 32,571 additions and 73,783 deletions.
diff --git a/.gitignore b/.gitignore
@@ -69,4 +69,3 @@ continuous_delivery_setup/secrets/
 continuous_delivery_setup/secrets.staging/
 helm/secrets/
 helm/secrets.staging/
-
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -47,20 +47,23 @@ pipeline {
                                     }
                                     sh """
                                         export PATH="/root/google-cloud-sdk/bin:\$PATH"
-                                        docker buildx create --driver docker-container --name chalk-default
-                                        docker buildx use chalk-default
+                                        docker buildx create --driver docker-container --name chalk-default --use
                                         docker buildx build --push \
-                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-ui,mode=max \
-                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui \
+                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_app,mode=max \
+                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_app \
+                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_test \
                                             --build-arg sentryDsn=\$SENTRY_DSN \
                                             -t ${GAR_REPO}/chalk-ui:${SANITIZED_BUILD_TAG} \
+                                            --target js_app_prod \
                                             ui
 
                                         docker buildx build --push \
-                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-ui,mode=max \
-                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui \
+                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_test,mode=max \
+                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_app \
+                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-ui-cache:ci_test \
+                                            --build-arg sentryDsn=\$SENTRY_DSN \
                                             -t ${GAR_REPO}/chalk-ui-base:${SANITIZED_BUILD_TAG} \
-                                            --target base \
+                                            --target js_test_env \
                                             ui
                                     """
                                 }
@@ -86,8 +89,6 @@ pipeline {
                                               limits:
                                                 cpu: "1000m"
                                                 memory: "3Gi"
-
-
                                     """
                                 }
                             }
@@ -96,13 +97,72 @@ pipeline {
                             }
                             steps {
                                 container('jenkins-worker-ui') {
-                                    dir('ui/js') {
-                                        sh 'cp -r /js/node_modules .'
-                                    }
-                                    dir('ui') {
-                                        sh 'make test'
-                                        junit testResults: 'js/junit.xml'
-                                    }
+                                    sh '''
+                                        JENKINS_WORKSPACE=$(pwd)
+                                        cp ui/Makefile /workspace/
+                                        cd /workspace
+                                        make test
+
+                                        cd $JENKINS_WORKSPACE
+                                        cp /workspace/js/junit.xml .
+                                    '''
+                                    junit testResults: 'junit.xml'
+                                }
+                            }
+                        }
+                        stage('Test UI Storybook') {
+                            agent {
+                                kubernetes {
+                                    yaml """
+                                        apiVersion: v1
+                                        kind: Pod
+                                        spec:
+                                          containers:
+                                          - name: jenkins-worker-storybook
+                                            image: ${GAR_REPO}/chalk-ui-base:${SANITIZED_BUILD_TAG}
+                                            command: ["/bin/sh", "-c"]
+                                            args:
+                                            - cd js && npx http-server -p 9009 /workspace/js/storybook-static
+                                            tty: true
+                                            ports:
+                                            - containerPort: 9009
+                                            resources:
+                                              requests:
+                                                cpu: "200m"
+                                                memory: "500Mi"
+                                              limits:
+                                                cpu: "500m"
+                                                memory: "500Mi"
+                                          - name: jenkins-worker-storybook-snapshots
+                                            image: ${GAR_REPO}/chalk-ui-base:${SANITIZED_BUILD_TAG}
+                                            command:
+                                            - cat
+                                            tty: true
+                                            resources:
+                                              requests:
+                                                cpu: "400m"
+                                                memory: "2Gi"
+                                              limits:
+                                                cpu: "1000m"
+                                                memory: "2Gi"
+                                    """
+                                }
+                            }
+                            options {
+                                timeout(time: 10, unit: 'MINUTES')
+                            }
+                            steps {
+                                container('jenkins-worker-storybook-snapshots') {
+                                    sh '''
+                                        JENKINS_WORKSPACE=$(pwd)
+                                        cp ui/Makefile /workspace/
+                                        cd /workspace
+                                        make test-storybook-inner TEST_ARGS="--junit --url http://127.0.0.1:9009"
+
+                                        cd $JENKINS_WORKSPACE
+                                        cp /workspace/js/junit.xml .
+                                    '''
+                                    junit testResults: 'junit.xml'
                                 }
                             }
                         }
@@ -127,11 +187,10 @@ pipeline {
                                     }
                                     sh """
                                         export PATH="/root/google-cloud-sdk/bin:\$PATH"
-                                        docker buildx create --driver docker-container --name chalk-default
-                                        docker buildx use chalk-default
+                                        docker buildx create --driver docker-container --name chalk-default --use
                                         docker buildx build --push \
-                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-server,mode=max \
-                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-server \
+                                            --cache-to type=registry,ref=${GAR_REPO}/chalk-server-cache:ci_server,mode=max \
+                                            --cache-from type=registry,ref=${GAR_REPO}/chalk-server-cache:ci_server \
                                             -t ${GAR_REPO}/chalk-server:${SANITIZED_BUILD_TAG} \
                                             server
                                     """
@@ -280,7 +339,7 @@ pipeline {
                                     string(credentialsId: 'chalk-prod-cd-oauth-refresh-token', variable: 'CHALK_OAUTH_REFRESH_TOKEN'),
                                 ]) {
                                     dir('tests') {
-                                        sh 'pip install "playwright==1.44.0" "pytest==8.2.2"'
+                                        sh 'pip install "playwright==1.46.0" "pytest==8.3.2"'
                                         sh "pytest . --server_domain ${SERVER_IP} --junitxml=playwright_results.xml || true"
 
                                         junit testResults: 'playwright_results.xml'

diff --git a/Makefile b/Makefile
@@ -17,19 +17,35 @@ UI_IMAGE_BASE = $(IMAGE_REPO)/chalk-ui-base
 # Build containers
 .PHONY: build
 build:
-	DOCKER_BUILDKIT=1 docker build -t $(SERVER_IMAGE):local-latest server
+	docker buildx create --driver docker-container --name chalk-default --use || true
+	docker buildx build --push \
+		--cache-to type=registry,ref=${IMAGE_REPO}/chalk-server-cache:dev_server,mode=max \
+		--cache-from type=registry,ref=${IMAGE_REPO}/chalk-server-cache:dev_server \
+		-t ${SERVER_IMAGE}:local-latest \
+		server
 	env $$(grep -v '^#' $(PROD_ENV_FILE) | xargs) sh -c ' \
-		DOCKER_BUILDKIT=1 docker build \
+		docker buildx build --push \
+			--cache-to type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_app,mode=max \
+			--cache-from type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_app \
+			--cache-from type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_test \
 			--build-arg sentryDsn=$$SENTRY_DSN \
-			-t $(UI_IMAGE):local-latest ui'
+			-t ${UI_IMAGE}:local-latest \
+			--target js_app_prod \
+			ui'
 	env $$(grep -v '^#' $(PROD_ENV_FILE) | xargs) sh -c ' \
-		DOCKER_BUILDKIT=1 docker build \
-			--target base \
-			-t $(UI_IMAGE_BASE):local-latest ui'
+		docker buildx build --push \
+			--cache-to type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_test,mode=max \
+			--cache-from type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_app \
+			--cache-from type=registry,ref=${IMAGE_REPO}/chalk-ui-cache:dev_test \
+			--build-arg sentryDsn=$$SENTRY_DSN \
+			-t ${UI_IMAGE_BASE}:local-latest \
+			--target js_test_env \
+			ui'
 
 # Test & lint
 .PHONY: test
 test: build
+	docker pull $(SERVER_IMAGE):local-latest
 	DOMAIN=localhost docker run --env-file .env --env DOMAIN --rm -t $(SERVER_IMAGE):local-latest make test
 	$(MAKE) -C ui containerized-test
 
@@ -59,28 +75,6 @@ format:
 	$(MAKE) -C ui format
 	$(MAKE) -C server format
 
-
-# Deploy to production
-# To delete: helm delete chalk-prod
-# Note, this shouldn't be needed as we deploy to prod in the Jenkinsfile
-.PHONY: deploy-k8s
-deploy-k8s: build
-	if [ "$$(kubectl config current-context)" != "$(K8S_CONTEXT)" ]; then \
-		exit 1; \
-	fi
-
-	docker push $(SERVER_IMAGE):local-latest
-	docker push $(UI_IMAGE):local-latest
-	env $$(grep -v '^#' $(PROD_ENV_FILE) | xargs) sh -c ' \
-		helm upgrade --install \
-			--set domain=chalk.$$ROOT_DOMAIN \
-			--set environment=$(ENVIRONMENT) \
-			--set gcpProject=$$GCP_PROJECT \
-			--set permittedUsers=$$PERMITTED_USERS \
-			--set server.dbPassword=$$DB_PASSWORD \
-			--set server.secretKey=$$SECRET_KEY \
-			chalk-prod helm'
-
 .PHONY: deploy-mobile-app
 deploy-mobile-app:
 	if [ "$(ENVIRONMENT)" = "PROD" ]; then \

diff --git a/README.md b/README.md
@@ -1,5 +1,10 @@
 # Chalk - Todo app
 
+## Install tools for dev env
+On Mac:  
+`brew install nginx`  
+`brew install coreutils`  
+
 ## Env Setup
 Copy .env_default to .env & .prod.env  
 
@@ -31,10 +36,6 @@ Delete BROWSERSTACK_USERNAME & BROWSERSTACK_ACCESS_KEY in prod.env
 See the OAuth Setup section below  
 Delete CHALK_OAUTH_REFRESH_TOKEN in prod.env  
 
-Install `nginx`  
-(which the dev env now requires)  
-On Mac: `brew install nginx`
-
 ## Allocate static IPs & create DNS entries
 Explore doing this w/ Config Connector once [#101](https://github.com/GoogleCloudPlatform/k8s-config-connector/issues/101) is resolved.
 

diff --git a/jenkins/gcloud_helm.Dockerfile b/jenkins/gcloud_helm.Dockerfile
@@ -4,7 +4,7 @@ RUN gcloud components install kubectl -q --no-user-output-enabled
 
 # Install Helm
 RUN mkdir helm_tmp \
-  && curl -o helm_tmp/helm.tar.gz https://get.helm.sh/helm-v3.15.1-linux-amd64.tar.gz \
+  && curl -o helm_tmp/helm.tar.gz https://get.helm.sh/helm-v3.15.4-linux-amd64.tar.gz \
   && tar -zxvf helm_tmp/helm.tar.gz -C helm_tmp \
   && mv helm_tmp/linux-amd64/helm /bin/helm \
   && rm -rf helm_tmp
diff --git a/server/Dockerfile b/server/Dockerfile
@@ -3,7 +3,7 @@ FROM python:3-bookworm
 WORKDIR /usr/src/app
 
 # Set log messages to immediately appear in the logs
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 # Install system packages.  We need postgresql-client for database backups
 RUN wget -q https://www.postgresql.org/media/keys/ACCC4CF8.asc -O - | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg && \

diff --git a/server/dev-requirements.txt b/server/dev-requirements.txt
@@ -1,8 +1,8 @@
 -r requirements.txt
 
 # These are used for development or linting
-flake8==7.0.0
-pylint==3.2.3
+flake8==7.1.1
+pylint==3.2.6
 pylint-django==2.5.5
-pytest==8.2.2
+pytest==8.3.2
 yapf==0.40.2
diff --git a/server/requirements.txt b/server/requirements.txt
@@ -1,7 +1,7 @@
-Django==4.2.13
+Django==4.2.15
 djangorestframework==3.15.2
 django-simple-history==3.7.0
-google-auth-oauthlib==1.2.0
-gunicorn==22.0.0
+google-auth-oauthlib==1.2.1
+gunicorn==23.0.0
 psycopg2==2.9.9
 requests==2.32.3
diff --git a/ui/Dockerfile b/ui/Dockerfile
@@ -1,6 +1,6 @@
 # Multistage build for JS artifacts
-FROM node:20 as base
-WORKDIR /js
+FROM node:20 AS base
+WORKDIR /workspace/js
 
 ENV ENVIRONMENT=prod
 ENV DEBUG=false
@@ -12,35 +12,45 @@ RUN apt-get update && apt-get install -y \
   bash \
   build-essential \
   && rm -rf /var/lib/apt/lists/*
-RUN npm install -g sharp-cli
+
+# Install sharp-cli for expo / metro bundler
+RUN npm install -g sharp-cli@^2.1.0
 
 # Install React dependencies
-COPY js/yarn.lock .
-COPY js/package.json .
-RUN yarn --silent install --immutable && yarn cache clean
+COPY js/yarn.lock js/package.json js/.yarnrc.yml ./
+COPY js/.yarn/ ./.yarn/
+RUN corepack enable && yarn install --immutable && yarn cache clean
+
+# Multistage - test container
+FROM base AS js_test_env
+RUN npx playwright install && npx playwright install-deps
+
+# Build storybook for testing
+COPY js/.eslintrc js/babel.config.js js/jest.config.js js/tsconfig.json js/App.tsx ./
+COPY js/.storybook/ ./.storybook/
+COPY js/assets/ ./assets/
+COPY js/__mocks__/ ./__mocks__/
+COPY js/src/ ./src/
+
+RUN npx storybook build --test --quiet
+WORKDIR /workspace
 
 # Multistage - built app
-FROM base as js_app
+FROM base AS js_app_build
 
 # Install React app
-COPY js/.prettierrc .
-COPY js/.eslintrc .
-COPY js/tsconfig.json .
-COPY js/.storybook ./.storybook
-
-COPY js/app.config.js ./app.config.js
-COPY js/babel.config.js ./babel.config.js
-COPY js/App.tsx ./App.tsx
-COPY js/assets ./assets
-COPY js/src ./src
+COPY js/.eslintrc js/.prettierrc js/app.config.js js/babel.config.js js/tsconfig.json js/App.tsx ./
+COPY js/.storybook/ ./.storybook/
+COPY js/assets/ ./assets/
+COPY js/src/ ./src/
 
 # Build React app
 RUN yarn build
 
 # Multistage - nginx container
-FROM nginx:alpine
+FROM nginx:alpine AS js_app_prod
 
 # Copy over built JS files from React app
-COPY --from=js_app /js/dist/ /www/client/
+COPY --from=js_app_build /workspace/js/dist/ /www/client/
 
 CMD ["nginx", "-g", "daemon off;"]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -69,4 +69,3 @@ continuous_delivery_setup/secrets/
		continuous_delivery_setup/secrets.staging/
		helm/secrets/
		helm/secrets.staging/