From f657e964318f52409f3071e1361cab31d4e11a2a Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Tue, 24 Aug 2021 11:46:22 -0700
Subject: [PATCH 1/9] Add preliminary just scripts

---
 README.md | 38 ++++++++++++++++++++------------------
 justfile  | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 18 deletions(-)
 create mode 100644 justfile

diff --git a/README.md b/README.md
index 4ed5d3506..7d19504b5 100644
--- a/README.md
+++ b/README.md
@@ -106,26 +106,28 @@ own dependency requirements.
 You'll need `docker` and `docker-compose` installed on your machine, with
 versions new enough to use version `3` of Docker Compose `.yml` files.
 
+You will also need the [`just`](https://github.com/casey/just#installation) command runner installed.
+
 To set up the local python environment along with the pre-commit hook, run:
 
 ```shell
 python3 -m venv venv
 source venv/bin/activate
-pip install -r requirements.txt
-pre-commit install
+just install
 ```
 
 Optionally, to install dependencies for your editor to introspect stuff about them:
 
 ```shell
-pip install -r openverse_catalog/requirements_dev.txt
+just installcatalog
 ```
 
+
 To set up environment variables, navigate to the
 [`openverse_catalog`][cc_airflow] directory, and run
 
 ```shell
-cp env.template .env
+just makeenv
 ```
 
 If needed, fill in API keys or other secrets and variables in `.env`. This is
@@ -134,7 +136,7 @@ not needed if you only want to run the tests. There is a
 [`openverse_catalog`][cc_airflow] directory, so from that directory, run
 
 ```shell
-docker-compose up -d
+just up
 ```
 
 This results, among other things, in the following running containers:
@@ -156,47 +158,47 @@ and some networking setup so that they can communicate. Note:
 At this stage, you can run the tests via:
 
 ```shell
-docker exec openverse_catalog_webserver_1 /usr/local/airflow/.local/bin/pytest
+just test
 ```
 
 Edits to the source files or tests can be made on your local machine, then tests
 can be run in the container via the above command to see the effects.
 
-If you'd like, it's possible to login to the webserver container via
+If you'd like, it's possible to login to the webserver container via:
 
 ```shell
-docker exec -it openverse_catalog_webserver_1 /bin/bash
+just shell
 ```
 
-It's also possible to attach to the running command process of the webserver
-container via
+If you just need to run an airflow command, you can use the `airflow` recipe. Arguments passed to airflow must be quoted:
 
 ```shell
-docker attach --sig-proxy=false openverse_catalog_webserver_1
+just airflow "config list"
 ```
 
-Attaching in this manner lets you see the output from both the Airflow webserver
-and scheduler, which can be useful for debugging purposes. To leave the
-container, (but keep it running), press `Ctrl-C` on \*nix platforms
+To tail the logs of the running container:
+
+```shell
+just logs
+```
 
 To see the Airflow web UI, point your browser to `localhost:9090`.
 
 If you'd like to bring down the containers, run
 
 ```shell
-docker-compose down
+just down
 ```
 
-from the [`openverse_catalog`][cc_airflow] directory.
-
 To reset the test DB (wiping out all databases, schemata, and tables), run
 
 ```shell
-docker-compose down -v
+just down -v
 ```
 
 `docker volume prune` can also be useful if you've already stopped the running containers, but be warned that it will remove all volumes associated with stopped containers, not just openverse-catalog ones.
 
+[justfile]: justfile
 [dockercompose]: openverse_catalog/docker-compose.yml
 [cc_airflow]: openverse_catalog/
 
diff --git a/justfile b/justfile
new file mode 100644
index 000000000..0800e263f
--- /dev/null
+++ b/justfile
@@ -0,0 +1,39 @@
+DOCKER_FILE := "openverse_catalog/docker-compose.yml"
+SERVICE := "webserver"
+
+
+install:
+    pip install -r requirements.txt
+    pre-commit install
+
+
+installcatalog:
+    pip install -r openverse_catalog/requirements_dev.txt
+
+
+makeenv:
+    @([ ! -f openverse_catalog/.env ] && cp openverse_catalog/env.template openverse_catalog/.env) || true
+
+
+up: makeenv
+    docker-compose --file={{ DOCKER_FILE }} up -d
+
+
+down flags="":
+    docker-compose --file={{ DOCKER_FILE }} down {{ flags }}
+
+
+logs: makeenv
+    docker-compose --file={{ DOCKER_FILE }} logs -f
+
+
+test: makeenv
+    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /usr/local/airflow/.local/bin/pytest
+
+
+shell: makeenv
+    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /bin/bash
+
+
+airflow command: makeenv
+    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} airflow {{ command }}

From bb47e44018ef9a29b268b18ce7f4305575004f8b Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Tue, 24 Aug 2021 11:57:22 -0700
Subject: [PATCH 2/9] Rename makeenv to dotenv

---
 README.md |  2 +-
 justfile  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 7d19504b5..b509b4d11 100644
--- a/README.md
+++ b/README.md
@@ -127,7 +127,7 @@ To set up environment variables, navigate to the
 [`openverse_catalog`][cc_airflow] directory, and run
 
 ```shell
-just makeenv
+just dotenv
 ```
 
 If needed, fill in API keys or other secrets and variables in `.env`. This is
diff --git a/justfile b/justfile
index 0800e263f..be6d34dee 100644
--- a/justfile
+++ b/justfile
@@ -11,11 +11,11 @@ installcatalog:
     pip install -r openverse_catalog/requirements_dev.txt
 
 
-makeenv:
+dotenv:
     @([ ! -f openverse_catalog/.env ] && cp openverse_catalog/env.template openverse_catalog/.env) || true
 
 
-up: makeenv
+up: dotenv
     docker-compose --file={{ DOCKER_FILE }} up -d
 
 
@@ -23,17 +23,17 @@ down flags="":
     docker-compose --file={{ DOCKER_FILE }} down {{ flags }}
 
 
-logs: makeenv
+logs: dotenv
     docker-compose --file={{ DOCKER_FILE }} logs -f
 
 
-test: makeenv
+test: dotenv
     docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /usr/local/airflow/.local/bin/pytest
 
 
-shell: makeenv
+shell: dotenv
     docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /bin/bash
 
 
-airflow command: makeenv
+airflow command: dotenv
     docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} airflow {{ command }}

From 9cecc481a78236e0156cd4b53c2e2fc84ca8bbda Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Tue, 24 Aug 2021 12:19:07 -0700
Subject: [PATCH 3/9] Remove directions to switch directories from README

---
 README.md | 6 ++----
 justfile  | 6 +-----
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index b509b4d11..bc0d313b9 100644
--- a/README.md
+++ b/README.md
@@ -116,15 +116,13 @@ source venv/bin/activate
 just install
 ```
 
-Optionally, to install dependencies for your editor to introspect stuff about them:
+Optionally, to install the catalog dependencies for your editor to introspect stuff about them:
 
 ```shell
 just installcatalog
 ```
 
-
-To set up environment variables, navigate to the
-[`openverse_catalog`][cc_airflow] directory, and run
+To set up environment variables run:
 
 ```shell
 just dotenv
diff --git a/justfile b/justfile
index be6d34dee..f7b281f89 100644
--- a/justfile
+++ b/justfile
@@ -3,14 +3,10 @@ SERVICE := "webserver"
 
 
 install:
-    pip install -r requirements.txt
+    pip install -r requirements.txt -r openverse_catalog/requirements_dev.txt
     pre-commit install
 
 
-installcatalog:
-    pip install -r openverse_catalog/requirements_dev.txt
-
-
 dotenv:
     @([ ! -f openverse_catalog/.env ] && cp openverse_catalog/env.template openverse_catalog/.env) || true
 

From cfcc19dcb7d0bb0c01855b3c1f74abd9d885d777 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Tue, 24 Aug 2021 12:56:12 -0700
Subject: [PATCH 4/9] Fix justfile to use dev configuration

---
 justfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/justfile b/justfile
index f7b281f89..6dcd3d177 100644
--- a/justfile
+++ b/justfile
@@ -1,4 +1,4 @@
-DOCKER_FILE := "openverse_catalog/docker-compose.yml"
+DEV_DOCKER_FILES := "--file=openverse_catalog/docker-compose.yml --file=openverse_catalog/docker-compose.override.yml"
 SERVICE := "webserver"
 
 
@@ -12,24 +12,24 @@ dotenv:
 
 
 up: dotenv
-    docker-compose --file={{ DOCKER_FILE }} up -d
+    docker-compose {{ DEV_DOCKER_FILES }} up -d
 
 
 down flags="":
-    docker-compose --file={{ DOCKER_FILE }} down {{ flags }}
+    docker-compose {{ DEV_DOCKER_FILES }} down {{ flags }}
 
 
 logs: dotenv
-    docker-compose --file={{ DOCKER_FILE }} logs -f
+    docker-compose {{ DEV_DOCKER_FILES }} logs -f
 
 
 test: dotenv
-    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /usr/local/airflow/.local/bin/pytest
+    docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} /usr/local/airflow/.local/bin/pytest
 
 
 shell: dotenv
-    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} /bin/bash
+    docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} /bin/bash
 
 
-airflow command: dotenv
-    docker-compose --file={{ DOCKER_FILE }} exec {{ SERVICE }} airflow {{ command }}
+airflow command="": dotenv
+    docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} airflow {{ command }}

From f7e7847e783fba365eb6c05b825ab8eb2181d3df Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Mon, 30 Aug 2021 08:17:49 -0700
Subject: [PATCH 5/9] Complete list of running containers

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index bc0d313b9..cb3273b3e 100644
--- a/README.md
+++ b/README.md
@@ -141,6 +141,7 @@ This results, among other things, in the following running containers:
 
 - `openverse_catalog_webserver_1`
 - `openverse_catalog_postgres_1`
+- `openverse_catalog_s3_1`
 
 and some networking setup so that they can communicate. Note:
 

From d30dd25bef472674588ea87ad08104f8819e4cbc Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Mon, 30 Aug 2021 08:18:53 -0700
Subject: [PATCH 6/9] Use more general language for logs recipe behvaior

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cb3273b3e..0b4e4ec44 100644
--- a/README.md
+++ b/README.md
@@ -175,7 +175,7 @@ If you just need to run an airflow command, you can use the `airflow` recipe. Ar
 just airflow "config list"
 ```
 
-To tail the logs of the running container:
+To follow the logs of the running container:
 
 ```shell
 just logs

From fa728f40ea3a76a54a49065d2703f55865942aa7 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Mon, 30 Aug 2021 11:00:03 -0700
Subject: [PATCH 7/9] Ensure containers are running before running exec

---
 justfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/justfile b/justfile
index 6dcd3d177..8d2aa7aed 100644
--- a/justfile
+++ b/justfile
@@ -19,17 +19,17 @@ down flags="":
     docker-compose {{ DEV_DOCKER_FILES }} down {{ flags }}
 
 
-logs: dotenv
+logs: dotenv up
     docker-compose {{ DEV_DOCKER_FILES }} logs -f
 
 
-test: dotenv
+test: dotenv up
     docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} /usr/local/airflow/.local/bin/pytest
 
 
-shell: dotenv
+shell: dotenv up
     docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} /bin/bash
 
 
-airflow command="": dotenv
+airflow command="": dotenv up
     docker-compose {{ DEV_DOCKER_FILES }} exec {{ SERVICE }} airflow {{ command }}

From 90461c2125ce3b9e1487f86ea0376e752d59033d Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Mon, 30 Aug 2021 11:19:30 -0700
Subject: [PATCH 8/9] Do not load any .env files for just

---
 justfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/justfile b/justfile
index 8d2aa7aed..6a0001c91 100644
--- a/justfile
+++ b/justfile
@@ -1,3 +1,5 @@
+set dotenv-load := false
+
 DEV_DOCKER_FILES := "--file=openverse_catalog/docker-compose.yml --file=openverse_catalog/docker-compose.override.yml"
 SERVICE := "webserver"
 

From c2d63246c4c828e99a0836dead5728df85ce55b3 Mon Sep 17 00:00:00 2001
From: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com>
Date: Tue, 31 Aug 2021 08:19:16 -0700
Subject: [PATCH 9/9] Remove old recipe from readme

---
 README.md | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/README.md b/README.md
index 0b4e4ec44..2edfabd8a 100644
--- a/README.md
+++ b/README.md
@@ -116,12 +116,6 @@ source venv/bin/activate
 just install
 ```
 
-Optionally, to install the catalog dependencies for your editor to introspect stuff about them:
-
-```shell
-just installcatalog
-```
-
 To set up environment variables run:
 
 ```shell