From f1630eae2de6136ec935a329f162b3ea0d9ba255 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:13:17 -0500
Subject: [PATCH 01/32] Add rust ci

---
 .github/workflows/testing.yml | 55 +++++++++++++++++++++++++++++++++++
 scripts/data/persons.csv      | 11 +++++++
 scripts/docker-compose.yml    | 13 +++++++++
 scripts/init.sql              | 21 +++++++++++++
 scripts/prep_tests.sh         |  4 +++
 5 files changed, 104 insertions(+)
 create mode 100644 .github/workflows/testing.yml
 create mode 100644 scripts/data/persons.csv
 create mode 100644 scripts/docker-compose.yml
 create mode 100644 scripts/init.sql
 create mode 100644 scripts/prep_tests.sh

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
new file mode 100644
index 0000000..f67154d
--- /dev/null
+++ b/.github/workflows/testing.yml
@@ -0,0 +1,55 @@
+name: Testing
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+  POSTGRES_PASSWORD: "test_password"
+jobs:
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.7.2
+      - name: Prep data # This prevents sqlx errors if the database isn't running
+        run: sh ./scripts/prep_tests.sh
+      - name: Run cargo clippy
+        run: cargo clippy --all-targets -- --deny warnings
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.7.2
+      - name: Prep data # This prevents sqlx errors if the database isn't running
+        run: sh ./scripts/prep_tests.sh
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check
+  test:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.7.2
+      - name: Prep data
+        run: sh ./scripts/prep_tests.sh
+      - name: Run cargo test
+        run: cargo test
diff --git a/scripts/data/persons.csv b/scripts/data/persons.csv
new file mode 100644
index 0000000..5d52514
--- /dev/null
+++ b/scripts/data/persons.csv
@@ -0,0 +1,11 @@
+id,name,age,isMarried,city,state,country
+1,Megan Chang,48,false,Fredonia,Antioquia,Colombia
+2,Billy Sheppard,38,false,Campeche,Campeche,Mexico
+3,Richard Bowers,53,false,Tahannawt,Marrakech-Safi,Morocco
+4,Tammy Howard,41,true,Somandepalle,Andhra Pradesh,India
+5,William Campbell,44,true,Dimiao,Bohol,Philippines
+6,Christine King,35,true,Kanur,Karnataka,India
+7,Kyle Blair,30,false,Ettapur,Tamil Nadu,India
+8,Thomas Garcia,30,false,Gurpinar,Van,Turkey
+9,Leslie Bowman,61,true,Madaba,Madaba,Jordan
+10,Tammy Woods,56,false,Vernon,British Columbia,Canada
diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
new file mode 100644
index 0000000..ad61a64
--- /dev/null
+++ b/scripts/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "3.9"
+
+services:
+  database:
+    image: "postgres:latest"
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    volumes:
+      - ./pg/:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+      - ./data:/data
diff --git a/scripts/init.sql b/scripts/init.sql
new file mode 100644
index 0000000..a38f148
--- /dev/null
+++ b/scripts/init.sql
@@ -0,0 +1,21 @@
+SELECT 'CREATE DATABASE etl'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'etl')\gexec
+
+\connect etl
+
+CREATE TABLE IF NOT EXISTS persons(
+    id integer PRIMARY KEY,
+    name text,
+    age smallint,
+    isMarried boolean,
+    city text,
+    state text,
+    country text
+)\gexec
+
+TRUNCATE TABLE persons\gexec
+
+COPY persons(id, name, age, isMarried, city, state, country)
+FROM '/data/persons.csv'
+DELIMITER ','
+CSV HEADER\gexec
diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
new file mode 100644
index 0000000..e250812
--- /dev/null
+++ b/scripts/prep_tests.sh
@@ -0,0 +1,4 @@
+#!bin/bash
+
+cp pieces/postgres_etl/data/persons.csv scripts/data
+docker compose -f scripts/docker-compose.yml up

From 26ac7a9961604464505ba37b4f1b06b2a0938dca Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:16:25 -0500
Subject: [PATCH 02/32] Ignore data in scripts/data

---
 .gitignore               |  2 ++
 scripts/data/persons.csv | 11 -----------
 2 files changed, 2 insertions(+), 11 deletions(-)
 delete mode 100644 scripts/data/persons.csv

diff --git a/.gitignore b/.gitignore
index 805e41a..bbdcee8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -164,3 +164,5 @@ cython_debug/
 
 # Rust
 /target
+
+/scripts/data/*
diff --git a/scripts/data/persons.csv b/scripts/data/persons.csv
deleted file mode 100644
index 5d52514..0000000
--- a/scripts/data/persons.csv
+++ /dev/null
@@ -1,11 +0,0 @@
-id,name,age,isMarried,city,state,country
-1,Megan Chang,48,false,Fredonia,Antioquia,Colombia
-2,Billy Sheppard,38,false,Campeche,Campeche,Mexico
-3,Richard Bowers,53,false,Tahannawt,Marrakech-Safi,Morocco
-4,Tammy Howard,41,true,Somandepalle,Andhra Pradesh,India
-5,William Campbell,44,true,Dimiao,Bohol,Philippines
-6,Christine King,35,true,Kanur,Karnataka,India
-7,Kyle Blair,30,false,Ettapur,Tamil Nadu,India
-8,Thomas Garcia,30,false,Gurpinar,Van,Turkey
-9,Leslie Bowman,61,true,Madaba,Madaba,Jordan
-10,Tammy Woods,56,false,Vernon,British Columbia,Canada

From b011da5119dcdfe58c839c49f3354a7f8c3e6b45 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:19:15 -0500
Subject: [PATCH 03/32] Run docker in detached mode

---
 scripts/prep_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index e250812..b1b7cfb 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -1,4 +1,4 @@
 #!bin/bash
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
-docker compose -f scripts/docker-compose.yml up
+docker compose -f scripts/docker-compose.yml up -d

From e0d30921613f731a9e64480a5174abb89b4dd675 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:28:57 -0500
Subject: [PATCH 04/32] Add db url environment vairable

---
 .github/workflows/testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index f67154d..96a4b07 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -10,6 +10,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: "-D warnings"
   POSTGRES_PASSWORD: "test_password"
+  DATABASE_URL: "postgresql://postgres:test_password@localhost:5432/etl"
 jobs:
   clippy:
     name: Clippy

From bd23521353030416d4f1d17f3296824a320bad62 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:32:36 -0500
Subject: [PATCH 05/32] Replace localhost with ip

---
 .github/workflows/testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 96a4b07..77e016a 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -10,7 +10,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: "-D warnings"
   POSTGRES_PASSWORD: "test_password"
-  DATABASE_URL: "postgresql://postgres:test_password@localhost:5432/etl"
+  DATABASE_URL: "postgresql://postgres:test_password@172.0.0.1:5432/etl"
 jobs:
   clippy:
     name: Clippy

From 3dc9b63f7ca5b036e8e0071d39a7d0bfc66a0eb1 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:35:55 -0500
Subject: [PATCH 06/32] Fix ip address

---
 .github/workflows/testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 77e016a..b2c5510 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -10,7 +10,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: "-D warnings"
   POSTGRES_PASSWORD: "test_password"
-  DATABASE_URL: "postgresql://postgres:test_password@172.0.0.1:5432/etl"
+  DATABASE_URL: "postgresql://postgres:test_password@127.0.0.1:5432/etl"
 jobs:
   clippy:
     name: Clippy

From 93d8d9159b15dd6446aa188743d6ec50222af438 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 10:52:07 -0500
Subject: [PATCH 07/32] Add health check

---
 scripts/docker-compose.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index ad61a64..e368f37 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -11,3 +11,8 @@ services:
       - ./pg/:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
       - ./data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5

From d49118cc4e97c7dd97746a1eedc6555c791bbf39 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:06:18 -0500
Subject: [PATCH 08/32] Give database time to get ready

---
 scripts/docker-compose.yml | 5 -----
 scripts/prep_tests.sh      | 1 +
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index e368f37..ad61a64 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -11,8 +11,3 @@ services:
       - ./pg/:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
       - ./data:/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index b1b7cfb..6ab6a8d 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -2,3 +2,4 @@
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
+sleep 5 # give db time to get ready

From fb87bb9982d61190ad44aa6084fe0f56cf4e757c Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:08:29 -0500
Subject: [PATCH 09/32] Remove sleep

---
 scripts/prep_tests.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index 6ab6a8d..b1b7cfb 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -2,4 +2,3 @@
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
-sleep 5 # give db time to get ready

From 9bc3cd7ab87962c04f8bbc5648635e7dac6cf2e5 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:13:11 -0500
Subject: [PATCH 10/32] Add debugging

---
 scripts/prep_tests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index b1b7cfb..f1d820f 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -2,3 +2,5 @@
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
+docker ps
+docker logs database

From c5b74bc2ea3f69a0d42e6f6cb99779b17db97336 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:14:12 -0500
Subject: [PATCH 11/32] Fix command

---
 scripts/prep_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index f1d820f..996ed1a 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -3,4 +3,4 @@
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
 docker ps
-docker logs database
+docker compose logs database

From e58aa5b7812fdf49c20a89ef5ed756cc7dd2ee86 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:15:17 -0500
Subject: [PATCH 12/32] Point to config

---
 scripts/prep_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index 996ed1a..8a20353 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -3,4 +3,4 @@
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
 docker ps
-docker compose logs database
+docker compose -f scripts/docker-compose.yml logs database

From 7bd4d80cac36b014289f39d2b422e140adb8d660 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:18:19 -0500
Subject: [PATCH 13/32] Pipe to tail

---
 scripts/prep_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index 8a20353..968677c 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -3,4 +3,4 @@
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
 docker ps
-docker compose -f scripts/docker-compose.yml logs database
+docker compose -f scripts/docker-compose.yml logs database | tail

From 890eeb0a07e192bf29eee690bf364d1622f90e9b Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:19:46 -0500
Subject: [PATCH 14/32] Add wait

---
 scripts/prep_tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index 968677c..d4b645a 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -2,5 +2,6 @@
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
+wait 10
 docker ps
 docker compose -f scripts/docker-compose.yml logs database | tail

From a8b529226640b6986ebb4574a2745318feca3e2f Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:22:59 -0500
Subject: [PATCH 15/32] Sleep in action

---
 .github/workflows/testing.yml | 4 +++-
 scripts/prep_tests.sh         | 3 ---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b2c5510..67f61f4 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -37,7 +37,9 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: sh ./scripts/prep_tests.sh
+        run: |
+          sh ./scripts/prep_tests.sh
+          sleep 5
       - name: Run cargo fmt
         run: cargo fmt --all -- --check
   test:
diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index d4b645a..b1b7cfb 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -2,6 +2,3 @@
 
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d
-wait 10
-docker ps
-docker compose -f scripts/docker-compose.yml logs database | tail

From 1142a3f8c2b55518c0477bffc51e32c9828402fc Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:24:38 -0500
Subject: [PATCH 16/32] Add sleep to all steps

---
 .github/workflows/testing.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 67f61f4..4a59c09 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -25,7 +25,10 @@ jobs:
       - name: Prep data # This prevents sqlx errors if the database isn't running
         run: sh ./scripts/prep_tests.sh
       - name: Run cargo clippy
-        run: cargo clippy --all-targets -- --deny warnings
+        run: |
+          cargo clippy --all-targets -- --deny warnings
+          echo "Loading data"
+          sleep 5
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -39,6 +42,7 @@ jobs:
       - name: Prep data # This prevents sqlx errors if the database isn't running
         run: |
           sh ./scripts/prep_tests.sh
+          echo "Loading data"
           sleep 5
       - name: Run cargo fmt
         run: cargo fmt --all -- --check
@@ -53,6 +57,9 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data
-        run: sh ./scripts/prep_tests.sh
+        run: |
+          sh ./scripts/prep_tests.sh
+          echo "Loading data"
+          sleep 5
       - name: Run cargo test
         run: cargo test

From 2e79e9cab90dc1242fc0cd1ced028e89d53b35fd Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:27:35 -0500
Subject: [PATCH 17/32] Try longer sleep

---
 .github/workflows/testing.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 4a59c09..4458578 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -23,12 +23,12 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: sh ./scripts/prep_tests.sh
-      - name: Run cargo clippy
         run: |
-          cargo clippy --all-targets -- --deny warnings
+          sh ./scripts/prep_tests.sh
           echo "Loading data"
-          sleep 5
+          sleep 10
+      - name: Run cargo clippy
+        run: cargo clippy --all-targets -- --deny warnings
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -43,7 +43,7 @@ jobs:
         run: |
           sh ./scripts/prep_tests.sh
           echo "Loading data"
-          sleep 5
+          sleep 10
       - name: Run cargo fmt
         run: cargo fmt --all -- --check
   test:
@@ -60,6 +60,6 @@ jobs:
         run: |
           sh ./scripts/prep_tests.sh
           echo "Loading data"
-          sleep 5
+          sleep 10
       - name: Run cargo test
         run: cargo test

From 139fd7d4b3fa51ecdcca7881fa1c6cf9e62f1bbf Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:32:18 -0500
Subject: [PATCH 18/32] Remove sleep

---
 .github/workflows/testing.yml | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 4458578..b2c5510 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -23,10 +23,7 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: |
-          sh ./scripts/prep_tests.sh
-          echo "Loading data"
-          sleep 10
+        run: sh ./scripts/prep_tests.sh
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -40,10 +37,7 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: |
-          sh ./scripts/prep_tests.sh
-          echo "Loading data"
-          sleep 10
+        run: sh ./scripts/prep_tests.sh
       - name: Run cargo fmt
         run: cargo fmt --all -- --check
   test:
@@ -57,9 +51,6 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data
-        run: |
-          sh ./scripts/prep_tests.sh
-          echo "Loading data"
-          sleep 10
+        run: sh ./scripts/prep_tests.sh
       - name: Run cargo test
         run: cargo test

From a58990e13f1c83af8c38b3ffdc01e5fd5fda012b Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:33:03 -0500
Subject: [PATCH 19/32] Run rust format

---
 pieces/intro/rust/src/main.rs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/pieces/intro/rust/src/main.rs b/pieces/intro/rust/src/main.rs
index beb4b55..609b64e 100644
--- a/pieces/intro/rust/src/main.rs
+++ b/pieces/intro/rust/src/main.rs
@@ -153,11 +153,18 @@ fn run8() {
     // Check for presence of value
     let value = "AMD Ryzen 3";
     let mut values = processors.values();
-    println!("Is \"AMD Ryzen 3\" in the hashmap of processors?: {}", values.any(|v| v == &value));
+    println!(
+        "Is \"AMD Ryzen 3\" in the hashmap of processors?: {}",
+        values.any(|v| v == &value)
+    );
     // Lookup by key
     let key = "13900KS";
     let lookup_by_key = processors.get(key);
-    println!("Key \"{}\" has the value \"{}\"", key, lookup_by_key.unwrap());
+    println!(
+        "Key \"{}\" has the value \"{}\"",
+        key,
+        lookup_by_key.unwrap()
+    );
     /*
     Is "AMD Ryzen 3" in the hashmap of processors?: true
     Key "13900KS" has the value "Intel Core i9"
@@ -178,7 +185,10 @@ fn run9() {
     processors.insert("AMD Ryzen 5");
     // Check for presence of value
     let value = "AMD Ryzen 3";
-    println!("Is \"AMD Ryzen 3\" in the hashset of processors?: {}", processors.contains(&value));
+    println!(
+        "Is \"AMD Ryzen 3\" in the hashset of processors?: {}",
+        processors.contains(&value)
+    );
     /*
     Is "AMD Ryzen 3" in the hashset of processors?: true
     */

From 09b15019018a9d7c54055d5cbc1460135591b935 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 11:57:55 -0500
Subject: [PATCH 20/32] Append environment vairable

---
 .github/workflows/testing.yml        | 6 +++---
 Cargo.lock                           | 1 +
 pieces/postgres_etl/rust/src/main.rs | 1 -
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index b2c5510..270607f 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -25,7 +25,7 @@ jobs:
       - name: Prep data # This prevents sqlx errors if the database isn't running
         run: sh ./scripts/prep_tests.sh
       - name: Run cargo clippy
-        run: cargo clippy --all-targets -- --deny warnings
+        run: DATABASE_URL=${{ env.DATABASE_URL }} cargo clippy --all-targets -- --deny warnings
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -39,7 +39,7 @@ jobs:
       - name: Prep data # This prevents sqlx errors if the database isn't running
         run: sh ./scripts/prep_tests.sh
       - name: Run cargo fmt
-        run: cargo fmt --all -- --check
+        run: DATABASE_URL=${{ env.DATABASE_URL }} cargo fmt --all -- --check
   test:
     name: Tests
     runs-on: ubuntu-latest
@@ -52,5 +52,5 @@ jobs:
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data
         run: sh ./scripts/prep_tests.sh
-      - name: Run cargo test
+      - name: DATABASE_URL=${{ env.DATABASE_URL }} Run cargo test
         run: cargo test
diff --git a/Cargo.lock b/Cargo.lock
index bfa217c..fb50673 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1332,6 +1332,7 @@ dependencies = [
  "approx",
  "csv",
  "serde",
+ "tempfile",
 ]
 
 [[package]]
diff --git a/pieces/postgres_etl/rust/src/main.rs b/pieces/postgres_etl/rust/src/main.rs
index d97c2b1..46e8eb3 100644
--- a/pieces/postgres_etl/rust/src/main.rs
+++ b/pieces/postgres_etl/rust/src/main.rs
@@ -51,7 +51,6 @@ async fn main() -> Result<(), sqlx::Error> {
     dotenv().ok();
     // Obtain connection
     let pg_uri = dotenvy::var("DATABASE_URL").unwrap();
-    // let pool = Arc::new(PgPool::connect(&pg_uri).await.unwrap());
     let pool = PgPoolOptions::new()
         .min_connections(5)
         .max_connections(5)

From 6ae1c3085fee62ac4b153d54d3627fe906550ac4 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:09:34 -0500
Subject: [PATCH 21/32] Create .env file in CI

---
 .github/workflows/testing.yml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 270607f..f4cce15 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -24,8 +24,11 @@ jobs:
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data # This prevents sqlx errors if the database isn't running
         run: sh ./scripts/prep_tests.sh
+      - name: Create .env file
+        run: |
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> .env
       - name: Run cargo clippy
-        run: DATABASE_URL=${{ env.DATABASE_URL }} cargo clippy --all-targets -- --deny warnings
+        run: cargo clippy --all-targets -- --deny warnings
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -36,10 +39,8 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
-      - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: sh ./scripts/prep_tests.sh
       - name: Run cargo fmt
-        run: DATABASE_URL=${{ env.DATABASE_URL }} cargo fmt --all -- --check
+        run: cargo fmt --all -- --check
   test:
     name: Tests
     runs-on: ubuntu-latest
@@ -52,5 +53,8 @@ jobs:
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data
         run: sh ./scripts/prep_tests.sh
-      - name: DATABASE_URL=${{ env.DATABASE_URL }} Run cargo test
+      - name: Create .env file
+        run: |
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> .env
+      - name: Run cargo test
         run: cargo test

From b0fe3678aef823fd95469c08f2d3c90841c3a3d9 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:11:40 -0500
Subject: [PATCH 22/32] Add file to rust directory

---
 .github/workflows/testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index f4cce15..d39aa5b 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,7 +26,7 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> .env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -55,6 +55,6 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> .env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
       - name: Run cargo test
         run: cargo test

From ba7911d93358f0ade455d3ae4b315d66ad95a24b Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:14:32 -0500
Subject: [PATCH 23/32] Debug

---
 .github/workflows/testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d39aa5b..0260b50 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,6 +26,7 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
+          echo DATABASE_URL=${{ env.DATABASE_URL }}
           echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings

From a4c5465c34bbff7aef86893601314806f3123b81 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:16:28 -0500
Subject: [PATCH 24/32] More debugging

---
 .github/workflows/testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 0260b50..ca0c790 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -28,6 +28,7 @@ jobs:
         run: |
           echo DATABASE_URL=${{ env.DATABASE_URL }}
           echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
+          cat pieces/postgres_etl/rust/.env
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:

From cf0c8510f17e02e9c626895be59cab120bf8e0ec Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:46:17 -0500
Subject: [PATCH 25/32] Use full path

---
 .github/workflows/testing.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index ca0c790..c242fb0 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,9 +26,7 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }}
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
-          cat pieces/postgres_etl/rust/.env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}]pieces/postgres_etl/rust/.env
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -57,6 +55,6 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> pieces/postgres_etl/rust/.env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}]pieces/postgres_etl/rust/.env
       - name: Run cargo test
         run: cargo test

From eca6a19745ad9d55cb13f5fceeec3ef68616b20c Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:47:31 -0500
Subject: [PATCH 26/32] Fix typo

---
 .github/workflows/testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index c242fb0..d170ee5 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -26,7 +26,7 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}]pieces/postgres_etl/rust/.env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -55,6 +55,6 @@ jobs:
         run: sh ./scripts/prep_tests.sh
       - name: Create .env file
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}]pieces/postgres_etl/rust/.env
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
       - name: Run cargo test
         run: cargo test

From ec3c02b855cf4bbd574e9b9cfca74e0670517ed6 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 12:56:48 -0500
Subject: [PATCH 27/32] Prepare sqlx

---
 .github/workflows/testing.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d170ee5..2c62407 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -22,11 +22,11 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
-      - name: Prep data # This prevents sqlx errors if the database isn't running
-        run: sh ./scripts/prep_tests.sh
-      - name: Create .env file
+      - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
+          sh ./scripts/prep_tests.sh
+          cargo install sqlx-cli
+          cargo sqlx prepare
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -51,10 +51,10 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
-      - name: Prep data
-        run: sh ./scripts/prep_tests.sh
-      - name: Create .env file
+      - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
+          sh ./scripts/prep_tests.sh
+          cargo install sqlx-cli
+          cargo sqlx prepare
       - name: Run cargo test
         run: cargo test

From 825006f50720217080f51a1e80ab88acc646c7b0 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 13:00:40 -0500
Subject: [PATCH 28/32] Add env file

---
 .github/workflows/testing.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 2c62407..d2fae84 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -25,6 +25,7 @@ jobs:
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
           sh ./scripts/prep_tests.sh
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
           cargo install sqlx-cli
           cargo sqlx prepare
       - name: Run cargo clippy
@@ -54,6 +55,7 @@ jobs:
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
           sh ./scripts/prep_tests.sh
+          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
           cargo install sqlx-cli
           cargo sqlx prepare
       - name: Run cargo test

From ed3fbc5c30d058173cccd28944d34c05696ff2cb Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 13:04:13 -0500
Subject: [PATCH 29/32] Use command line flag

---
 .github/workflows/testing.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d2fae84..e67d43b 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -25,9 +25,8 @@ jobs:
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
           sh ./scripts/prep_tests.sh
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
           cargo install sqlx-cli
-          cargo sqlx prepare
+          cargo sqlx prepare --database-url=${{ env.DATABASE_URL }}
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -55,8 +54,7 @@ jobs:
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
         run: |
           sh ./scripts/prep_tests.sh
-          echo DATABASE_URL=${{ env.DATABASE_URL }} >> ${{ github.workspace}}/pieces/postgres_etl/rust/.env
           cargo install sqlx-cli
-          cargo sqlx prepare
+          cargo sqlx prepare --database-url=${{ env.DATABASE_URL }}
       - name: Run cargo test
         run: cargo test

From bf3f4ac2808155f3d37850ac310e66a805004bad Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 13:36:50 -0500
Subject: [PATCH 30/32] Test hard coded password

---
 .github/workflows/testing.yml | 10 ++--------
 scripts/docker-compose.yml    |  3 ++-
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index e67d43b..0043434 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -23,10 +23,7 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
-        run: |
-          sh ./scripts/prep_tests.sh
-          cargo install sqlx-cli
-          cargo sqlx prepare --database-url=${{ env.DATABASE_URL }}
+        run: sh ./scripts/prep_tests.sh
       - name: Run cargo clippy
         run: cargo clippy --all-targets -- --deny warnings
   fmt:
@@ -52,9 +49,6 @@ jobs:
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.2
       - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
-        run: |
-          sh ./scripts/prep_tests.sh
-          cargo install sqlx-cli
-          cargo sqlx prepare --database-url=${{ env.DATABASE_URL }}
+        run: sh ./scripts/prep_tests.sh
       - name: Run cargo test
         run: cargo test
diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index ad61a64..7f891e7 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -6,7 +6,8 @@ services:
     ports:
       - 5432:5432
     environment:
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_PASSWORD=test_password
+      # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
     volumes:
       - ./pg/:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql

From 08db5eed666131ff129f339503e8f7c47b2ec1d2 Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 13:38:15 -0500
Subject: [PATCH 31/32] Remove hard code

---
 scripts/docker-compose.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
index 7f891e7..ad61a64 100644
--- a/scripts/docker-compose.yml
+++ b/scripts/docker-compose.yml
@@ -6,8 +6,7 @@ services:
     ports:
       - 5432:5432
     environment:
-      - POSTGRES_PASSWORD=test_password
-      # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
     volumes:
       - ./pg/:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql

From e28d941df86bbb0844091de771d427d80f01944b Mon Sep 17 00:00:00 2001
From: Paul Sanders <psanders1@gmail.com>
Date: Thu, 11 Jan 2024 16:10:58 -0500
Subject: [PATCH 32/32] Make test data directory

---
 scripts/prep_tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/prep_tests.sh b/scripts/prep_tests.sh
index b1b7cfb..86203d0 100644
--- a/scripts/prep_tests.sh
+++ b/scripts/prep_tests.sh
@@ -1,4 +1,5 @@
 #!bin/bash
 
+mkdir scripts/data
 cp pieces/postgres_etl/data/persons.csv scripts/data
 docker compose -f scripts/docker-compose.yml up -d