Use SQLite3 instead of MariaDB

This is part of a series of work to reduce the amount of setup
required for running the script and remove as many external
dependencies as possible.

By removing the dependency on MariaDB, this workflow can run in a
container without requiring a sidecar.

All the information that we need for invoicing is present in
two tables from one single database (instances, and instance_extra)
which have a simple foreign key relationship, we're not relying
on any special features from MariaDB that are not present in SQLite.

It is not possible to import a `mysqldump` generated file into Sqlite
though, but there is a script of GitHub that performs the conversion
successfully. https://github.com/dumblob/mysql2sqlite

A follow-up patch will introduce a Dockerfile that includes the
binary and automatically does the conversion process.

- Introduces a `--sql-dump-file` argument. That file is opened and
  executed inside an in-memory sqlite3 database.
- File must have been previously converted using `mysql2sqlite`.
- Slight changes to syntax.
  - Distinct instead of unique()
  - sqlite3.Row as a row_factory instead of providing
    dictionary=True to the cursor.
- Additionally, I had to had a check to _clamp_time that ensures
  the type argument was not of type string, and converted it if so.
  I ran into an error that the provided argument (coming from the db)
  was of type string somehow.

requirements.txt

@@ -1,3 +1,2 @@
 boto3
 dataclasses-json
-mysql-connector-python

src/openstack_billing_db/billing.py

@@ -200,9 +200,10 @@ def write(invoices, output, invoice_month=None):
 def generate_billing(start, end, output, rates,
                      coldfront_data_file=None,
                      invoice_month=None,
-                     upload_to_s3=False):
+                     upload_to_s3=False,
+                     sql_dump_file=None):
 
-    database = model.Database(start=start)
+    database = model.Database(start, sql_dump_file)
 
     invoices = collect_invoice_data_from_openstack(database, start, end, rates)
     if coldfront_data_file:

src/openstack_billing_db/main.py

@@ -38,6 +38,12 @@ def main():
         help=("Path to JSON Output of ColdFront's /api/allocations."
               "Used for populating project names and PIs.")
     )
+    parser.add_argument(
+        "--sql-dump-file",
+        required=True,
+        help=("Path to SQL Dump of Nova DB. Must have been converted to SQLite3"
+              "compatible format using https://github.com/dumblob/mysql2sqlite.")
+    )
     parser.add_argument(
         "--rate-cpu-su",
         default=0,
@@ -110,6 +116,7 @@ def main():
         coldfront_data_file=args.coldfront_data_file,
         invoice_month=args.invoice_month,
         upload_to_s3=args.upload_to_s3,
+        sql_dump_file=args.sql_dump_file,
     )
 
 

src/openstack_billing_db/model.py

@@ -4,9 +4,9 @@
 import datetime
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
+import sqlite3
 from typing import Optional
 
-import mysql.connector
 
 
 @dataclass_json()
@@ -77,6 +77,10 @@ class Instance(object):
 
     @staticmethod
     def _clamp_time(time, min_time, max_time):
+        # Note(knikolla): SQLite gives me a string here, sometimes.
+        if isinstance(time, str):
+            time = datetime.datetime.fromisoformat(time)
+
         if time < min_time:
             time = min_time
         if time > max_time:
@@ -170,13 +174,11 @@ def projects(self) -> list[Project]:
 
 class Database(BaseDatabase):
 
-    def __init__(self, start):
-        self.db_nova = mysql.connector.connect(
-            host="127.0.0.1",
-            database="nova",
-            user="root",
-            password="root",
-        )
+    def __init__(self, start, sql_dump_location: str):
+        self.db_nova = sqlite3.connect(":memory:")
+        self.db_nova.row_factory = sqlite3.Row
+        with open(sql_dump_location, 'r') as sql:
+            self.db_nova.executescript(sql.read())
         self.start = start
 
         self._projects = None
@@ -189,7 +191,7 @@ def projects(self) -> list[Project]:
         return self._projects
 
     def get_events(self, instance_uuid) -> list[InstanceEvent]:
-        cursor = self.db_nova.cursor(dictionary=True)
+        cursor = self.db_nova.cursor()
         cursor.execute(
             f"select created_at, action, message from instance_actions where"
             f" instance_uuid = \"{instance_uuid}\" order by created_at"
@@ -205,7 +207,7 @@ def get_events(self, instance_uuid) -> list[InstanceEvent]:
     def get_instances(self, project) -> list[Instance]:
         instances = []
 
-        cursor = self.db_nova.cursor(dictionary=True)
+        cursor = self.db_nova.cursor()
         cursor.execute(f"""
             select
                 instances.uuid,
@@ -273,7 +275,7 @@ def get_instances(self, project) -> list[Instance]:
 
     def get_projects(self) -> list[Project]:
         cursor = self.db_nova.cursor()
-        cursor.execute("select unique(project_id) from instances")
+        cursor.execute("select distinct project_id from instances")
         return [
             Project(
                 uuid=project[0],