Merge pull request #32 from knikolla/refactor_2024_02/sqlite

Use SQLite3 instead of MariaDB

requirements.txt

@@ -1,3 +1,2 @@
 boto3
 dataclasses-json
-mysql-connector-python

src/openstack_billing_db/billing.py

@@ -200,9 +200,10 @@ def write(invoices, output, invoice_month=None):
 def generate_billing(start, end, output, rates,
                      coldfront_data_file=None,
                      invoice_month=None,
-                     upload_to_s3=False):
+                     upload_to_s3=False,
+                     sql_dump_file=None):
 
-    database = model.Database(start=start)
+    database = model.Database(start, sql_dump_file)
 
     invoices = collect_invoice_data_from_openstack(database, start, end, rates)
     if coldfront_data_file:

src/openstack_billing_db/main.py

@@ -38,6 +38,12 @@ def main():
         help=("Path to JSON Output of ColdFront's /api/allocations."
               "Used for populating project names and PIs.")
     )
+    parser.add_argument(
+        "--sql-dump-file",
+        required=True,
+        help=("Path to SQL Dump of Nova DB. Must have been converted to SQLite3"
+              "compatible format using https://github.com/dumblob/mysql2sqlite.")
+    )
     parser.add_argument(
         "--rate-cpu-su",
         default=0,
@@ -110,6 +116,7 @@ def main():
         coldfront_data_file=args.coldfront_data_file,
         invoice_month=args.invoice_month,
         upload_to_s3=args.upload_to_s3,
+        sql_dump_file=args.sql_dump_file,
     )
 
 

src/openstack_billing_db/model.py

@@ -4,9 +4,9 @@
 import datetime
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
+import sqlite3
 from typing import Optional
 
-import mysql.connector
 
 
 @dataclass_json()
@@ -77,6 +77,10 @@ class Instance(object):
 
     @staticmethod
     def _clamp_time(time, min_time, max_time):
+        # Note(knikolla): SQLite gives me a string here, sometimes.
+        if isinstance(time, str):
+            time = datetime.datetime.fromisoformat(time)
+
         if time < min_time:
             time = min_time
         if time > max_time:
@@ -170,13 +174,11 @@ def projects(self) -> list[Project]:
 
 class Database(BaseDatabase):
 
-    def __init__(self, start):
-        self.db_nova = mysql.connector.connect(
-            host="127.0.0.1",
-            database="nova",
-            user="root",
-            password="root",
-        )
+    def __init__(self, start, sql_dump_location: str):
+        self.db_nova = sqlite3.connect(":memory:")
+        self.db_nova.row_factory = sqlite3.Row
+        with open(sql_dump_location, 'r') as sql:
+            self.db_nova.executescript(sql.read())
         self.start = start
 
         self._projects = None
@@ -189,7 +191,7 @@ def projects(self) -> list[Project]:
         return self._projects
 
     def get_events(self, instance_uuid) -> list[InstanceEvent]:
-        cursor = self.db_nova.cursor(dictionary=True)
+        cursor = self.db_nova.cursor()
         cursor.execute(
             f"select created_at, action, message from instance_actions where"
             f" instance_uuid = \"{instance_uuid}\" order by created_at"
@@ -205,7 +207,7 @@ def get_events(self, instance_uuid) -> list[InstanceEvent]:
     def get_instances(self, project) -> list[Instance]:
         instances = []
 
-        cursor = self.db_nova.cursor(dictionary=True)
+        cursor = self.db_nova.cursor()
         cursor.execute(f"""
             select
                 instances.uuid,
@@ -273,7 +275,7 @@ def get_instances(self, project) -> list[Instance]:
 
     def get_projects(self) -> list[Project]:
         cursor = self.db_nova.cursor()
-        cursor.execute("select unique(project_id) from instances")
+        cursor.execute("select distinct project_id from instances")
         return [
             Project(
                 uuid=project[0],