--dirs scan mechanism, work in progress - refs #417

datasette/app.py

@@ -31,6 +31,7 @@
     escape_css_string,
     escape_sqlite,
     format_bytes,
+    is_valid_sqlite,
     get_plugins,
     module_from_path,
     sqlite3,
@@ -148,6 +149,7 @@ class Datasette:
     def __init__(
         self,
         files,
+        dirs=None,
         immutables=None,
         cache_headers=True,
         cors=False,
@@ -162,6 +164,7 @@ def __init__(
         version_note=None,
     ):
         immutables = immutables or []
+        self.dirs = dirs or []
         self.files = tuple(files) + tuple(immutables)
         self.immutables = set(immutables)
         if not self.files:
@@ -181,6 +184,7 @@ def __init__(
             if db.name in self.databases:
                 raise Exception("Multiple files with same stem: {}".format(db.name))
             self.add_database(db.name, db)
+        self.scan_dirs()
         self.cache_headers = cache_headers
         self.cors = cors
         self._metadata = metadata or {}
@@ -216,6 +220,25 @@ def add_database(self, name, db):
     def remove_database(self, name):
         self.databases.pop(name)
 
+    def scan_dirs(self):
+        # Recurse through self.dirs looking for new SQLite DBs
+        i = 0
+        for dir in self.dirs:
+            print(dir)
+            for filepath in Path(dir).glob("**/*.db"):
+                print(filepath)
+                if is_valid_sqlite(filepath):
+                    self.add_database(
+                        str(filepath)
+                        .replace("../", "")
+                        .replace("/", "_")
+                        .replace(".db", ""),
+                        Database(self, filepath, is_mutable=True),
+                    )
+                    i += 1
+                    if i >= 20:
+                        break
+
     async def run_sanity_checks(self):
         # Only one check right now, for Spatialite
         for database_name, database in self.databases.items():

datasette/cli.py

@@ -227,6 +227,13 @@ def package(
 
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
+@click.option(
+    "-d",
+    "--dir",
+    type=click.Path(exists=True),
+    help="Directories to scan for SQLite files to serve",
+    multiple=True,
+)
 @click.option(
     "-i",
     "--immutable",
@@ -305,6 +312,7 @@ def package(
 @click.option("--help-config", is_flag=True, help="Show available config options")
 def serve(
     files,
+    dir,
     immutable,
     host,
     port,
@@ -355,6 +363,7 @@ def serve(
     )
     ds = Datasette(
         files,
+        dir,
         immutables=immutable,
         cache_headers=not debug and not reload,
         cors=cors,

datasette/utils/__init__.py

@@ -589,6 +589,27 @@ def to_css_class(s):
     return "-".join(bits)
 
 
+SQLITE_MAGIC = b"SQLite format 3\x00"
+
+
+def is_valid_sqlite(path):
+    if not path.is_file():
+        return False
+    try:
+        with open(path, "rb") as fp:
+            has_magic = fp.read(len(SQLITE_MAGIC)) == SQLITE_MAGIC
+    except PermissionError:
+        return False
+    if not has_magic:
+        return False
+    # Check we can run `select * from sqlite_master`
+    try:
+        sqlite3.connect(str(path)).execute("select * from sqlite_master")
+    except Exception:
+        return False
+    return True
+
+
 def link_or_copy(src, dst):
     # Intended for use in populating a temp directory. We link if possible,
     # but fall back to copying if the temp directory is on a different device

docs/datasette-serve-help.txt

@@ -5,6 +5,7 @@ Usage: datasette serve [OPTIONS] [FILES]...
   Serve up specified SQLite database files with a web UI
 
 Options:
+  -d, --dir PATH            Directories to scan for SQLite files to serve
   -i, --immutable PATH      Database files to open in immutable mode
   -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means only
                             connections from the local machine will be allowed. Use