Improve pgosm_flex meta table

docker/db.py

@@ -316,15 +316,11 @@ def pgosm_after_import(paths):
 
     cmds = ['lua', 'run-sql.lua']
 
-    inner_env = os.environ.copy()
-    inner_env['PGOSM_CONN'] = connection_string(db_name='pgosm')
-
     output = subprocess.run(cmds,
                             text=True,
                             capture_output=True,
                             cwd=paths['flex_path'],
-                            check=True,
-                            env=inner_env)
+                            check=True)
     LOGGER.info(f'Post-processing output: \n {output.stderr}')
 
 

docker/pgosm_flex.py

@@ -192,6 +192,9 @@ def set_env_vars(region, subregion, srid, language, pgosm_date, layerset,
 
     os.environ['PGOSM_LAYERSET'] = layerset
 
+    os.environ['PGOSM_CONN'] = db.connection_string(db_name='pgosm')
+
+
 
 def setup_logger(log_file, debug):
     """Prepares logging.
@@ -620,6 +623,7 @@ def run_osm2pgsql(osm2pgsql_command, paths):
     """
     logger = logging.getLogger('pgosm-flex')
     logger.info(f'Running {osm2pgsql_command}')
+
     output = subprocess.run(osm2pgsql_command.split(),
                             text=True,
                             cwd=paths['flex_path'],

flex-config/helpers.lua

@@ -38,7 +38,7 @@ if pgosm_language_env then
     pgosm_language = pgosm_language_env
     print('INFO - Language Code set to ' .. pgosm_language)
 else
-    pgosm_language = nil
+    pgosm_language = ''
     print('INFO - Default language not set. Using OSM Wiki priority for name. Set PGOSM_LANGUAGE to customize.')
 end
 

flex-config/run-sql.lua

@@ -21,8 +21,8 @@ layers = {'amenity', 'building', 'indoor', 'infrastructure', 'landuse'
 
 
 local function post_processing(layerset)
-	print(string.format('Post-processing %s', layerset))
-	local filename = string.format('sql/%s.sql', layerset)
+    print(string.format('Post-processing %s', layerset))
+    local filename = string.format('sql/%s.sql', layerset)
     local sql_file = io.open(filename, 'r')
     sql_raw = sql_file:read( '*all' )
     sql_file:close()

flex-config/sql/pgosm-meta.sql

@@ -1,11 +1,12 @@
 COMMENT ON SCHEMA osm IS 'Schema populated by PgOSM-Flex.  SELECT * FROM osm.pgosm_flex; for details.';
-COMMENT ON TABLE osm.pgosm_flex IS 'Provides meta information on the PgOSM-Flex project including version and SRID used during the import.';
+COMMENT ON TABLE osm.pgosm_flex IS 'Provides meta information on the PgOSM-Flex project including version and SRID used during the import. One row per import.';
 
-COMMENT ON COLUMN osm.pgosm_flex.osm_date IS 'Indicates when the OpenStreetMap data was loaded.  Recommended to set PGOSM_DATE env var at runtime, otherwise defaults to the date PgOSM-Flex was run.';
+COMMENT ON COLUMN osm.pgosm_flex.imported IS 'Indicates when the import was ran.';
+COMMENT ON COLUMN osm.pgosm_flex.osm_date IS 'Indicates the date of the OpenStreetMap data loaded.  Recommended to set PGOSM_DATE env var at runtime, otherwise defaults to the date PgOSM-Flex was run.';
 COMMENT ON COLUMN osm.pgosm_flex.default_date IS 'If true, the value in osm_date represents the date PgOSM-Flex was ran.  If False, the date was set via env var and should indicate the date the OpenStreetMap data is from.';
 COMMENT ON COLUMN osm.pgosm_flex.project_url IS 'PgOSM-Flex project URL.';
-COMMENT ON COLUMN osm.pgosm_flex.srid IS 'SRID used to run PgOSM-Flex.';
+COMMENT ON COLUMN osm.pgosm_flex.srid IS 'SRID of imported data.';
 COMMENT ON COLUMN osm.pgosm_flex.pgosm_flex_version IS 'Version of PgOSM-Flex used to generate schema.';
 COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_version IS 'Version of osm2pgsql used to load data.';
 COMMENT ON COLUMN osm.pgosm_flex.region IS 'Region specified at run time via env var PGOSM_REGION.';
-COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE.  NULL when not defined.';
+COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE.  Empty string when not defined.';

flex-config/style/pgosm-meta.lua

@@ -1,22 +1,36 @@
 require "helpers"
 
+local driver = require('luasql.postgres')
+local env = driver.postgres()
+
+local pgosm_conn_env = os.getenv("PGOSM_CONN")
+local pgosm_conn = nil
+
+if pgosm_conn_env then
+    pgosm_conn = pgosm_conn_env
+else
+    error('ENV VAR PGOSM_CONN must be set.')
+end
+
+
 local tables = {}
 
+sql_create_table = [=[
+CREATE TABLE IF NOT EXISTS osm.pgosm_flex (
+    id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
+    imported TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    osm_date date NOT NULL,
+    default_date bool NOT NULL,
+    region text NOT NULL,
+    pgosm_flex_version text NOT NULL,
+    srid text NOT NULL,
+    project_url text NOT NULL,
+    osm2pgsql_version text NOT NULL,
+    "language" text NOT NULL,
+    CONSTRAINT pk_osm_pgosm_flex PRIMARY KEY (id)
+);
+]=]
 
-tables.pgosm_flex_meta = osm2pgsql.define_table({
-    name = 'pgosm_flex',
-    schema = schema_name,
-    columns = {
-        { column = 'osm_date',            sql_type = 'date', not_null = true },
-        { column = 'default_date',        type = 'bool', not_null = true },
-        { column = 'region',              type = 'text', not_null = true},
-        { column = 'pgosm_flex_version',  type = 'text', not_null = true },
-        { column = 'srid',                type = 'text', not_null = true },
-        { column = 'project_url',         type = 'text', not_null = true },
-        { column = 'osm2pgsql_version',   type = 'text', not_null = true},
-        { column = 'language',            type = 'text', not_null = false}
-    }
-})
 
 
 function pgosm_get_commit_hash()
@@ -48,35 +62,30 @@ local pgosm_flex_version = git_tag .. '-' .. commit_hash
 local project_url = 'https://github.com/rustprooflabs/pgosm-flex'
 
 
--- Couldn't find a better way to only add one row from Lua, adds one row then flips meta_added.
-local meta_added = false
-function pgosm_meta_load_row(object)
-    if meta_added then
-        return
-    end
-
-    tables.pgosm_flex_meta:add_row({
-        pgosm_flex_version = pgosm_flex_version,
-        srid = srid,
-        project_url = project_url,
-        osm2pgsql_version = osm2pgsql_version,
-        osm_date = pgosm_date,
-        default_date = default_date,
-        region = pgosm_region,
-        language = pgosm_language
-    })
-
-    meta_added = true
-end
+-- Establish connection to Postgres
+con = assert (env:connect(pgosm_conn))
+
+print('ensuring pgosm_flex table exists.')
+con:execute(sql_create_table)
 
--- Choice of way was arbitrary.  Could use relation or node.
-if osm2pgsql.process_way == nil then
-    osm2pgsql.process_way = pgosm_meta_load_row
+if default_date then
+    default_date_str = 'true'
 else
-    local nested = osm2pgsql.process_way
-    osm2pgsql.process_way = function(object)
-        local object_copy = deep_copy(object)
-        nested(object)
-        pgosm_meta_load_row(object_copy)
-    end
+    default_date_str = 'false'
 end
+
+local sql_insert = [[ INSERT INTO osm.pgosm_flex (osm_date, default_date, region, pgosm_flex_version, srid, project_url, osm2pgsql_version, "language") ]] ..
+ [[ VALUES (']] ..
+ con:escape(pgosm_date) .. [[', ]] ..
+ default_date_str .. [[ , ']] .. -- special handling for boolean
+ con:escape(pgosm_region) .. [[', ']] ..
+ con:escape(pgosm_flex_version) .. [[', ']] ..
+ con:escape(srid) .. [[', ']] ..
+ con:escape(project_url) .. [[', ']] ..
+ con:escape(osm2pgsql_version) .. [[', ']] ..
+ con:escape(pgosm_language) .. [[' );]]
+
+
+-- simple query to verify connection
+cur = con:execute( sql_insert )
+