diff --git a/docker/db.py b/docker/db.py index f670713..7ff6c91 100644 --- a/docker/db.py +++ b/docker/db.py @@ -316,15 +316,11 @@ def pgosm_after_import(paths): cmds = ['lua', 'run-sql.lua'] - inner_env = os.environ.copy() - inner_env['PGOSM_CONN'] = connection_string(db_name='pgosm') - output = subprocess.run(cmds, text=True, capture_output=True, cwd=paths['flex_path'], - check=True, - env=inner_env) + check=True) LOGGER.info(f'Post-processing output: \n {output.stderr}') diff --git a/docker/pgosm_flex.py b/docker/pgosm_flex.py index b20530e..f1a0454 100644 --- a/docker/pgosm_flex.py +++ b/docker/pgosm_flex.py @@ -192,6 +192,9 @@ def set_env_vars(region, subregion, srid, language, pgosm_date, layerset, os.environ['PGOSM_LAYERSET'] = layerset + os.environ['PGOSM_CONN'] = db.connection_string(db_name='pgosm') + + def setup_logger(log_file, debug): """Prepares logging. @@ -620,6 +623,7 @@ def run_osm2pgsql(osm2pgsql_command, paths): """ logger = logging.getLogger('pgosm-flex') logger.info(f'Running {osm2pgsql_command}') + output = subprocess.run(osm2pgsql_command.split(), text=True, cwd=paths['flex_path'], diff --git a/flex-config/helpers.lua b/flex-config/helpers.lua index 3c75a1f..10b6eda 100644 --- a/flex-config/helpers.lua +++ b/flex-config/helpers.lua @@ -38,7 +38,7 @@ if pgosm_language_env then pgosm_language = pgosm_language_env print('INFO - Language Code set to ' .. pgosm_language) else - pgosm_language = nil + pgosm_language = '' print('INFO - Default language not set. Using OSM Wiki priority for name. Set PGOSM_LANGUAGE to customize.') end diff --git a/flex-config/run-sql.lua b/flex-config/run-sql.lua index 4013425..a4929b3 100644 --- a/flex-config/run-sql.lua +++ b/flex-config/run-sql.lua @@ -21,8 +21,8 @@ layers = {'amenity', 'building', 'indoor', 'infrastructure', 'landuse' local function post_processing(layerset) - print(string.format('Post-processing %s', layerset)) - local filename = string.format('sql/%s.sql', layerset) + print(string.format('Post-processing %s', layerset)) + local filename = string.format('sql/%s.sql', layerset) local sql_file = io.open(filename, 'r') sql_raw = sql_file:read( '*all' ) sql_file:close() diff --git a/flex-config/sql/pgosm-meta.sql b/flex-config/sql/pgosm-meta.sql index b468100..6bfebf6 100644 --- a/flex-config/sql/pgosm-meta.sql +++ b/flex-config/sql/pgosm-meta.sql @@ -1,11 +1,12 @@ COMMENT ON SCHEMA osm IS 'Schema populated by PgOSM-Flex. SELECT * FROM osm.pgosm_flex; for details.'; -COMMENT ON TABLE osm.pgosm_flex IS 'Provides meta information on the PgOSM-Flex project including version and SRID used during the import.'; +COMMENT ON TABLE osm.pgosm_flex IS 'Provides meta information on the PgOSM-Flex project including version and SRID used during the import. One row per import.'; -COMMENT ON COLUMN osm.pgosm_flex.osm_date IS 'Indicates when the OpenStreetMap data was loaded. Recommended to set PGOSM_DATE env var at runtime, otherwise defaults to the date PgOSM-Flex was run.'; +COMMENT ON COLUMN osm.pgosm_flex.imported IS 'Indicates when the import was ran.'; +COMMENT ON COLUMN osm.pgosm_flex.osm_date IS 'Indicates the date of the OpenStreetMap data loaded. Recommended to set PGOSM_DATE env var at runtime, otherwise defaults to the date PgOSM-Flex was run.'; COMMENT ON COLUMN osm.pgosm_flex.default_date IS 'If true, the value in osm_date represents the date PgOSM-Flex was ran. If False, the date was set via env var and should indicate the date the OpenStreetMap data is from.'; COMMENT ON COLUMN osm.pgosm_flex.project_url IS 'PgOSM-Flex project URL.'; -COMMENT ON COLUMN osm.pgosm_flex.srid IS 'SRID used to run PgOSM-Flex.'; +COMMENT ON COLUMN osm.pgosm_flex.srid IS 'SRID of imported data.'; COMMENT ON COLUMN osm.pgosm_flex.pgosm_flex_version IS 'Version of PgOSM-Flex used to generate schema.'; COMMENT ON COLUMN osm.pgosm_flex.osm2pgsql_version IS 'Version of osm2pgsql used to load data.'; COMMENT ON COLUMN osm.pgosm_flex.region IS 'Region specified at run time via env var PGOSM_REGION.'; -COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE. NULL when not defined.'; +COMMENT ON COLUMN osm.pgosm_flex.language IS 'Preferred language specified at run time via env var PGOSM_LANGUAGE. Empty string when not defined.'; diff --git a/flex-config/style/pgosm-meta.lua b/flex-config/style/pgosm-meta.lua index ce431e5..0f758cb 100644 --- a/flex-config/style/pgosm-meta.lua +++ b/flex-config/style/pgosm-meta.lua @@ -1,22 +1,36 @@ require "helpers" +local driver = require('luasql.postgres') +local env = driver.postgres() + +local pgosm_conn_env = os.getenv("PGOSM_CONN") +local pgosm_conn = nil + +if pgosm_conn_env then + pgosm_conn = pgosm_conn_env +else + error('ENV VAR PGOSM_CONN must be set.') +end + + local tables = {} +sql_create_table = [=[ +CREATE TABLE IF NOT EXISTS osm.pgosm_flex ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + imported TIMESTAMPTZ NOT NULL DEFAULT NOW(), + osm_date date NOT NULL, + default_date bool NOT NULL, + region text NOT NULL, + pgosm_flex_version text NOT NULL, + srid text NOT NULL, + project_url text NOT NULL, + osm2pgsql_version text NOT NULL, + "language" text NOT NULL, + CONSTRAINT pk_osm_pgosm_flex PRIMARY KEY (id) +); +]=] -tables.pgosm_flex_meta = osm2pgsql.define_table({ - name = 'pgosm_flex', - schema = schema_name, - columns = { - { column = 'osm_date', sql_type = 'date', not_null = true }, - { column = 'default_date', type = 'bool', not_null = true }, - { column = 'region', type = 'text', not_null = true}, - { column = 'pgosm_flex_version', type = 'text', not_null = true }, - { column = 'srid', type = 'text', not_null = true }, - { column = 'project_url', type = 'text', not_null = true }, - { column = 'osm2pgsql_version', type = 'text', not_null = true}, - { column = 'language', type = 'text', not_null = false} - } -}) function pgosm_get_commit_hash() @@ -48,35 +62,30 @@ local pgosm_flex_version = git_tag .. '-' .. commit_hash local project_url = 'https://github.com/rustprooflabs/pgosm-flex' --- Couldn't find a better way to only add one row from Lua, adds one row then flips meta_added. -local meta_added = false -function pgosm_meta_load_row(object) - if meta_added then - return - end - - tables.pgosm_flex_meta:add_row({ - pgosm_flex_version = pgosm_flex_version, - srid = srid, - project_url = project_url, - osm2pgsql_version = osm2pgsql_version, - osm_date = pgosm_date, - default_date = default_date, - region = pgosm_region, - language = pgosm_language - }) - - meta_added = true -end +-- Establish connection to Postgres +con = assert (env:connect(pgosm_conn)) + +print('ensuring pgosm_flex table exists.') +con:execute(sql_create_table) --- Choice of way was arbitrary. Could use relation or node. -if osm2pgsql.process_way == nil then - osm2pgsql.process_way = pgosm_meta_load_row +if default_date then + default_date_str = 'true' else - local nested = osm2pgsql.process_way - osm2pgsql.process_way = function(object) - local object_copy = deep_copy(object) - nested(object) - pgosm_meta_load_row(object_copy) - end + default_date_str = 'false' end + +local sql_insert = [[ INSERT INTO osm.pgosm_flex (osm_date, default_date, region, pgosm_flex_version, srid, project_url, osm2pgsql_version, "language") ]] .. + [[ VALUES (']] .. + con:escape(pgosm_date) .. [[', ]] .. + default_date_str .. [[ , ']] .. -- special handling for boolean + con:escape(pgosm_region) .. [[', ']] .. + con:escape(pgosm_flex_version) .. [[', ']] .. + con:escape(srid) .. [[', ']] .. + con:escape(project_url) .. [[', ']] .. + con:escape(osm2pgsql_version) .. [[', ']] .. + con:escape(pgosm_language) .. [[' );]] + + +-- simple query to verify connection +cur = con:execute( sql_insert ) +