From e4d9ed7f09ed2de1ab21522bf4ff89e7a795d2ac Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Fri, 11 Aug 2023 16:50:57 +0100 Subject: [PATCH 1/6] Add SNS metadata spec and update some of ours. https://docs.smartnoise.org/sql/metadata.html#yaml-format --- docs/source/_static/config_schema.html | 2 +- sqlsynthgen/json_schemas/config_schema.json | 41 ++++++++++++++++++--- tests/examples/example_config.yaml | 13 ++++++- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/docs/source/_static/config_schema.html b/docs/source/_static/config_schema.html index d81eb9c..da965c5 100644 --- a/docs/source/_static/config_schema.html +++ b/docs/source/_static/config_schema.html @@ -1 +1 @@ - SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

root row_generators_module
Type: string

root story_generators_module
Type: string

root src-stats
Type: array

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

root src-stats src-stats items query
Type: string

root src-stats src-stats items epsilon
Type: number

root src-stats src-stats items delta
Type: number

root src-stats src-stats items snsql-metadata
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!censor_dims).*$
root src-stats src-stats items snsql-metadata ^(?!censor_dims).*$
Type: object

root story_generators
Type: array of object

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

root story_generators story_generators items num_stories_per_pass
Type: integer

root max-unique-constraint-tries
Type: integer

root tables
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object
No Additional Properties

root tables .* vocabulary_table
Type: boolean

root tables .* num_rows_per_pass
Type: integer

root tables .* row_generators
Type: array of object

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file + SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

root row_generators_module
Type: string

root story_generators_module
Type: string

root src-stats
Type: array

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

root src-stats src-stats items query
Type: string

root src-stats src-stats items epsilon
Type: number

root src-stats src-stats items delta
Type: number

root src-stats src-stats items snsql-metadata
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

root story_generators story_generators items num_stories_per_pass
Type: integer

root max-unique-constraint-tries
Type: integer

root tables
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object
No Additional Properties

root tables .* vocabulary_table
Type: boolean

root tables .* num_rows_per_pass
Type: integer

root tables .* row_generators
Type: array of object

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file diff --git a/sqlsynthgen/json_schemas/config_schema.json b/sqlsynthgen/json_schemas/config_schema.json index 7972577..e954652 100644 --- a/sqlsynthgen/json_schemas/config_schema.json +++ b/sqlsynthgen/json_schemas/config_schema.json @@ -14,7 +14,6 @@ "story_generators_module": { "type": "string" }, - "src-stats": { "type": "array", "items": { @@ -36,26 +35,55 @@ "delta": { "type": "number" }, - "snsql-metadata": { "type": "object", "properties": { + "max_ids": { + "type": "integer" + }, + "row_privacy": { + "type": "boolean" + }, + "sample_max_ids": { + "type": "boolean" + }, "censor_dims": { "type": "boolean" + }, + "clamp_counts": { + "type": "boolean" + }, + "clamp_columns": { + "type": "boolean" + }, + "use_dpsu": { + "type": "boolean" } }, "patternProperties": { - "^(?!censor_dims).*$": { + "^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$": { "type": "object", + "additionalProperties": false, + "required": ["type"], "properties": { - "name": { - "type": "string" - }, "type": { "type": "string" }, "private_id": { "type": "boolean" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + }, + "nullable": { + "type": "boolean" + }, + "missing_value": {}, + "sensitivity": { + "type": "number" } } } @@ -109,6 +137,7 @@ "type": "array", "items": { "type": "object", + "required": ["name", "columns_assigned"], "properties": { "name": { "type": "string" diff --git a/tests/examples/example_config.yaml b/tests/examples/example_config.yaml index fe3b458..d95e9ad 100644 --- a/tests/examples/example_config.yaml +++ b/tests/examples/example_config.yaml @@ -25,13 +25,22 @@ src-stats: snsql-metadata: # You may well want censor_dims to be on, but we turn it off for the # tests to silence a smartnoise-sql nag warning. + max_ids: 10 + row_privacy: True + sample_max_ids: True censor_dims: False + clamp_counts: True + clamp_columns: True + use_dpsu: True person_id: - name: person_id type: int private_id: True + lower: 10 + upper: 100 + nullable: True + missing_value: -1 + sensitivity: 1 research_opt_out: - name: research_opt_out type: boolean private_id: False From eaf41be19ddf8e393e2a11a9d980a80a376c8f6d Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Wed, 23 Aug 2023 09:54:07 +0100 Subject: [PATCH 2/6] WIP --- docs/source/configuration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 848d514..354ad22 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,5 +1,7 @@ Configuration ------------- +SqlSynthGen is configured using a YAML file. The schema for the configuration: + .. raw:: html :file: _static/config_schema.html From 623997f6b2e2d652eebdfadcab51dff3c3cb9fb7 Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:44:24 +0100 Subject: [PATCH 3/6] Fix tests --- docs/source/configuration.rst | 6 +++++- tests/examples/example_config.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 354ad22..0798fd3 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,7 +1,11 @@ Configuration ------------- -SqlSynthGen is configured using a YAML file. The schema for the configuration: +SqlSynthGen is configured using a YAML file, which is passed to several commands with the ``--config`` option. +Throughout the docs, we will refer to this file as ``config.yaml`` but it can be called anything (the exception being that there will be a naming conflict if you have a vocabulary table called ``config``). + +Below, we see the schema for the configuration file. +Note that our config file format includes a section of SmartNoise SQL metadata, which is explained more fully `here `_. .. raw:: html :file: _static/config_schema.html diff --git a/tests/examples/example_config.yaml b/tests/examples/example_config.yaml index d95e9ad..8bb7fdd 100644 --- a/tests/examples/example_config.yaml +++ b/tests/examples/example_config.yaml @@ -26,20 +26,20 @@ src-stats: # You may well want censor_dims to be on, but we turn it off for the # tests to silence a smartnoise-sql nag warning. max_ids: 10 - row_privacy: True + row_privacy: False sample_max_ids: True censor_dims: False - clamp_counts: True - clamp_columns: True - use_dpsu: True + clamp_counts: False + clamp_columns: False + use_dpsu: False person_id: type: int private_id: True lower: 10 upper: 100 nullable: True - missing_value: -1 sensitivity: 1 + # missing_value: breaks things research_opt_out: type: boolean private_id: False From 16c35613e12c70205c228575bf10f0b3467dd90e Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Wed, 23 Aug 2023 17:09:27 +0100 Subject: [PATCH 4/6] WIP config descriptions --- docs/source/_static/config_schema.html | 2 +- sqlsynthgen/json_schemas/config_schema.json | 22 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/source/_static/config_schema.html b/docs/source/_static/config_schema.html index da965c5..f4c7237 100644 --- a/docs/source/_static/config_schema.html +++ b/docs/source/_static/config_schema.html @@ -1 +1 @@ - SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

root row_generators_module
Type: string

root story_generators_module
Type: string

root src-stats
Type: array

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

root src-stats src-stats items query
Type: string

root src-stats src-stats items epsilon
Type: number

root src-stats src-stats items delta
Type: number

root src-stats src-stats items snsql-metadata
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

root story_generators story_generators items num_stories_per_pass
Type: integer

root max-unique-constraint-tries
Type: integer

root tables
Type: object

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object
No Additional Properties

root tables .* vocabulary_table
Type: boolean

root tables .* num_rows_per_pass
Type: integer

root tables .* row_generators
Type: array of object

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file + SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

Run source-statistics queries using asyncpg.

root row_generators_module
Type: string

The name of a local Python module of row generators (excluding .py).

root story_generators_module
Type: string

The name of a local Python module of story generators (excluding .py).

root src-stats
Type: array

An array of source statistics queries.

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

A name for the query.

root src-stats src-stats items query
Type: string

A SQL query.

root src-stats src-stats items dp-query
Type: string

A SmartNoise SQL query.

root src-stats src-stats items epsilon
Type: number

The differential privacy epsilon value for the DP query.

root src-stats src-stats items delta
Type: number

The differential privacy delta value for the DP query.

root src-stats src-stats items snsql-metadata
Type: object

See https://docs.smartnoise.org/sql/metadata.html#yaml-format.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

An array of story generators.

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

todo.

root story_generators story_generators items num_stories_per_pass
Type: integer

root max-unique-constraint-tries
Type: integer

todo.

root tables
Type: object

Table configurations.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object

A table configuration.

 No Additional Properties

root tables .* vocabulary_table
Type: boolean

Whether to export the table data.

root tables .* num_rows_per_pass
Type: integer

The number of rows to generate per pass.

root tables .* row_generators
Type: array of object

An array of row generators to create column values.

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items name
Type: string

The name of a (built-in or custom) function.

root tables .* row_generators row_generators items args
Type: array

The arguments to pass to the function.

root tables .* row_generators row_generators items kwargs
Type: object

The keyword arguments to pass to the function.

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

One or more columns to assign the return value to.

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file diff --git a/sqlsynthgen/json_schemas/config_schema.json b/sqlsynthgen/json_schemas/config_schema.json index e954652..9c2f8e5 100644 --- a/sqlsynthgen/json_schemas/config_schema.json +++ b/sqlsynthgen/json_schemas/config_schema.json @@ -6,36 +6,46 @@ "additionalProperties": false, "properties": { "use-asyncio": { + "description": "Run source-statistics queries using asyncpg.", "type": "boolean" }, "row_generators_module": { + "description": "The name of a local Python module of row generators (excluding .py).", "type": "string" }, "story_generators_module": { + "description": "The name of a local Python module of story generators (excluding .py).", "type": "string" }, "src-stats": { + "description": "An array of source statistics queries.", "type": "array", "items": { "additionalProperties": false, "required": ["name", "query"], "properties": { "name": { + "description": "A name for the query.", "type": "string" }, "query": { + "description": "A SQL query.", "type": "string" }, "dp-query": { + "description": "A SmartNoise SQL query.", "type": "string" }, "epsilon": { + "description": "The differential privacy epsilon value for the DP query.", "type": "number" }, "delta": { + "description": "The differential privacy delta value for the DP query.", "type": "number" }, "snsql-metadata": { + "description": "See https://docs.smartnoise.org/sql/metadata.html#yaml-format.", "type": "object", "properties": { "max_ids": { @@ -94,6 +104,7 @@ }, "story_generators": { + "description": "An array of story generators.", "type": "array", "items": { "type": "object", @@ -101,6 +112,7 @@ "required": ["name", "num_stories_per_pass"], "properties": { "name": { + "description": "todo.", "type": "string" }, "args": { @@ -117,38 +129,48 @@ }, "max-unique-constraint-tries": { + "description": "todo.", "type": "integer" }, "tables": { + "description": "Table configurations.", "type": "object", "patternProperties": { ".*": { + "description": "A table configuration.", "additionalProperties": false, "type": "object", "properties": { "vocabulary_table": { + "description": "Whether to export the table data.", "type": "boolean" }, "num_rows_per_pass": { + "description": "The number of rows to generate per pass.", "type": "integer" }, "row_generators": { + "description": "An array of row generators to create column values.", "type": "array", "items": { "type": "object", "required": ["name", "columns_assigned"], "properties": { "name": { + "description": "The name of a (built-in or custom) function.", "type": "string" }, "args": { + "description": "The arguments to pass to the function.", "type": "array" }, "kwargs": { + "description": "The keyword arguments to pass to the function.", "type": "object" }, "columns_assigned": { + "description": "One or more columns to assign the return value to.", "type": ["array", "string"], "items": { "type": "string" From c9f54640e0a448d87281ca1658ccdcc2fc804e13 Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Wed, 23 Aug 2023 17:31:34 +0100 Subject: [PATCH 5/6] WIP config descriptions --- docs/source/_static/config_schema.html | 2 +- sqlsynthgen/json_schemas/config_schema.json | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/source/_static/config_schema.html b/docs/source/_static/config_schema.html index f4c7237..7660629 100644 --- a/docs/source/_static/config_schema.html +++ b/docs/source/_static/config_schema.html @@ -1 +1 @@ - SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

Run source-statistics queries using asyncpg.

root row_generators_module
Type: string

The name of a local Python module of row generators (excluding .py).

root story_generators_module
Type: string

The name of a local Python module of story generators (excluding .py).

root src-stats
Type: array

An array of source statistics queries.

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

A name for the query.

root src-stats src-stats items query
Type: string

A SQL query.

root src-stats src-stats items dp-query
Type: string

A SmartNoise SQL query.

root src-stats src-stats items epsilon
Type: number

The differential privacy epsilon value for the DP query.

root src-stats src-stats items delta
Type: number

The differential privacy delta value for the DP query.

root src-stats src-stats items snsql-metadata
Type: object

See https://docs.smartnoise.org/sql/metadata.html#yaml-format.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

An array of story generators.

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

todo.

root story_generators story_generators items num_stories_per_pass
Type: integer

root max-unique-constraint-tries
Type: integer

todo.

root tables
Type: object

Table configurations.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object

A table configuration.

 No Additional Properties

root tables .* vocabulary_table
Type: boolean

Whether to export the table data.

root tables .* num_rows_per_pass
Type: integer

The number of rows to generate per pass.

root tables .* row_generators
Type: array of object

An array of row generators to create column values.

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items name
Type: string

The name of a (built-in or custom) function.

root tables .* row_generators row_generators items args
Type: array

The arguments to pass to the function.

root tables .* row_generators row_generators items kwargs
Type: object

The keyword arguments to pass to the function.

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

One or more columns to assign the return value to.

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file + SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

Run source-statistics queries using asyncpg.

root row_generators_module
Type: string

The name of a local Python module of row generators (excluding .py).

root story_generators_module
Type: string

The name of a local Python module of story generators (excluding .py).

root src-stats
Type: array

An array of source statistics queries.

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

A name for the query, which will be used in the stats file.

root src-stats src-stats items query
Type: string

A SQL query.

root src-stats src-stats items dp-query
Type: string

A SmartNoise SQL query.

root src-stats src-stats items epsilon
Type: number

The differential privacy epsilon value for the DP query.

root src-stats src-stats items delta
Type: number

The differential privacy delta value for the DP query.

root src-stats src-stats items snsql-metadata
Type: object

See https://docs.smartnoise.org/sql/metadata.html#yaml-format.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

An array of story generators.

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

The full name of a story generator (e.g. mystorygenerators.short_story).

root story_generators story_generators items args
Type: array

Positional arguments to pass to the story generator.

root story_generators story_generators items kwargs
Type: object

Keyword arguments to pass to the story generator.

root story_generators story_generators items num_stories_per_pass
Type: integer

todo

root max-unique-constraint-tries
Type: integer

todo.

root tables
Type: object

Table configurations.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object

A table configuration.

 No Additional Properties

root tables .* vocabulary_table
Type: boolean

Whether to export the table data.

root tables .* num_rows_per_pass
Type: integer

The number of rows to generate per pass.

root tables .* row_generators
Type: array of object

An array of row generators to create column values.

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items name
Type: string

The name of a (built-in or custom) function (e.g. max or myrowgenerators.my_gen).

root tables .* row_generators row_generators items args
Type: array

Positional arguments to pass to the function.

root tables .* row_generators row_generators items kwargs
Type: object

Keyword arguments to pass to the function.

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

One or more columns to assign the return value to.

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file diff --git a/sqlsynthgen/json_schemas/config_schema.json b/sqlsynthgen/json_schemas/config_schema.json index 9c2f8e5..f67d042 100644 --- a/sqlsynthgen/json_schemas/config_schema.json +++ b/sqlsynthgen/json_schemas/config_schema.json @@ -25,7 +25,7 @@ "required": ["name", "query"], "properties": { "name": { - "description": "A name for the query.", + "description": "A name for the query, which will be used in the stats file.", "type": "string" }, "query": { @@ -112,16 +112,19 @@ "required": ["name", "num_stories_per_pass"], "properties": { "name": { - "description": "todo.", + "description": "The full name of a story generator (e.g. my_story_generators.short_story).", "type": "string" }, "args": { + "description": "Positional arguments to pass to the story generator.", "type": "array" }, "kwargs": { + "description": "Keyword arguments to pass to the story generator.", "type": "object" }, "num_stories_per_pass": { + "description": "todo", "type": "integer" } } @@ -158,15 +161,15 @@ "required": ["name", "columns_assigned"], "properties": { "name": { - "description": "The name of a (built-in or custom) function.", + "description": "The name of a (built-in or custom) function (e.g. max or my_row_generators.my_gen).", "type": "string" }, "args": { - "description": "The arguments to pass to the function.", + "description": "Positional arguments to pass to the function.", "type": "array" }, "kwargs": { - "description": "The keyword arguments to pass to the function.", + "description": "Keyword arguments to pass to the function.", "type": "object" }, "columns_assigned": { From da5d3ee99145b2b620b0319fe024e73cbd932e11 Mon Sep 17 00:00:00 2001 From: Iain-S <25081046+Iain-S@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:31:08 +0100 Subject: [PATCH 6/6] Add remaining config descriptions --- docs/source/_static/config_schema.html | 2 +- sqlsynthgen/json_schemas/config_schema.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/_static/config_schema.html b/docs/source/_static/config_schema.html index 7660629..e891b63 100644 --- a/docs/source/_static/config_schema.html +++ b/docs/source/_static/config_schema.html @@ -1 +1 @@ - SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

Run source-statistics queries using asyncpg.

root row_generators_module
Type: string

The name of a local Python module of row generators (excluding .py).

root story_generators_module
Type: string

The name of a local Python module of story generators (excluding .py).

root src-stats
Type: array

An array of source statistics queries.

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

A name for the query, which will be used in the stats file.

root src-stats src-stats items query
Type: string

A SQL query.

root src-stats src-stats items dp-query
Type: string

A SmartNoise SQL query.

root src-stats src-stats items epsilon
Type: number

The differential privacy epsilon value for the DP query.

root src-stats src-stats items delta
Type: number

The differential privacy delta value for the DP query.

root src-stats src-stats items snsql-metadata
Type: object

See https://docs.smartnoise.org/sql/metadata.html#yaml-format.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

An array of story generators.

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

The full name of a story generator (e.g. mystorygenerators.short_story).

root story_generators story_generators items args
Type: array

Positional arguments to pass to the story generator.

root story_generators story_generators items kwargs
Type: object

Keyword arguments to pass to the story generator.

root story_generators story_generators items num_stories_per_pass
Type: integer

todo

root max-unique-constraint-tries
Type: integer

todo.

root tables
Type: object

Table configurations.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object

A table configuration.

 No Additional Properties

root tables .* vocabulary_table
Type: boolean

Whether to export the table data.

root tables .* num_rows_per_pass
Type: integer

The number of rows to generate per pass.

root tables .* row_generators
Type: array of object

An array of row generators to create column values.

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items name
Type: string

The name of a (built-in or custom) function (e.g. max or myrowgenerators.my_gen).

root tables .* row_generators row_generators items args
Type: array

Positional arguments to pass to the function.

root tables .* row_generators row_generators items kwargs
Type: object

Keyword arguments to pass to the function.

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

One or more columns to assign the return value to.

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file + SQLSynthGen Config

SQLSynthGen Config

Type: object

A SQLSynthGen configuration YAML file

 No Additional Properties

root use-asyncio
Type: boolean

Run source-statistics queries using asyncpg.

root row_generators_module
Type: string

The name of a local Python module of row generators (excluding .py).

root story_generators_module
Type: string

The name of a local Python module of story generators (excluding .py).

root src-stats
Type: array

An array of source statistics queries.

Each item of this array must be:

root src-stats src-stats items
Type: object
No Additional Properties

root src-stats src-stats items name
Type: string

A name for the query, which will be used in the stats file.

root src-stats src-stats items query
Type: string

A SQL query.

root src-stats src-stats items dp-query
Type: string

A SmartNoise SQL query.

root src-stats src-stats items epsilon
Type: number

The differential privacy epsilon value for the DP query.

root src-stats src-stats items delta
Type: number

The differential privacy delta value for the DP query.

root src-stats src-stats items snsql-metadata
Type: object

See https://docs.smartnoise.org/sql/metadata.html#yaml-format.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
root src-stats src-stats items snsql-metadata ^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$
Type: object
No Additional Properties

root story_generators
Type: array of object

An array of story generators.

Each item of this array must be:

root story_generators story_generators items
Type: object
No Additional Properties

root story_generators story_generators items name
Type: string

The full name of a story generator (e.g. mystorygenerators.short_story).

root story_generators story_generators items args
Type: array

Positional arguments to pass to the story generator.

root story_generators story_generators items kwargs
Type: object

Keyword arguments to pass to the story generator.

root story_generators story_generators items num_stories_per_pass
Type: integer

The number of times to call the story generator per pass.

root max-unique-constraint-tries
Type: integer

The maximum number of tries to respect a uniqueness constraint.

root tables
Type: object

Table configurations.

All properties whose name matches the following regular expression must respect the following conditions

Property name regular expression: .*
root tables .*
Type: object

A table configuration.

 No Additional Properties

root tables .* vocabulary_table
Type: boolean

Whether to export the table data.

root tables .* num_rows_per_pass
Type: integer

The number of rows to generate per pass.

root tables .* row_generators
Type: array of object

An array of row generators to create column values.

Each item of this array must be:

root tables .* row_generators row_generators items
Type: object

root tables .* row_generators row_generators items name
Type: string

The name of a (built-in or custom) function (e.g. max or myrowgenerators.my_gen).

root tables .* row_generators row_generators items args
Type: array

Positional arguments to pass to the function.

root tables .* row_generators row_generators items kwargs
Type: object

Keyword arguments to pass to the function.

root tables .* row_generators row_generators items columns_assigned
Type: array of string or string

One or more columns to assign the return value to.

Each item of this array must be:

root tables .* row_generators row_generators items columns_assigned columns_assigned items
Type: string
\ No newline at end of file diff --git a/sqlsynthgen/json_schemas/config_schema.json b/sqlsynthgen/json_schemas/config_schema.json index f67d042..f3e3c65 100644 --- a/sqlsynthgen/json_schemas/config_schema.json +++ b/sqlsynthgen/json_schemas/config_schema.json @@ -124,7 +124,7 @@ "type": "object" }, "num_stories_per_pass": { - "description": "todo", + "description": "The number of times to call the story generator per pass.", "type": "integer" } } @@ -132,7 +132,7 @@ }, "max-unique-constraint-tries": { - "description": "todo.", + "description": "The maximum number of tries to respect a uniqueness constraint.", "type": "integer" },