diff --git a/HISTORY.md b/HISTORY.md index 04a6ccc..9a68430 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # History +## WIP + +- Add options `--json-ocaml-type`, `--only-matching`, + `--avoid-dangling-refs`, and `--skip-doc`. + ## 0.0.3 - Add decimal string format support - Add unix-time int format support diff --git a/README.md b/README.md index af278b7..a8547c4 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,15 @@ You can call `jsonschema2atd` and `atdgen` in your `dune` file to generate OCaml (run %{bin:atdgen} -t %{deps})))) ``` +Other options can be used to control the output: + +- `--json-ocaml-type KEYWORD:MODULE.PATH:TYPE-NAME` to control the defitiion of + the `json` type used as default/fallback. +- `--only-matching REGEXP` to limit the JSONSchema types to convert, when used + together with `--avoid-dangling-refs`, missing types are replaced with `json`. + +See also `jsonschema2atd --help`. + ## ToDo - [X] Base types diff --git a/bin/main.ml b/bin/main.ml index 66f1e12..91773df 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -12,13 +12,13 @@ module Input_format = struct let all = [ JSONSchema; OpenAPI ] end -let generate_atd input_format paths = +let generate_atd state input_format paths = let generate = match input_format with - | Input_format.JSONSchema -> Generator.make_atd_of_jsonschema - | OpenAPI -> Generator.make_atd_of_openapi + | Input_format.JSONSchema -> Generator.make_atd_of_jsonschema ~state + | OpenAPI -> Generator.make_atd_of_openapi ~state in - print_endline (Generator.base (String.concat " " (List.map Filename.basename paths))); + print_endline (Generator.base state (String.concat " " (List.map Filename.basename paths))); let root = match paths with | [ _ ] -> `Default @@ -44,8 +44,42 @@ let input_format_term = let main = let doc = "Generate an ATD file from a list of JSON Schema / OpenAPI document" in + let state_term = + Term.( + const (fun skip_doc pad toplevel_types avoid_dangling_refs json_ocaml_type -> + Generator. + { + with_doc = not skip_doc; + protect_against_duplicates = (if pad then Some (ref []) else None); + toplevel_types; + avoid_dangling_refs; + json_ocaml_type; + } + ) + $ Arg.(value (flag (info [ "skip-doc" ] ~doc:"Skip documentation annotations."))) + $ Arg.(value (flag (info [ "protect-against-duplicates" ] ~doc:"Make sure no duplicate types are generated."))) + $ (const (function + | [] -> `All + | some -> `Only some + ) + $ Arg.( + value (opt_all string [] (info [ "only-matching" ] ~docv:"REGEXP" ~doc:"Only output types matching REGEXP.")) + ) + ) + $ Arg.(value (flag (info [ "avoid-dangling-refs" ] ~doc:"Convert dangling refs to json."))) + $ Arg.( + let keyword = enum [ "module", `Module; "from", `From ] in + value + (opt (t3 ~sep:':' keyword string string) Generator.default_state.json_ocaml_type + (info [ "json-ocaml-type" ] ~docv:"KEYWORD:MODULE.PATH:TYPE-NAME" + ~doc:"Use an alternate Mod.type for `json`, e.g. from:My_mod.Submod:json_type." + ) + ) + ) + ) + in let paths = Arg.(non_empty & pos_all file [] & info [] ~docv:"FILES" ~doc) in - let term = Term.(const generate_atd $ input_format_term $ paths) in + let term = Term.(const generate_atd $ state_term $ input_format_term $ paths) in let info = Cmd.info "jsonschema2atd" ~doc ~version:(Version.get ()) in Cmd.v info term diff --git a/lib/generator.ml b/lib/generator.ml index 6d3531d..c041aa9 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -2,23 +2,54 @@ open Json_schema_t open Printf open Utils -let record_field_name str = +type state = { + with_doc : bool; + protect_against_duplicates : string list ref option; + toplevel_types : [ `All | `Only of string list ]; + avoid_dangling_refs : bool; + json_ocaml_type : [ `From | `Module ] * string * string; +} + +let default_state = + { + with_doc = true; + protect_against_duplicates = None; + toplevel_types = `All; + avoid_dangling_refs = false; + json_ocaml_type = `Module, "Yojson.Basic", "t"; + } + +let record_field_name _state str = let cleaned_field_name = Utils.sanitize_name str in if String.equal str cleaned_field_name then str else sprintf {|%s |} cleaned_field_name str -let doc_annotation text = sprintf {||} text +let doc_annotation state text = if state.with_doc then sprintf {||} text else "" +let defined_types : string list ref = ref [] -let define_type ~doc ~name ~type_ = +let define_type state ~doc ~name ~type_ = let doc = match doc with | None -> "" - | Some doc -> doc_annotation doc + | Some doc -> doc_annotation state doc in - sprintf "type %s = %s %s\n" (type_name name) type_ doc + let out () = sprintf "type %s = %s %s\n" (type_name name) type_ doc in + begin + match state.protect_against_duplicates with + | None -> out () + | Some defined_types -> begin + match List.exists (( = ) name) !defined_types with + | false -> + defined_types := name :: !defined_types; + out () + | true -> + eprintf "Warning: Ignoring duplicate type %S.\n" name; + "" + end + end -let process_int_type schema = +let process_int_type _state schema = match schema.format with - | None | Some `Int32 | Some `UnixTime -> "int" + | None | Some `Int32 | Some `UnixTime | Some `Enum -> "int" | Some `Int64 -> "int64" | _ -> failwith "int has unexpected format" @@ -86,7 +117,7 @@ let make_atd_default_value ~typ enum json_value = failwith (sprintf "only string enum default values are supported, can't process: %s" (Yojson.Basic.to_string json)) | None, json -> ocaml_value_of_json ~typ json -let nullable = Printf.sprintf "%s nullable" +let nullable = sprintf "%s nullable" let merge_all_of schema = match schema.all_of with @@ -146,17 +177,17 @@ let merge_all_of schema = nullable = schemas |> List.exists (fun schema -> schema.nullable); } -let rec process_schema_type ~ancestors (schema : schema) = - let schema = merge_all_of schema in +let rec process_schema_type state ~ancestors (input_schema : schema) = + let schema = merge_all_of input_schema in let maybe_nullable type_ = if schema.nullable then nullable type_ else type_ in match schema.one_of with - | Some schemas -> process_one_of ~ancestors schemas + | Some schemas -> process_one_of state ~ancestors schemas | None -> match schema.enum, schema.typ with - | Some enums, (Some String | None) -> process_string_enums enums + | Some enums, (Some String | None) -> process_string_enums state enums | Some _, Some Integer -> (* this is more lenient than it should *) - maybe_nullable (process_int_type schema) + maybe_nullable (process_int_type state schema) | Some _, Some Number -> (* this is more lenient than it should *) maybe_nullable "float" @@ -166,44 +197,55 @@ let rec process_schema_type ~ancestors (schema : schema) = | Some _, Some typ -> failwith (Printf.sprintf "only string enums are supported : on field %s got typ %s" (Option.value ~default:"" schema.title) (Json_schema_j.string_of_typ typ)) | None, _ -> match schema.typ with - | Some Integer -> maybe_nullable (process_int_type schema) + | Some Integer -> maybe_nullable (process_int_type state schema) | Some Number -> maybe_nullable "float" | Some String -> maybe_nullable "string" | Some Boolean -> maybe_nullable "bool" - | Some Array -> maybe_nullable (process_array_type ~ancestors schema |> String.concat " ") - | Some Object -> process_object_type ~ancestors schema + | Some Array -> maybe_nullable (process_array_type state ~ancestors schema |> String.concat " ") + | Some Object -> process_object_type state ~ancestors schema | None -> (* fallback to untyped if schema type is not defined *) - maybe_nullable "json" + ksprintf maybe_nullable "json (* %s *)" + (String.concat "," + (List.map + (function + | Ref s -> s + | _ -> "_" + ) + (Option.value ~default:[ Ref (String.concat "/" (List.rev ancestors)) ] input_schema.all_of) + ) + ) -and process_array_type ~ancestors schema = +and process_array_type state ~ancestors schema = match schema.items with - | Some schema_or_ref -> [ make_type_from_schema_or_ref ~ancestors schema_or_ref; "list" ] + | Some schema_or_ref -> [ make_type_from_schema_or_ref state ~ancestors schema_or_ref; "list" ] | None -> failwith "items is not specified for array" -and process_nested_schema_type ~ancestors schema = +and process_nested_schema_type state ~ancestors schema = match merge_all_of schema with | { one_of = Some _; _ } | { typ = Some Object; properties = Some _; _ } | { enum = Some _; _ } -> let nested_type_name = concat_camelCase (List.rev ancestors) in let nested = - define_type ~name:nested_type_name ~type_:(process_schema_type ~ancestors schema) ~doc:schema.description + define_type state ~name:nested_type_name + ~type_:(process_schema_type state ~ancestors schema) + ~doc:schema.description in Buffer.add_string output (nested ^ "\n"); type_name nested_type_name - | _ -> process_schema_type ~ancestors schema + | _ -> process_schema_type state ~ancestors schema -and process_object_type ~ancestors schema = +and process_object_type state ~ancestors schema = let is_required field_name = List.exists (String.equal field_name) schema.required in let make_record_field (field_name, schema_or_ref) = - let type_ = make_type_from_schema_or_ref ~ancestors:(field_name :: ancestors) schema_or_ref in - let record_field_name = record_field_name field_name in + let type_ = make_type_from_schema_or_ref state ~ancestors:(field_name :: ancestors) schema_or_ref in + let record_field_name = record_field_name state field_name in let doc = let content = match schema_or_ref with | Ref _ -> None | Obj schema -> schema.description in - Option.map doc_annotation content |> Option.value ~default:"" + Option.map (doc_annotation state) content |> Option.value ~default:"" in match schema_or_ref, is_required field_name with | Obj { default = Some default; enum; typ; _ }, _ -> @@ -212,33 +254,41 @@ and process_object_type ~ancestors schema = | _, false -> sprintf " ?%s %s: %s option;" record_field_name doc type_ in match schema.properties with + | Some [] -> sprintf "{\n dummy: unit\n}" | Some properties -> sprintf "{\n%s\n}" (properties |> List.map make_record_field |> String.concat "\n") | None -> "json" -and make_type_from_schema_or_ref ~ancestors (schema_or_ref : schema or_ref) = +and make_type_from_schema_or_ref state ~ancestors (schema_or_ref : schema or_ref) = match schema_or_ref, ancestors with - | Obj schema, ([] | [ _ ]) -> process_schema_type ~ancestors schema - | Obj schema, ancestors -> process_nested_schema_type ~ancestors schema - | Ref ref_, _ -> type_name (get_ref_name ref_) + | Obj schema, ([] | [ _ ]) -> process_schema_type state ~ancestors schema + | Obj schema, ancestors -> process_nested_schema_type state ~ancestors schema + | Ref ref_, _ -> begin + match + (not state.avoid_dangling_refs) + || List.exists (fun (name, _schema) -> String.equal (get_ref_name ref_) name) !input_toplevel_schemas + with + | true -> type_name (get_ref_name ref_) + | false -> sprintf "json (* %s *)" ref_ + end -and process_one_of ~ancestors (schemas_or_refs : schema or_ref list) = +and process_one_of state ~ancestors (schemas_or_refs : schema or_ref list) = let determine_variant_name = function | Ref ref_ -> variant_name (get_ref_name ref_) | Obj schema -> match (merge_all_of schema).typ with - | Some Array -> concat_camelCase (process_array_type ~ancestors schema) + | Some Array -> concat_camelCase (process_array_type state ~ancestors schema) | Some Object -> "Json" | _ -> variant_name (process_schema_type ~ancestors schema) in let make_one_of_variant schema_or_ref = let variant_name = determine_variant_name schema_or_ref in sprintf " | %s of %s" variant_name - (make_type_from_schema_or_ref ~ancestors:(variant_name :: ancestors) schema_or_ref) + (make_type_from_schema_or_ref state ~ancestors:(variant_name :: ancestors) schema_or_ref) in let variants = List.map make_one_of_variant schemas_or_refs |> String.concat "\n" in sprintf "[\n%s\n] " variants -and process_string_enums enums = +and process_string_enums _state enums = let enums = List.filter_map (function @@ -256,7 +306,7 @@ and process_string_enums enums = let variants = List.map make_enum_variant enums |> String.concat "\n" in sprintf "[\n%s\n]" variants -let process_schemas (schemas : (string * schema or_ref) list) = +let process_schemas state (schemas : (string * schema or_ref) list) = List.fold_left (fun acc (name, schema_or_ref) -> let doc = @@ -264,19 +314,32 @@ let process_schemas (schemas : (string * schema or_ref) list) = | Ref _ -> None | Obj schema -> schema.description in - define_type ~doc ~name ~type_:(make_type_from_schema_or_ref ~ancestors:[ name ] schema_or_ref) :: acc + define_type state ~doc ~name ~type_:(make_type_from_schema_or_ref ~ancestors:[ name ] schema_or_ref) :: acc ) [] schemas -let base from = +let base state from = + let keyword, module_path, type_t = state.json_ocaml_type in sprintf {|(* Generated by jsonschema2atd from %s *) -type json = abstract +type json = abstract type int64 = int |} from + ( match keyword with + | `From -> "from" + | `Module -> "module" + ) + module_path type_t -let make_atd_of_schemas schemas = +let make_atd_of_schemas state schemas = + let schemas = + match state.toplevel_types with + | `All -> schemas + | `Only l -> + let res = List.map (ksprintf Str.regexp "^%s$") l in + List.filter (fun (name, _) -> List.exists (fun re -> Str.string_match re name 0) res) schemas + in input_toplevel_schemas := List.filter_map (function @@ -285,23 +348,23 @@ let make_atd_of_schemas schemas = ) schemas; Buffer.clear output; - Buffer.add_string output (String.concat "\n" (process_schemas schemas)); + Buffer.add_string output (String.concat "\n" (process_schemas state schemas)); Buffer.contents output -let make_atd_of_jsonschema ?(root = "root") input = +let make_atd_of_jsonschema ?(root = "root") ?(state = default_state) input = let schema = Json_schema_j.schema_of_string input in let root_type_name = Option.value ~default:root schema.title in let defs = let defs = List.concat_map Utils.list_of_nonempty [ schema.defs; schema.definitions ] in List.map (fun (name, schema) -> name, Obj schema) defs in - make_atd_of_schemas ([ root_type_name, Obj schema ] @ defs) + make_atd_of_schemas state ([ root_type_name, Obj schema ] @ defs) -let make_atd_of_openapi ?root:_ input = +let make_atd_of_openapi ?root:_ ?(state = default_state) input = let root = Openapi_j.root_of_string input in match root.components with | None -> failwith "components are empty" | Some components -> match components.schemas with - | Some schemas -> make_atd_of_schemas schemas + | Some schemas -> make_atd_of_schemas state schemas | None -> failwith "components schemas are empty" diff --git a/lib/json_schema.atd b/lib/json_schema.atd index 1d97f26..0357473 100644 --- a/lib/json_schema.atd +++ b/lib/json_schema.atd @@ -5,6 +5,11 @@ type 'a or_ref = [ | Ref of ref_ ] +type 'a or_bool = [ + | Obj of 'a + | Bool of bool +] + type typ = [ | Integer | Number @@ -18,6 +23,7 @@ type int_format = [ | Int32 | Int64 | UnixTime + | Enum ] type number_format = [ @@ -33,6 +39,8 @@ type str_format = [ | Email | Idn_email | Decimal + | Bytes + | Field_mask | Uri | Binary | Uuid @@ -65,7 +73,7 @@ type schema = { (* 10.3.2 keywords for applying subschemas to objects *) ~properties : (string * schema or_ref) list nullable; - ~additional_properties : schema nullable; + ~additional_properties : schema or_bool nullable; (* fields from Json Schema Validation https://json-schema.org/draft/2020-12/json-schema-validation.html *) diff --git a/lib/json_schema_adapters.ml b/lib/json_schema_adapters.ml index 5805fa1..009e68a 100644 --- a/lib/json_schema_adapters.ml +++ b/lib/json_schema_adapters.ml @@ -9,4 +9,15 @@ module Or_ref = struct | x -> x end +module Or_bool = struct + let normalize : Yojson.Safe.t -> Yojson.Safe.t = function + | `Bool b -> `List [ `String "Bool"; `Bool b ] + | obj -> `List [ `String "Obj"; obj ] + + let restore = function + | `List [ `String "Bool"; `Bool b ] -> `Bool b + | `List [ `String "Obj"; obj ] -> obj + | x -> x +end + module Ref = Utils.Fresh (String) () diff --git a/tests/grok.t b/tests/grok.t index d0511d6..8c0547d 100644 --- a/tests/grok.t +++ b/tests/grok.t @@ -337,7 +337,7 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types type matcherConfig = { ~id : string; - ?options : json option; + ?options : json (* MatcherConfig/options *) option; } type mappingType = [ @@ -388,7 +388,7 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types ?mappings : valueMapping list option; ?thresholds : thresholdsConfig option; ?color : fieldColor option; - ?links : json list option; + ?links : json (* FieldConfig/links *) list option; ?noValue : string option; ?custom : json option; } @@ -425,14 +425,14 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types type dynamicConfigValue = { ~id : string; - ?value : json option; + ?value : json (* DynamicConfigValue/value *) option; } type dataTransformerConfig = { id : string; ?disabled : bool option; ?filter : matcherConfig option; - options : json; + options : json (* DataTransformerConfig/options *); } type dataSourceRef = { @@ -487,3 +487,49 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types type annotationContainer = { ?list : annotationQuery list option; } + +Generate ATD types from grok, only partially using `--only-matching` + $ jsonschema2atd --format openapi --only-matching 'Annotation.*' ./mocks/dashboard_types_gen.json + (* Generated by jsonschema2atd from dashboard_types_gen.json *) + type json = abstract + type int64 = int + + type annotationTarget = { + limit : int64; + matchAny : bool; + tags : string list; + type_ : string; + } + + type annotationQuery = { + name : string; + datasource : dataSourceRef; + ~enable : bool; + ~hide : bool; + iconColor : string; + ?filter : annotationPanelFilter option; + ?target : annotationTarget option; + ?type_ : string option; + ~builtIn : float; + } + + type annotationPanelFilter = { + ~exclude : bool; + ids : int list; + } + + type annotationContainer = { + ?list : annotationQuery list option; + } + +Generate ATD types from grok, only partially using `--only-matching`, `--avoid-dangling` and with a custom JSON type + $ jsonschema2atd --format openapi --only-matching 'FieldColor' --json-ocaml-type from:CustomJson:json --avoid-dangling-refs ./mocks/dashboard_types_gen.json + (* Generated by jsonschema2atd from dashboard_types_gen.json *) + type json = abstract + type int64 = int + + type fieldColor = { + mode : json (* #/components/schemas/FieldColorModeId *); + ?fixedColor : string option; + ?seriesBy : json (* #/components/schemas/FieldColorSeriesByMode *) option; + } diff --git a/tests/smoke.t b/tests/smoke.t index a75ba66..4bc8ede 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -89,6 +89,13 @@ Generate ATD out of JSON Schema that uses references ?ff : address option; } +Generate ATD out of JSON Schema that uses references + $ jsonschema2atd --format=jsonschema --only-matching nothing ./mocks/jsonschema_refs.json + (* Generated by jsonschema2atd from jsonschema_refs.json *) + type json = abstract + type int64 = int + + Generate ATD out of JSON Schema that uses enums $ jsonschema2atd --format=jsonschema ./mocks/jsonschema_enums.json (* Generated by jsonschema2atd from jsonschema_enums.json *)