feat: enhance get schema type API with more information. (#633)

* feat: enhance get schema ty API with more information.

* fix: sub schema attribute doc mis-overwritten and add more tests

internal/spec/gpyrpc/gpyrpc.proto

@@ -144,12 +144,11 @@ message ExecProgram_Args {
 
 	// yaml/json: sort keys
 	bool sort_keys = 12;
-	// include schema type path in JSON/YAML result
-	bool include_schema_type_path = 13;
 
 	// -E --external : external packages path
-	repeated CmdExternalPkgSpec external_pkgs = 14;
+	repeated CmdExternalPkgSpec external_pkgs = 13;
 }
+
 message ExecProgram_Result {
 	string json_result = 1;
 	string yaml_result = 2;
@@ -281,7 +280,7 @@ message KeyValuePair {
 }
 
 // ----------------------------------------------------------------------------
-// JSON Schema Lit
+// KCL Type Structure
 // ----------------------------------------------------------------------------
 
 message KclType {
@@ -300,6 +299,11 @@ message KclType {
 	int32 line = 10;
 
 	repeated Decorator decorators = 11;  // schema decorators
+
+	string filename = 12;                // `filename` represents the absolute path of the file name where the attribute is located.
+	string pkg_path = 13;                // `pkg_path` represents the path name of the package where the attribute is located.
+	string description = 14;             // `description` represents the document of the attribute.
+	map<string, Example> examples = 15;  // A map object to hold examples, the key is the example name.
 }
 
 message Decorator {
@@ -308,6 +312,12 @@ message Decorator {
 	map<string, string> keywords = 3;
 }
 
+message Example {
+	string summary = 1;                // Short description for the example.
+	string description = 2;            // Long description for the example.
+	google.protobuf.Any value = 3;     // Embedded literal example.
+}
+
 // ----------------------------------------------------------------------------
 // END
 // ----------------------------------------------------------------------------

kclvm/api/src/capi_test.rs

@@ -105,7 +105,7 @@ where
 fn test_c_api<A, R, F>(svc_name: &str, input: &str, output: &str, wrapper: F)
 where
     A: Message + DeserializeOwned,
-    R: Message + Default + PartialEq + DeserializeOwned + serde::Serialize,
+    R: Message + Default + PartialEq + DeserializeOwned + serde::Serialize + ?Sized,
     F: Fn(&mut R),
 {
     let _test_lock = TEST_MUTEX.lock().unwrap();
@@ -122,6 +122,7 @@ where
     let result = unsafe { CStr::from_ptr(result_ptr) };
 
     let mut result = R::decode(result.to_bytes()).unwrap();
+    let result_json = serde_json::to_string(&result).unwrap();
     let except_result_path = Path::new(TEST_DATA_PATH).join(output);
     let except_result_json = fs::read_to_string(&except_result_path).unwrap_or_else(|_| {
         panic!(
@@ -132,7 +133,7 @@ where
     let mut except_result = serde_json::from_str::<R>(&except_result_json).unwrap();
     wrapper(&mut result);
     wrapper(&mut except_result);
-    assert_eq!(result, except_result);
+    assert_eq!(result, except_result, "\nresult json is {result_json}");
     unsafe {
         kclvm_service_delete(serv);
         kclvm_service_free_string(result_ptr);

kclvm/api/src/service/service_impl.rs

@@ -313,7 +313,6 @@ impl KclvmServiceImpl {
             &FormatOptions {
                 recursively,
                 is_stdout: false,
-                ..Default::default()
             },
         )?;
         Ok(FormatPathResult { changed_paths })

kclvm/api/src/service/ty.rs

@@ -47,6 +47,9 @@ pub(crate) fn kcl_schema_ty_to_pb_ty(schema_ty: &SchemaType) -> KclType {
                 ..Default::default()
             })
             .collect(),
+        filename: schema_ty.filename.clone(),
+        pkg_path: schema_ty.pkgpath.clone(),
+        description: schema_ty.doc.clone(),
         ..Default::default()
     }
 }
@@ -62,6 +65,7 @@ fn get_schema_ty_attributes(schema_ty: &SchemaType, line: &mut i32) -> HashMap<S
         if key != SCHEMA_SETTINGS_ATTR_NAME {
             let mut ty = kcl_ty_to_pb_ty(&attr.ty);
             ty.line = *line;
+            ty.description = attr.doc.clone().unwrap_or_default();
             type_mapping.insert(key.to_string(), ty);
             *line += 1
         }
@@ -79,8 +83,8 @@ fn get_schema_ty_required_attributes(schema_ty: &SchemaType) -> Vec<String> {
         Vec::new()
     };
     let mut attr_set = IndexSet::new();
-    for (key, _) in &schema_ty.attrs {
-        if key != SCHEMA_SETTINGS_ATTR_NAME {
+    for (key, attr) in &schema_ty.attrs {
+        if key != SCHEMA_SETTINGS_ATTR_NAME && !attr.is_optional {
             attr_set.insert(key.to_string());
         }
     }

kclvm/api/src/testdata/get-schema-type-mapping.json

@@ -1,4 +1,4 @@
 {
 	"file": "schema.k",
-	"code": "schema Person:\n    name: str\n    age: int\n\nperson = Person {\n    name = \"Alice\"\n    age = 18\n}\n"
+	"code": "schema Server:\n    \"\"\"Server is the common user interface for long-running\n    services adopting the best practice of Kubernetes.\n\n    Attributes\n    ----------\n    workloadType: str, default is \"Deployment\", required\n        Use this attribute to specify which kind of long-running service you want.\n        Valid values: Deployment, CafeDeployment.\n        See also: kusion_models/core/v1/workload_metadata.k.\n    name: str, required\n        A Server-level attribute.\n        The name of the long-running service.\n        See also: kusion_models/core/v1/metadata.k.\n    labels: {str:str}, optional\n        A Server-level attribute.\n        The labels of the long-running service.\n        See also: kusion_models/core/v1/metadata.k.\n\n    Examples\n    ----------------------\n    myCustomApp = AppConfiguration {\n        name = \"componentName\"\n    }\n    \"\"\"\n    workloadType: str = \"Deployment\"\n    name: str\n    labels?: {str:str}\n    containers: [Container]\n\nschema Container:\n    \"\"\"Container is the common user interface for long-running services.\n\n    Attributes\n    ----------\n    name: str, required\n        The name of the long-running container.\n    \"\"\"\n    name: str\n"
 }

kclvm/api/src/testdata/get-schema-type-mapping.response.json

@@ -1,24 +1,76 @@
 {
     "schema_type_mapping": {
-        "person": {
+        "Server": {
             "type": "schema",
             "union_types": [],
             "default": "",
-            "schema_name": "Person",
-            "schema_doc": "",
+            "schema_name": "Server",
+            "schema_doc": "Server is the common user interface for long-running services adopting the best practice of Kubernetes.",
             "properties": {
-                "age": {
-                    "type": "int",
+                "containers": {
+                    "type": "list",
                     "union_types": [],
                     "default": "",
                     "schema_name": "",
                     "schema_doc": "",
                     "properties": {},
                     "required": [],
-                    "line": 2,
-                    "decorators": []
+                    "item": {
+                        "type": "schema",
+                        "union_types": [],
+                        "default": "",
+                        "schema_name": "Container",
+                        "schema_doc": "Container is the common user interface for long-running services.",
+                        "properties": {
+                            "name": {
+                                "type": "str",
+                                "union_types": [],
+                                "default": "",
+                                "schema_name": "",
+                                "schema_doc": "",
+                                "properties": {},
+                                "required": [],
+                                "line": 1,
+                                "decorators": [],
+                                "filename": "",
+                                "pkg_path": "",
+                                "description": "The name of the long-running container.",
+                                "examples": {}
+                            }
+                        },
+                        "required": [
+                            "name"
+                        ],
+                        "line": 0,
+                        "decorators": [],
+                        "filename": "schema.k",
+                        "pkg_path": "__main__",
+                        "description": "Container is the common user interface for long-running services.",
+                        "examples": {}
+                    },
+                    "line": 4,
+                    "decorators": [],
+                    "filename": "",
+                    "pkg_path": "",
+                    "description": "",
+                    "examples": {}
                 },
                 "name": {
+                    "type": "str",
+                    "union_types": [],
+                    "default": "",
+                    "schema_name": "",
+                    "schema_doc": "",
+                    "properties": {},
+                    "required": [],
+                    "line": 2,
+                    "decorators": [],
+                    "filename": "",
+                    "pkg_path": "",
+                    "description": "A Server-level attribute.\nThe name of the long-running service.\nSee also: kusion_models/core/v1/metadata.k.",
+                    "examples": {}
+                },
+                "workloadType": {
                     "type": "str",
                     "union_types": [],
                     "default": "",
@@ -27,22 +79,76 @@
                     "properties": {},
                     "required": [],
                     "line": 1,
-                    "decorators": []
+                    "decorators": [],
+                    "filename": "",
+                    "pkg_path": "",
+                    "description": "Use this attribute to specify which kind of long-running service you want.\nValid values: Deployment, CafeDeployment.\nSee also: kusion_models/core/v1/workload_metadata.k.",
+                    "examples": {}
+                },
+                "labels": {
+                    "type": "dict",
+                    "union_types": [],
+                    "default": "",
+                    "schema_name": "",
+                    "schema_doc": "",
+                    "properties": {},
+                    "required": [],
+                    "key": {
+                        "type": "str",
+                        "union_types": [],
+                        "default": "",
+                        "schema_name": "",
+                        "schema_doc": "",
+                        "properties": {},
+                        "required": [],
+                        "line": 0,
+                        "decorators": [],
+                        "filename": "",
+                        "pkg_path": "",
+                        "description": "",
+                        "examples": {}
+                    },
+                    "item": {
+                        "type": "str",
+                        "union_types": [],
+                        "default": "",
+                        "schema_name": "",
+                        "schema_doc": "",
+                        "properties": {},
+                        "required": [],
+                        "line": 0,
+                        "decorators": [],
+                        "filename": "",
+                        "pkg_path": "",
+                        "description": "",
+                        "examples": {}
+                    },
+                    "line": 3,
+                    "decorators": [],
+                    "filename": "",
+                    "pkg_path": "",
+                    "description": "A Server-level attribute.\nThe labels of the long-running service.\nSee also: kusion_models/core/v1/metadata.k.",
+                    "examples": {}
                 }
             },
             "required": [
+                "workloadType",
                 "name",
-                "age"
+                "containers"
             ],
             "line": 0,
-            "decorators": []
+            "decorators": [],
+            "filename": "schema.k",
+            "pkg_path": "__main__",
+            "description": "Server is the common user interface for long-running services adopting the best practice of Kubernetes.",
+            "examples": {}
         },
-        "Person": {
+        "Container": {
             "type": "schema",
             "union_types": [],
             "default": "",
-            "schema_name": "Person",
-            "schema_doc": "",
+            "schema_name": "Container",
+            "schema_doc": "Container is the common user interface for long-running services.",
             "properties": {
                 "name": {
                     "type": "str",
@@ -53,26 +159,22 @@
                     "properties": {},
                     "required": [],
                     "line": 1,
-                    "decorators": []
-                },
-                "age": {
-                    "type": "int",
-                    "union_types": [],
-                    "default": "",
-                    "schema_name": "",
-                    "schema_doc": "",
-                    "properties": {},
-                    "required": [],
-                    "line": 2,
-                    "decorators": []
+                    "decorators": [],
+                    "filename": "",
+                    "pkg_path": "",
+                    "description": "The name of the long-running container.",
+                    "examples": {}
                 }
             },
             "required": [
-                "name",
-                "age"
+                "name"
             ],
             "line": 0,
-            "decorators": []
+            "decorators": [],
+            "filename": "schema.k",
+            "pkg_path": "__main__",
+            "description": "Container is the common user interface for long-running services.",
+            "examples": {}
         }
     }
 }

kclvm/runner/src/runner.rs

@@ -48,8 +48,6 @@ pub struct ExecProgramArgs {
     pub debug: i32,
     // yaml/json: sort keys
     pub sort_keys: bool,
-    // include schema type path in JSON/YAML result
-    pub include_schema_type_path: bool,
     // plugin_agent is the address of plugin.
     #[serde(skip)]
     pub plugin_agent: u64,

kclvm/runner/src/test_datas/exec_prog_args/default.json

@@ -1 +1 @@
-{"work_dir":null,"k_filename_list":[],"external_pkgs":[],"k_code_list":[],"args":[],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false,"include_schema_type_path":false}
+{"work_dir":null,"k_filename_list":[],"external_pkgs":[],"k_code_list":[],"args":[],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false}

kclvm/runner/src/test_datas/settings_file/settings.json

@@ -1 +1 @@
-{"work_dir":null,"k_filename_list":["../main.k","./before/base.k","./main.k","./sub/sub.k"],"external_pkgs":[],"k_code_list":[],"args":[{"name":"app-name","value":"\"kclvm\""},{"name":"image","value":"\"kclvm:v0.0.1\""}],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false,"include_schema_type_path":false}
+{"work_dir":null,"k_filename_list":["../main.k","./before/base.k","./main.k","./sub/sub.k"],"external_pkgs":[],"k_code_list":[],"args":[{"name":"app-name","value":"\"kclvm\""},{"name":"image","value":"\"kclvm:v0.0.1\""}],"overrides":[],"disable_yaml_result":false,"print_override_ast":false,"strict_range_check":false,"disable_none":false,"verbose":0,"debug":0,"sort_keys":false}