[BACKPORT 2.14] [#13176] DocDB: Generate gFlags metadata xml at build…

… time

Summary:
Generate the gFlags xml metadata file for yb-master and yb-tserver at build time and include them in the release package.
gFlags are defined with a hard coded value in one file which then get linked to multiple applications. The only way for one of the application to have a custom default value is to update the flag at runtime at process startup before calling google::ParseCommandLineNonHelpFlags. Ex: yb-master sets evict_failed_followers to false.
New flag dump_flags_xml will dump the flag description in xml format with the current runtime value of the flags displayed as default.
This is called at build time for both yb-master and yb-tserver and the output xml files are made part of the tar package.

Fixes #13176

Original commit: 2a7ffce / D18153

Test Plan:
./yb_build.sh on m1 Mac and gcp Dev Server
All Detective builds

Reviewers: rahuldesirazu, mbautin, slingam

Reviewed By: slingam

Subscribers: bogdan, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D18212

build-support/gen_flags_metadata_wrapper

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) YugaByte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+#
+set -euo pipefail
+. "${BASH_SOURCE%/*}/common-build-env.sh"
+
+activate_virtualenv
+set_pythonpath
+"$YB_SRC_ROOT"/python/yb/gen_flags_metadata.py "$@"

python/yb/gen_flags_metadata.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# Copyright (c) YugaByte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+A wrapper to generate gFlags metadata for yb-master and yb-tserver.
+"""
+
+import sys
+import os
+import logging
+import time
+
+from yugabyte_pycommon import init_logging, run_program, WorkDirContext  # type: ignore
+
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: {} <program_name> <output_file_path>".format(sys.argv[0]))
+        sys.exit(1)
+    start_time_sec = time.time()
+    build_root = os.environ['YB_BUILD_ROOT']
+    program_name = sys.argv[1]
+    flags_metadata_file = sys.argv[2]
+
+    with WorkDirContext(build_root):
+        content = run_program(
+                    [
+                        os.path.join(build_root, 'bin', program_name),
+                        "--dump_flags_xml",
+                    ],
+                    shell=True
+                )
+
+        with open(flags_metadata_file, 'w+', encoding='utf-8') as f:
+            f.write(content.stdout)
+
+    elapsed_time_sec = time.time() - start_time_sec
+    logging.info("Generated flags_metadata for %s in %.1f sec", program_name, elapsed_time_sec)
+
+
+if __name__ == '__main__':
+    init_logging()
+    main()

python/yb/lto.py

@@ -126,10 +126,11 @@ def process_original_link_cmd(original_link_cmd: List[str]) -> List[str]:
     assert original_link_cmd[0] == ':'
     assert original_link_cmd[1] == '&&'
     assert original_link_cmd[2].endswith('/compiler-wrappers/c++')
-    assert original_link_cmd[-2] == '&&'
-    assert original_link_cmd[-1] == ':'
+    end_index = original_link_cmd[2:].index('&&') + 2
+    assert end_index > 3
+    assert end_index < len(original_link_cmd)
 
-    return original_link_cmd[3:-2]
+    return original_link_cmd[3:end_index]
 
 
 def is_yb_library(rel_path: str) -> bool:

src/yb/master/CMakeLists.txt

@@ -216,3 +216,9 @@ add_executable(yb-master master_main.cc)
 target_link_libraries(yb-master
   master
   ${YB_BASE_LIBS})
+
+# Generate master_flags.xml
+set(MASTER_FLAGS_METADATA_FILE ${YB_BUILD_ROOT}/master_flags.xml)
+add_custom_command(TARGET yb-master POST_BUILD
+  COMMAND "${BUILD_SUPPORT_DIR}/gen_flags_metadata_wrapper" "yb-master" "${MASTER_FLAGS_METADATA_FILE}"
+  BYPRODUCTS "${MASTER_FLAGS_METADATA_FILE}")

src/yb/tserver/CMakeLists.txt

@@ -255,6 +255,14 @@ target_link_libraries(yb-tserver
   master
   ${YB_BASE_LIBS})
 
+#########################################
+# Generate tserver_flags.xml
+#########################################
+set(TSERVER_FLAGS_METADATA_FILE ${YB_BUILD_ROOT}/tserver_flags.xml)
+add_custom_command(TARGET yb-tserver POST_BUILD
+  COMMAND "${BUILD_SUPPORT_DIR}/gen_flags_metadata_wrapper" "yb-tserver" "${TSERVER_FLAGS_METADATA_FILE}"
+  BYPRODUCTS "${TSERVER_FLAGS_METADATA_FILE}")
+
 #########################################
 # tserver_test_util
 #########################################

src/yb/util/flags.cc

@@ -76,6 +76,16 @@ TAG_FLAG(heap_profile_path, advanced);
 DEFINE_int32(svc_queue_length_default, 50, "Default RPC queue length for a service");
 TAG_FLAG(svc_queue_length_default, advanced);
 
+// This provides a more accurate representation of default gFlag values for application like
+// yb-master which override the hard coded values at process startup time.
+DEFINE_bool(
+    dump_flags_xml, false,
+    "Dump a XLM document describing all of gFlags used in this binary. Differs from helpxml by "
+    "displaying the current runtime value as the default instead of the hard coded values from the "
+    "flag definitions. ");
+TAG_FLAG(dump_flags_xml, stable);
+TAG_FLAG(dump_flags_xml, advanced);
+
 // Tag a bunch of the flags that we inherit from glog/gflags.
 
 //------------------------------------------------------------
@@ -231,23 +241,31 @@ void AppendXMLTag(const char* tag, const string& txt, string* r) {
   strings::SubstituteAndAppend(r, "<$0>$1</$0>", tag, EscapeForHtmlToString(txt));
 }
 
-static string DescribeOneFlagInXML(const CommandLineFlagInfo& flag) {
+YB_STRONGLY_TYPED_BOOL(OnlyDisplayDefaultFlagValue);
+
+static string DescribeOneFlagInXML(
+    const CommandLineFlagInfo& flag, OnlyDisplayDefaultFlagValue only_display_default_values) {
   unordered_set<string> tags;
   GetFlagTags(flag.name, &tags);
 
   string r("<flag>");
   AppendXMLTag("file", flag.filename, &r);
   AppendXMLTag("name", flag.name, &r);
   AppendXMLTag("meaning", flag.description, &r);
-  AppendXMLTag("default", flag.default_value, &r);
-  AppendXMLTag("current", flag.current_value, &r);
+  if (only_display_default_values) {
+    // use the current value as the default
+    AppendXMLTag("default", flag.current_value, &r);
+  } else {
+    AppendXMLTag("default", flag.default_value, &r);
+    AppendXMLTag("current", flag.current_value, &r);
+  }
   AppendXMLTag("type", flag.type, &r);
   AppendXMLTag("tags", JoinStrings(tags, ","), &r);
   r += "</flag>";
   return r;
 }
 
-void DumpFlagsXML() {
+void DumpFlagsXML(OnlyDisplayDefaultFlagValue only_display_default_values) {
   vector<CommandLineFlagInfo> flags;
   GetAllFlags(&flags);
 
@@ -261,11 +279,11 @@ void DumpFlagsXML() {
       EscapeForHtmlToString(google::ProgramUsage())) << endl;
 
   for (const CommandLineFlagInfo& flag : flags) {
-    cout << DescribeOneFlagInXML(flag) << std::endl;
+    cout << DescribeOneFlagInXML(flag, only_display_default_values) << std::endl;
   }
 
   cout << "</AllFlags>" << endl;
-  exit(1);
+  exit(0);
 }
 
 void ShowVersionAndExit() {
@@ -279,7 +297,9 @@ int ParseCommandLineFlags(int* argc, char*** argv, bool remove_flags) {
   int ret = google::ParseCommandLineNonHelpFlags(argc, argv, remove_flags);
 
   if (FLAGS_helpxml) {
-    DumpFlagsXML();
+    DumpFlagsXML(OnlyDisplayDefaultFlagValue::kFalse);
+  } else if (FLAGS_dump_flags_xml) {
+    DumpFlagsXML(OnlyDisplayDefaultFlagValue::kTrue);
   } else if (FLAGS_dump_metrics_json) {
     std::stringstream s;
     JsonWriter w(&s, JsonWriter::PRETTY);

yb_release_manifest.json

@@ -3,7 +3,11 @@
     "%symlinks%": {
       "bin/ysqlsh": "../postgres/bin/ysqlsh"
     },
-    ".": ["$BUILD_ROOT/version_metadata.json"],
+    ".": [
+      "$BUILD_ROOT/master_flags.xml",
+      "$BUILD_ROOT/tserver_flags.xml",
+      "$BUILD_ROOT/version_metadata.json"
+    ],
     "bin": [
       "$BUILD_ROOT/bin/ldb",
       "$BUILD_ROOT/bin/log-dump",