lua-yaml: encode malformed error messages in base64

If MP_ERROR contains an error message with invalid UTF-8 sequences,
encode it in Base64 to obtain printable string.

Closes tarantool#6781
Closes tarantool#6934

NO_DOC=bugfix

changelogs/unreleased/gh-6934-nonprintable-error-message.md

@@ -0,0 +1,4 @@
+## bugfix/core
+
+* Fixed tarantool terminations on error messages with invalid UTF-8
+  sequences (gh-6781 and gh-6934).

test/box-luatest/gh_6934_nonprintable_error_message_test.lua

@@ -0,0 +1,17 @@
+local console = require('console')
+local t = require('luatest')
+local g = t.group()
+
+-- Check that an error message with invalid UTF-8 sequences is encoded to Base64
+
+g.test_bad_utf8_in_error_msg1 = function()
+    local res = console.eval("box.error.new(box.error.ILLEGAL_PARAMS, 'bad: \x80')")
+    local ref = "---\n- !!binary SWxsZWdhbCBwYXJhbWV0ZXJzLCBiYWQ6IIA=\n...\n"
+    t.assert_equals(res, ref)
+end
+
+g.test_bad_utf8_in_error_msg2 = function()
+    local res = console.eval("require('net.box').self:call('bad: \x8a')")
+    local ref = "---\n- error: !!binary UHJvY2VkdXJlICdiYWQ6IIonIGlzIG5vdCBkZWZpbmVk\n...\n"
+    t.assert_equals(res, ref)
+end

third_party/lua-yaml/lyaml.cc

@@ -711,6 +711,14 @@ static int dump_node(struct lua_yaml_dumper *dumper)
       case MP_ERROR:
          str = field.errorval->errmsg;
          len = strlen(str);
+         if (!utf8_check_printable(str, len)) {
+            is_binary = 1;
+            lua_pop(dumper->L, 1);
+            lua_pushlstring(dumper->L, str, len);
+            tobase64(dumper->L, -1);
+            str = lua_tolstring(dumper->L, -1, &len);
+            tag = (yaml_char_t *) LUAYAML_TAG_PREFIX "binary";
+         }
          break;
       case MP_DATETIME:
          len = datetime_to_string(field.dateval, buf, sizeof(buf));