From 87a2248114667e11ae9c1fdb4faae97e82518747 Mon Sep 17 00:00:00 2001
From: Ilya Verbin <iverbin@tarantool.org>
Date: Fri, 22 Apr 2022 15:46:56 +0300
Subject: [PATCH] console: check lua_yaml_encode error in
 lbox_console_format_yaml

This patch adds a missed check for `lua_yaml_encode` return value,
similar to the check in `console_dump_plain`. Now Lua error is raised
if YAML encoding failed for any reason (e.g. OOM or formatting error).

Before:
```
tarantool> box.error.new(box.error.ILLEGAL_PARAMS, '\x80')
~/test$ echo $?
0
~/test$
```

After:
```
tarantool> box.error.new(box.error.ILLEGAL_PARAMS, '\x80')
---
- error: 'console: an exception occurred when formatting the output:
    expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS'
...

tarantool>
```

Part of #6781
Part of #6934

NO_DOC=bugfix
NO_TEST=see later commits
NO_CHANGELOG=see later commits
---
 src/box/lua/console.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/box/lua/console.c b/src/box/lua/console.c
index d203629d1237..8c0c69043dfc 100644
--- a/src/box/lua/console.c
+++ b/src/box/lua/console.c
@@ -471,7 +471,17 @@ lbox_console_format_yaml(struct lua_State *L)
 	}
 	lua_replace(L, 1);
 	lua_settop(L, 1);
-	return lua_yaml_encode(L, serializer_yaml, NULL, NULL);
+	int ret = lua_yaml_encode(L, serializer_yaml, NULL, NULL);
+	if (ret == 2) {
+		/*
+		 * Nil and error object are pushed onto the stack.
+		 */
+		assert(lua_isnil(L, -2));
+		assert(lua_isstring(L, -1));
+		return luaL_error(L, lua_tostring(L, -1));
+	}
+	assert(ret == 1);
+	return ret;
 }
 
 /**