console: check lua_yaml_encode error in lbox_console_format_yaml

This patch adds a missed check for `lua_yaml_encode` return value,
similar to the check in `console_dump_plain`. Now Lua error is raised
if YAML encoding failed for any reason (e.g. OOM or formatting error).

Before:
```
tarantool> box.error.new(box.error.ILLEGAL_PARAMS, '\x80')
~/test$ echo $?
0
~/test$
```

After:
```
tarantool> box.error.new(box.error.ILLEGAL_PARAMS, '\x80')
---
- error: 'console: an exception occurred when formatting the output:
    expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS'
...

tarantool>
```

Part of tarantool#6781
Part of tarantool#6934

NO_DOC=bugfix
NO_TEST=see later commits
NO_CHANGELOG=see later commits

src/box/lua/console.c

@@ -471,7 +471,17 @@ lbox_console_format_yaml(struct lua_State *L)
 	}
 	lua_replace(L, 1);
 	lua_settop(L, 1);
-	return lua_yaml_encode(L, serializer_yaml, NULL, NULL);
+	int ret = lua_yaml_encode(L, serializer_yaml, NULL, NULL);
+	if (ret == 2) {
+		/*
+		 * Nil and error object are pushed onto the stack.
+		 */
+		assert(lua_isnil(L, -2));
+		assert(lua_isstring(L, -1));
+		return luaL_error(L, lua_tostring(L, -1));
+	}
+	assert(ret == 1);
+	return ret;
 }
 
 /**