diff --git a/CMakeLists.txt b/CMakeLists.txt index f926560683..5a107402f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,6 +241,9 @@ if(MSVC) # because `/W4;/WX` is returned as a single string. add_compile_options( /W4;$<$:/WX> ) add_compile_options( $,/Gz,/O2> ) + # Enable UTF-8 support + add_compile_options( $<$:/utf-8> ) + add_compile_options( $<$:/utf-8> ) elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") add_compile_options( -Wall -Wextra $<$:-Werror>) diff --git a/tests/cts b/tests/cts index 97819bf441..f58caac519 160000 --- a/tests/cts +++ b/tests/cts @@ -1 +1 @@ -Subproject commit 97819bf441b370e22e5b3fcba210754fb88a6aeb +Subproject commit f58caac519ded6c3f4efba44dcb0863a669734e6 diff --git a/tools/ktx/CMakeLists.txt b/tools/ktx/CMakeLists.txt index b24a86587c..e0300ae31b 100644 --- a/tools/ktx/CMakeLists.txt +++ b/tools/ktx/CMakeLists.txt @@ -22,6 +22,7 @@ add_executable(ktxtools metrics_utils.h transcode_utils.cpp transcode_utils.h + platform_utils.h utility.h validate.cpp validate.h diff --git a/tools/ktx/command.cpp b/tools/ktx/command.cpp index 0c3324176a..01a118bdce 100644 --- a/tools/ktx/command.cpp +++ b/tools/ktx/command.cpp @@ -4,19 +4,17 @@ #include "command.h" +#include "platform_utils.h" #include "version.h" #include "ktx.h" #include "sbufstream.h" +#include #include #include #include #include -#if defined(_WIN32) && defined(DEBUG) -#include // For functions used by launchDebugger -#endif - // ------------------------------------------------------------------------------------------------- #define QUOTE(x) #x @@ -111,7 +109,7 @@ InputStream::InputStream(const std::string& filepath, Reporter& report) : stdinBuffer << std::cin.rdbuf(); activeStream = &stdinBuffer; } else { - file.open(filepath, std::ios::binary | std::ios::in); + file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in); if (!file) report.fatal(rc::IO_FAILURE, "Could not open input file \"{}\": {}.", filepath, errnoMessage()); activeStream = &file; @@ -134,7 +132,11 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : #endif file = stdout; } else { - file = std::fopen(filepath.c_str(), "wb"); +#if defined(_WIN32) + file = _wfopen(DecodeUTF8Path(filepath).c_str(), L"wb"); +#else + file = fopen(DecodeUTF8Path(filepath).c_str(), "wb"); +#endif if (!file) report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}.", filepath, errnoMessage()); } @@ -152,7 +154,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : // #endif // activeStream = &std::cout; // } else { - // file.open(filepath, std::ios::binary | std::ios::out); + // file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::out); // if (!file) // report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}", filepath, errnoMessage()); // activeStream = &file; @@ -161,7 +163,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) : OutputStream::~OutputStream() { if (file != stdout) - std::fclose(file); + fclose(file); } void OutputStream::write(const char* data, std::size_t size, Reporter& report) { @@ -174,7 +176,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) { const auto ret = ktxTexture_WriteToStdioStream(texture, file); if (KTX_SUCCESS != ret) { if (file != stdout) - std::filesystem::remove(filepath); + std::filesystem::remove(DecodeUTF8Path(filepath).c_str()); report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": KTX error: {}.", filepath, ktxErrorString(ret)); } @@ -185,7 +187,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) { // // if (KTX_SUCCESS != ret) { // if (activeStream != &std::cout) - // std::filesystem::remove(filepath); + // std::filesystem::remove(DecodeUTF8Path(filepath).c_str()); // report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": {}.", fmtOutFile(filepath), ktxErrorString(ret)); // } } diff --git a/tools/ktx/command_create.cpp b/tools/ktx/command_create.cpp index 96d0bbf88c..aa5506ba0f 100644 --- a/tools/ktx/command_create.cpp +++ b/tools/ktx/command_create.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "metrics_utils.h" #include "compress_utils.h" #include "encode_utils.h" @@ -1249,8 +1250,9 @@ void CommandCreate::executeCreate() { } // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/command_encode.cpp b/tools/ktx/command_encode.cpp index da9fc7f4e7..dd3b5162c8 100644 --- a/tools/ktx/command_encode.cpp +++ b/tools/ktx/command_encode.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "metrics_utils.h" #include "compress_utils.h" #include "encode_utils.h" @@ -219,8 +220,9 @@ void CommandEncode::executeEncode() { } // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/command_extract.cpp b/tools/ktx/command_extract.cpp index 1efabf4cdd..4787d97c02 100644 --- a/tools/ktx/command_extract.cpp +++ b/tools/ktx/command_extract.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "format_descriptor.h" #include "formats.h" #include "fragment_uri.h" @@ -386,13 +387,14 @@ void CommandExtract::executeExtract() { (!options.fragmentURI.facial.is_undefined() && options.fragmentURI.facial.is_multi()) || ((options.globalAll || options.depthFlagUsed) && options.depth.is_multi()); try { + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputPath)); if (isMultiOutput) { - if (std::filesystem::exists(options.outputPath) && !std::filesystem::is_directory(options.outputPath)) + if (std::filesystem::exists(outputPath) && !std::filesystem::is_directory(outputPath)) fatal_usage("Specified output path must be a directory for multi-output extract: \"{}\".", options.outputPath); - std::filesystem::create_directories(options.outputPath); + std::filesystem::create_directories(outputPath); } else { - if (std::filesystem::path(options.outputPath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputPath).parent_path()); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); } } catch (const std::filesystem::filesystem_error& e) { fatal(rc::IO_FAILURE, "Failed to create the output directory \"{}\": {}.", e.path1().generic_string(), e.what()); diff --git a/tools/ktx/command_help.cpp b/tools/ktx/command_help.cpp index b19b3f82ac..0e335ab9af 100644 --- a/tools/ktx/command_help.cpp +++ b/tools/ktx/command_help.cpp @@ -10,7 +10,14 @@ #include #if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif #include // For GetModuleFileNameW +#include // For ShellExecuteW #include // For PathCchRemoveFileSpec #include // For wchat_t format #endif @@ -161,15 +168,15 @@ void CommandHelp::executeHelp() { fatal(rc::RUNTIME_ERROR, "Failed to open the html documentation: ERROR {}", r); #else -# if defined(__APPLE__) +#if defined(__APPLE__) char buf[PATH_MAX]; uint32_t bufsize = PATH_MAX; if (const auto ec = _NSGetExecutablePath(buf, &bufsize)) fatal(rc::RUNTIME_ERROR, "Failed to determine executable path: ERROR {}", ec); const auto executablePath = std::filesystem::canonical(buf); -# else // Linux +#else // Linux const auto executablePath = std::filesystem::canonical("/proc/self/exe"); -# endif +#endif const auto executableDir = std::filesystem::path(executablePath).remove_filename(); const auto manFile = fmt::format("{}/../share/man/man1/ktx{}{}.1", diff --git a/tools/ktx/command_transcode.cpp b/tools/ktx/command_transcode.cpp index f8a4330314..05b9e31bab 100644 --- a/tools/ktx/command_transcode.cpp +++ b/tools/ktx/command_transcode.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "command.h" +#include "platform_utils.h" #include "compress_utils.h" #include "transcode_utils.h" #include "formats.h" @@ -171,8 +172,9 @@ void CommandTranscode::executeTranscode() { writer.c_str()); // Save output file - if (std::filesystem::path(options.outputFilepath).has_parent_path()) - std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path()); + const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath)); + if (outputPath.has_parent_path()) + std::filesystem::create_directories(outputPath.parent_path()); OutputStream outputFile(options.outputFilepath, *this); outputFile.writeKTX2(texture, *this); diff --git a/tools/ktx/ktx_main.cpp b/tools/ktx/ktx_main.cpp index 95e624e029..24e5125158 100644 --- a/tools/ktx/ktx_main.cpp +++ b/tools/ktx/ktx_main.cpp @@ -7,12 +7,24 @@ #include "stdafx.h" #include #include +#include #include #include #include #include +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#endif + // ------------------------------------------------------------------------------------------------- namespace ktx { @@ -185,6 +197,24 @@ int _tmain(int argc, _TCHAR* argv[]) { // // pbxproj file, so it can't be disabled in a generated project. // // Remove these from the arguments under consideration. +#if defined(_WIN32) + // Windows does not support UTF-8 argv so we have to manually acquire it + LPWSTR commandLine = GetCommandLineW(); + LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc); + std::vector> utf8Argv(argc); + for (int i = 0; i < argc; ++i) { + int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr); + utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize); + WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr); + argv[i] = utf8Argv[i].get(); + } + + // Set UTF-8 codepage for the console + if (!SetConsoleOutputCP(CP_UTF8)) { + fmt::print(std::cerr, "{} warning: failed to set UTF-8 code page for console output.\n", argv[0]); + } +#endif + if (argc >= 2) { // Has a subcommand, attempt to lookup diff --git a/tools/ktx/platform_utils.h b/tools/ktx/platform_utils.h new file mode 100644 index 0000000000..d2b208d98e --- /dev/null +++ b/tools/ktx/platform_utils.h @@ -0,0 +1,34 @@ +// Copyright 2022-2023 The Khronos Group Inc. +// Copyright 2022-2023 RasterGrid Kft. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +// For Windows, we convert the UTF-8 path to a UTF-16 path to force using the APIs +// that correctly handle unicode characters +static std::wstring DecodeUTF8Path(std::string path) { + std::wstring result; + int len = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), NULL, 0); + if (len > 0) + { + result.resize(len); + MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast(path.length()), &result[0], len); + } + return result; +} +#else +// For other platforms there is no need for any conversion, they support UTF-8 natively +static std::string DecodeUTF8Path(std::string path) { + return path; +} +#endif