Add UTF-8 filename support on Windows

CMakeLists.txt

@@ -241,6 +241,9 @@ if(MSVC)
     # because `/W4;/WX` is returned as a single string.
     add_compile_options( /W4;$<$<BOOL:${KTX_WERROR}>:/WX> )
     add_compile_options( $<IF:$<CONFIG:Debug>,/Gz,/O2> )
+    # Enable UTF-8 support
+    add_compile_options( $<$<C_COMPILER_ID:MSVC>:/utf-8> )
+    add_compile_options( $<$<CXX_COMPILER_ID:MSVC>:/utf-8> )
 elseif(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU"
        OR ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
     add_compile_options( -Wall -Wextra $<$<BOOL:${KTX_WERROR}>:-Werror>)

tools/ktx/CMakeLists.txt

@@ -22,6 +22,7 @@ add_executable(ktxtools
         metrics_utils.h
         transcode_utils.cpp
         transcode_utils.h
+        platform_utils.h
         utility.h
         validate.cpp
         validate.h

tools/ktx/command.cpp

@@ -4,6 +4,7 @@
 
 
 #include "command.h"
+#include "platform_utils.h"
 #include "version.h"
 #include "ktx.h"
 #include "sbufstream.h"
@@ -13,10 +14,6 @@
 #include <fmt/ostream.h>
 #include <fmt/printf.h>
 
-#if defined(_WIN32) && defined(DEBUG)
-#include <windows.h> // For functions used by launchDebugger 
-#endif
-
 // -------------------------------------------------------------------------------------------------
 
 #define QUOTE(x) #x
@@ -111,7 +108,7 @@ InputStream::InputStream(const std::string& filepath, Reporter& report) :
         stdinBuffer << std::cin.rdbuf();
         activeStream = &stdinBuffer;
     } else {
-        file.open(filepath, std::ios::binary | std::ios::in);
+        file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::in);
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open input file \"{}\": {}.", filepath, errnoMessage());
         activeStream = &file;
@@ -134,7 +131,11 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
         #endif
         file = stdout;
     } else {
-        file = std::fopen(filepath.c_str(), "wb");
+#if defined(_WIN32)
+        file = _wfopen(DecodeUTF8Path(filepath).c_str(), L"wb");
+#else
+        file = std::fopen(DecodeUTF8Path(filepath).c_str(), "wb");
+#endif
         if (!file)
             report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}.", filepath, errnoMessage());
     }
@@ -152,7 +153,7 @@ OutputStream::OutputStream(const std::string& filepath, Reporter& report) :
     //     #endif
     //     activeStream = &std::cout;
     // } else {
-    //     file.open(filepath, std::ios::binary | std::ios::out);
+    //     file.open(DecodeUTF8Path(filepath).c_str(), std::ios::binary | std::ios::out);
     //     if (!file)
     //         report.fatal(rc::IO_FAILURE, "Could not open output file \"{}\": {}", filepath, errnoMessage());
     //     activeStream = &file;
@@ -174,7 +175,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) {
     const auto ret = ktxTexture_WriteToStdioStream(texture, file);
     if (KTX_SUCCESS != ret) {
         if (file != stdout)
-            std::filesystem::remove(filepath);
+            std::filesystem::remove(DecodeUTF8Path(filepath).c_str());
         report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": KTX error: {}.", filepath, ktxErrorString(ret));
     }
 
@@ -185,7 +186,7 @@ void OutputStream::writeKTX2(ktxTexture* texture, Reporter& report) {
     //
     // if (KTX_SUCCESS != ret) {
     //     if (activeStream != &std::cout)
-    //         std::filesystem::remove(filepath);
+    //         std::filesystem::remove(DecodeUTF8Path(filepath).c_str());
     //     report.fatal(rc::IO_FAILURE, "Failed to write KTX file \"{}\": {}.", fmtOutFile(filepath), ktxErrorString(ret));
     // }
 }

tools/ktx/command_create.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "metrics_utils.h"
 #include "compress_utils.h"
 #include "encode_utils.h"
@@ -1249,8 +1250,9 @@ void CommandCreate::executeCreate() {
     }
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/command_encode.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "metrics_utils.h"
 #include "compress_utils.h"
 #include "encode_utils.h"
@@ -219,8 +220,9 @@ void CommandEncode::executeEncode() {
     }
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/command_extract.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "format_descriptor.h"
 #include "formats.h"
 #include "fragment_uri.h"
@@ -386,13 +387,14 @@ void CommandExtract::executeExtract() {
             (!options.fragmentURI.facial.is_undefined() && options.fragmentURI.facial.is_multi()) ||
             ((options.globalAll || options.depthFlagUsed) && options.depth.is_multi());
     try {
+        const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputPath));
         if (isMultiOutput) {
-            if (std::filesystem::exists(options.outputPath) && !std::filesystem::is_directory(options.outputPath))
+            if (std::filesystem::exists(outputPath) && !std::filesystem::is_directory(outputPath))
                 fatal_usage("Specified output path must be a directory for multi-output extract: \"{}\".", options.outputPath);
-            std::filesystem::create_directories(options.outputPath);
+            std::filesystem::create_directories(outputPath);
         } else {
-            if (std::filesystem::path(options.outputPath).has_parent_path())
-                std::filesystem::create_directories(std::filesystem::path(options.outputPath).parent_path());
+            if (outputPath.has_parent_path())
+                std::filesystem::create_directories(outputPath.parent_path());
         }
     } catch (const std::filesystem::filesystem_error& e) {
         fatal(rc::IO_FAILURE, "Failed to create the output directory \"{}\": {}.", e.path1().generic_string(), e.what());

tools/ktx/command_help.cpp

@@ -10,7 +10,10 @@
 #include <fmt/format.h>
 
 #if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
 #include <windows.h> // For GetModuleFileNameW
+#include <shellapi.h> // For ShellExecuteW
 #include <pathcch.h> // For PathCchRemoveFileSpec
 #include <fmt/xchar.h> // For wchat_t format
 #endif
@@ -161,15 +164,15 @@ void CommandHelp::executeHelp() {
         fatal(rc::RUNTIME_ERROR, "Failed to open the html documentation: ERROR {}", r);
 
 #else
-#   if defined(__APPLE__)
+#if defined(__APPLE__)
     char buf[PATH_MAX];
     uint32_t bufsize = PATH_MAX;
     if (const auto ec = _NSGetExecutablePath(buf, &bufsize))
         fatal(rc::RUNTIME_ERROR, "Failed to determine executable path: ERROR {}", ec);
     const auto executablePath = std::filesystem::canonical(buf);
-#   else // Linux
+#else // Linux
     const auto executablePath = std::filesystem::canonical("/proc/self/exe");
-#   endif
+#endif
 
     const auto executableDir = std::filesystem::path(executablePath).remove_filename();
     const auto manFile = fmt::format("{}/../share/man/man1/ktx{}{}.1",

tools/ktx/command_transcode.cpp

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "command.h"
+#include "platform_utils.h"
 #include "compress_utils.h"
 #include "transcode_utils.h"
 #include "formats.h"
@@ -171,8 +172,9 @@ void CommandTranscode::executeTranscode() {
             writer.c_str());
 
     // Save output file
-    if (std::filesystem::path(options.outputFilepath).has_parent_path())
-        std::filesystem::create_directories(std::filesystem::path(options.outputFilepath).parent_path());
+    const auto outputPath = std::filesystem::path(DecodeUTF8Path(options.outputFilepath));
+    if (outputPath.has_parent_path())
+        std::filesystem::create_directories(outputPath.parent_path());
 
     OutputStream outputFile(options.outputFilepath, *this);
     outputFile.writeKTX2(texture, *this);

tools/ktx/ktx_main.cpp

@@ -7,12 +7,20 @@
 #include "stdafx.h"
 #include <iostream>
 #include <string>
+#include <memory>
 #include <unordered_map>
 
 #include <cxxopts.hpp>
 #include <fmt/ostream.h>
 #include <fmt/printf.h>
 
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+#include <shellapi.h>
+#endif
+
 // -------------------------------------------------------------------------------------------------
 
 namespace ktx {
@@ -185,6 +193,24 @@ int _tmain(int argc, _TCHAR* argv[]) {
     //      // pbxproj file, so it can't be disabled in a generated project.
     //      // Remove these from the arguments under consideration.
 
+#if defined(_WIN32)
+    // Windows does not support UTF-8 argv so we have to manually acquire it
+    LPWSTR commandLine = GetCommandLineW();
+    LPWSTR* wideArgv = CommandLineToArgvW(commandLine, &argc);
+    std::vector<std::unique_ptr<_TCHAR[]>> utf8Argv(argc);
+    for (int i = 0; i < argc; ++i) {
+        int byteSize = WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, nullptr, 0, nullptr, nullptr);
+        utf8Argv[i] = std::make_unique<_TCHAR[]>(byteSize);
+        WideCharToMultiByte(CP_UTF8, 0, wideArgv[i], -1, utf8Argv[i].get(), byteSize, nullptr, nullptr);
+        argv[i] = utf8Argv[i].get();
+    }
+
+    // Set UTF-8 codepage for the console
+    if (!SetConsoleOutputCP(CP_UTF8)) {
+        fmt::print(std::cerr, "{} warning: failed to set UTF-8 code page for console output.\n", argv[0]);
+    }
+#endif
+
     if (argc >= 2) {
         // Has a subcommand, attempt to lookup
 

tools/ktx/platform_utils.h

@@ -0,0 +1,30 @@
+// Copyright 2022-2023 The Khronos Group Inc.
+// Copyright 2022-2023 RasterGrid Kft.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <string>
+
+#if defined(_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+// For Windows, we convert the UTF-8 path to a UTF-16 path to force using the APIs
+// that correctly handle unicode characters
+static std::wstring DecodeUTF8Path(std::string path) {
+    std::wstring result;
+    int len = MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast<int>(path.length()), NULL, 0);
+    if (len > 0)
+    {
+        result.resize(len);
+        MultiByteToWideChar(CP_UTF8, 0, path.c_str(), static_cast<int>(path.length()), &result[0], len);
+    }
+    return result;
+}
+#else
+// For other platforms there is no need for any conversion, they support UTF-8 natively
+static std::string DecodeUTF8Path(std::string path) {
+    return path;
+}
+#endif