Add: better long-paths support on Windows

daemon/extension/ManifestFile.cpp

@@ -19,28 +19,34 @@
 
 #include "nzbget.h"
 
-#include <fstream>
 #include "ManifestFile.h"
 #include "Json.h"
 #include "FileSystem.h"
+#include "Log.h"
 
 namespace ManifestFile
 {
-	const char* MANIFEST_FILE = "manifest.json";
-	const char* DEFAULT_SECTION_NAME = "options";
-
 	bool Load(Manifest& manifest, const char* directory)
 	{
-		BString<1024> path("%s%c%s", directory, PATH_SEPARATOR, MANIFEST_FILE);
-		std::ifstream fs(path);
-		if (!fs.is_open())
-			return false;
+		std::string path = std::string(directory) + PATH_SEPARATOR + MANIFEST_FILE;
+		DiskFile file;
+		if (!file.Open(path.c_str(), DiskFile::omRead)) return false;
 
-		Json::ErrorCode ec;
-		Json::JsonValue jsonValue = Json::Deserialize(fs, ec);
-		if (ec)
+		std::string jsonStr;
+		char buffer[BUFFER_SIZE];
+		while (file.ReadLine(buffer, BUFFER_SIZE))
+		{
+			jsonStr += buffer;
+		}
+
+		auto desRes = Json::Deserialize(jsonStr);
+		if (!desRes.has_value())
+		{
+			error("Failed to parse %s. Syntax error.", path.c_str());
 			return false;
+		}
 
+		Json::JsonValue jsonValue = std::move(desRes.value());
 		Json::JsonObject json = jsonValue.as_object();
 
 		if (!ValidateAndSet(json, manifest))

daemon/extension/ManifestFile.h

@@ -30,8 +30,9 @@ namespace ManifestFile
 {
 	using SelectOption = std::variant<double, std::string>;
 
-	extern const char* MANIFEST_FILE;
-	extern const char* DEFAULT_SECTION_NAME;
+	inline constexpr size_t BUFFER_SIZE = 4096;
+	inline const char* MANIFEST_FILE = "manifest.json";
+	inline const char* DEFAULT_SECTION_NAME = "options";
 
 	struct Section
 	{

daemon/main/nzbget.h

@@ -56,7 +56,7 @@
 #pragma warning(disable:4800) // 'type' : forcing value to bool 'true' or 'false' (performance warning)
 #pragma warning(disable:4267) // 'var' : conversion from 'size_t' to 'type', possible loss of data
 
-#define popen _popen
+#define popen _wpopen
 #define pclose _pclose
 
 #endif

daemon/postprocess/ParChecker.cpp

@@ -112,26 +112,27 @@ Par2::Result Repairer::PreProcess(const char *parFilename)
 {
 	std::string memParam = "-m" + std::to_string(m_memToUse);
 	std::string threadsParam = "-t" + std::to_string(m_threadsToUse);
+	std::string parFilePath = FileSystem::GetRealPath(parFilename).value_or(parFilename);
 
 	if (g_Options->GetParScan() == Options::psFull)
 	{
-		BString<1024> wildcardParam(parFilename, 1024);
+		BString<1024> wildcardParam(parFilePath.c_str(), 1024);
 		char* basename = FileSystem::BaseFileName(wildcardParam);
 		if (basename != wildcardParam && strlen(basename) > 0)
 		{
 			basename[0] = '*';
 			basename[1] = '\0';
 		}
 
-		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename, wildcardParam };
+		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilePath.c_str(), wildcardParam };
 		if (!m_commandLine.Parse(7, (char**)argv))
 		{
 			return Par2::eInvalidCommandLineArguments;
 		}
 	}
 	else
 	{
-		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename };
+		const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilePath.c_str() };
 		if (!m_commandLine.Parse(6, (char**)argv))
 		{
 			return Par2::eInvalidCommandLineArguments;

daemon/system/SystemInfo.cpp

@@ -195,7 +195,7 @@ namespace System
 
 		std::string path = FileSystem::ExtractFilePathFromCmd(cmd);
 
-		auto result = FileSystem::GetFileRealPath(path);
+		auto result = FileSystem::GetRealPath(path);
 
 		if (result.has_value() && FileSystem::FileExists(result.value().c_str()))
 		{

daemon/util/FileSystem.cpp

@@ -24,6 +24,10 @@
 #include "Util.h"
 #include "Log.h"
 
+#ifdef WIN32
+#include "utf8.h"
+#endif
+
 const char* RESERVED_DEVICE_NAMES[] = { "CON", "PRN", "AUX", "NUL",
 	"COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
 	"LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", NULL };
@@ -56,21 +60,39 @@ void FileSystem::NormalizePathSeparators(char* path)
 	}
 }
 
-std::optional<std::string> FileSystem::GetFileRealPath(const std::string& path)
+std::optional<std::string> FileSystem::GetRealPath(const std::string& path) noexcept
 {
+	if (path.empty()) return std::nullopt;
+
 #ifdef WIN32
-	char buffer[MAX_PATH];
-	DWORD len = GetFullPathName(path.c_str(), MAX_PATH, buffer, nullptr);
-	if (len != 0)
+	auto res = Utf8::Utf8ToWide(path);
+	if (!res.has_value()) return std::nullopt;
+
+	std::wstring wpath = std::move(res.value());
+
+	if (wpath.size() > MAX_DIR_PATH && std::wcsncmp(wpath.c_str(), L"\\\\", 2) == 0)
 	{
-		return std::optional{ buffer };
+		wpath = L"\\\\?\\UNC" + wpath;
+	}
+	else if (wpath.size() > MAX_DIR_PATH)
+	{
+		wpath = L"\\\\?\\" + wpath;
+	}
+
+	DWORD len = GetFullPathNameW(wpath.c_str(), 0, nullptr, nullptr);
+	if (len)
+	{
+		std::wstring buffer(len, '\0');
+		GetFullPathNameW(wpath.c_str(), len, &buffer[0], nullptr);
+
+		return Utf8::WideToUtf8(buffer);
 	}
 #else
 	if (char* realPath = realpath(path.c_str(), nullptr))
 	{
 		std::string res = realPath;
 		free(realPath);
-		return std::optional(std::move(res));
+		return res;
 	}
 #endif
 
@@ -621,7 +643,7 @@ std::string FileSystem::ExtractFilePathFromCmd(const std::string& path)
 {
 	if (path.empty())
 	{
-		return std::string(path);
+		return path;
 	}
 
 	size_t lastSeparatorPos = path.find_last_of(PATH_SEPARATOR);

daemon/util/FileSystem.h

@@ -27,6 +27,10 @@
 #include "NString.h"
 #include "Options.h"
 
+#ifdef WIN32
+inline constexpr size_t MAX_DIR_PATH = 248;
+#endif
+
 class FileSystem
 {
 	struct DiskState
@@ -40,7 +44,7 @@ class FileSystem
 	static char* BaseFileName(const char* filename);
 	static bool SameFilename(const char* filename1, const char* filename2);
 	static void NormalizePathSeparators(char* path);
-	static std::optional<std::string> GetFileRealPath(const std::string& path);
+	static std::optional<std::string> GetRealPath(const std::string& path) noexcept;
 	static bool LoadFileIntoBuffer(const char* filename, CharBuffer& buffer, bool addTrailingNull);
 	static bool SaveBufferIntoFile(const char* filename, const char* buffer, int bufLen);
 	static bool AllocateFile(const char* filename, int64 size, bool sparse, CString& errmsg);

daemon/util/Json.cpp

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <https://nzbget.com>.
  *
- *  Copyright (C) 2023 Denis <[email protected]>
+ *  Copyright (C) 2023-2024 Denis <[email protected]>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -21,32 +21,37 @@
 
 #include "Json.h"
 
-namespace Json {
-	JsonValue Deserialize(std::istream& is, ErrorCode& ec)
+namespace Json
+{
+	std::optional<JsonValue> Deserialize(std::basic_istream<char>& is) noexcept
 	{
+		ErrorCode ec;
 		StreamParser parser;
 		std::string line;
-
 		while (std::getline(is, line))
 		{
 			parser.write(line, ec);
-			if (ec)
-			{
-				return nullptr;
-			}
+			if (ec) return std::nullopt;
 		}
 
 		parser.finish(ec);
 
-		if (ec)
-		{
-			return nullptr;
-		}
+		if (ec) return std::nullopt;
 
 		return parser.release();
 	}
 
-	std::string Serialize(const JsonObject& json)
+	std::optional<JsonValue> Deserialize(const std::string& jsonStr) noexcept
+	{
+		ErrorCode ec;
+		JsonValue value = parse(jsonStr, ec);
+
+		if (ec) return std::nullopt;
+
+		return value;
+	}
+
+	std::string Serialize(const JsonObject& json) noexcept
 	{
 		return serialize(json);
 	}

daemon/util/Json.h

@@ -1,7 +1,7 @@
 /*
  *  This file is part of nzbget. See <https://nzbget.com>.
  *
- *  Copyright (C) 2023 Denis <[email protected]>
+ *  Copyright (C) 2023-2024 Denis <[email protected]>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -22,6 +22,8 @@
 
 #include <boost/json.hpp>
 #include <iostream>
+#include <optional>
+#include <string>
 
 namespace Json
 {
@@ -32,8 +34,9 @@ namespace Json
 	using StreamParser = boost::json::stream_parser;
 	using ErrorCode = boost::system::error_code;
 
-	JsonValue Deserialize(std::istream& is, ErrorCode& ec);
-	std::string Serialize(const JsonObject& json);
+	std::optional<JsonValue> Deserialize(std::basic_istream<char>& is) noexcept;
+	std::optional<JsonValue> Deserialize(const std::string& jsonStr) noexcept;
+	std::string Serialize(const JsonObject& json) noexcept;
 }
 
 #endif

daemon/util/Utf8.cpp

@@ -22,9 +22,7 @@
 #include "nzbget.h"
 
 #include <windows.h>
-
 #include "Utf8.h"
-#include "Log.h"
 
 namespace Utf8
 {
@@ -35,12 +33,17 @@ namespace Utf8
 		int requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, nullptr, 0);
 		if (requiredSize <= 0) return std::nullopt;
 
-		std::wstring wstr(requiredSize, '\0');
+		wchar_t* buffer = new (std::nothrow) wchar_t[requiredSize];
+		if (!buffer) return std::nullopt;
 
-		requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, wstr.data(), requiredSize);
-		if (requiredSize <= 0) return std::nullopt;
+		requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, buffer, requiredSize);
+		if (requiredSize <= 0)
+		{
+			delete[] buffer;
+			return std::nullopt;
+		}
 
-		return wstr;
+		return std::wstring(buffer);
 	}
 
 	std::optional<std::string> WideToUtf8(const std::wstring& wstr) noexcept
@@ -50,11 +53,16 @@ namespace Utf8
 		int requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, nullptr, 0, nullptr, nullptr);
 		if (requiredSize <= 0) return std::nullopt;
 
-		std::string str(requiredSize, '\0');
+		char* buffer = new (std::nothrow) char[requiredSize];
+		if (!buffer) return std::nullopt;
 
-		requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, str.data(), requiredSize, nullptr, nullptr);
-		if (requiredSize <= 0) return std::nullopt;
+		requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, buffer, requiredSize, nullptr, nullptr);
+		if (requiredSize <= 0)
+		{
+			delete[] buffer;
+			return std::nullopt;
+		}
 
-		return str;
+		return std::string(buffer);
 	}
 }

daemon/util/Util.cpp

@@ -27,6 +27,8 @@
 #include "YEncode.h"
 
 #ifdef WIN32
+#include "utf8.h"
+
 // getopt for WIN32:
 // from http://www.codeproject.com/cpp/xgetopt.asp
 // Original Author:  Hans Dietrich ([email protected])
@@ -707,7 +709,15 @@ uint32 Util::HashBJ96(const char* buffer, int bufSize, uint32 initValue)
 
 std::unique_ptr<FILE, std::function<void(FILE*)>> Util::MakePipe(const std::string& cmd)
 {
+#ifdef WIN32
+	auto res = Utf8::Utf8ToWide(cmd);
+	if (!res.has_value()) return nullptr;
+
+	FILE* pipe = popen(res.value().c_str(), L"r");
+#else
 	FILE* pipe = popen(cmd.c_str(), "r");
+#endif
+
 	return std::unique_ptr<FILE, std::function<void(FILE*)>>(pipe, [](FILE* pipe)
 		{
 			if (pipe)

tests/extension/CMakeLists.txt

@@ -1,4 +1,4 @@
-file(GLOB ExtensionSrc
+set(ExtensionSrc
 	main.cpp
 	ManifestFileTest.cpp
 	ExtensionLoaderTest.cpp
@@ -31,6 +31,10 @@ file(GLOB ExtensionSrc
 	${CMAKE_SOURCE_DIR}/daemon/connect/TlsSocket.cpp
 )
 
+if(WIN32)
+	set(ExtensionSrc ${ExtensionSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp)
+endif()
+
 add_executable(ExtensionTests ${ExtensionSrc})
 target_link_libraries(ExtensionTests PRIVATE ${LIBS})
 target_include_directories(ExtensionTests PRIVATE ${INCLUDES})

tests/feed/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_executable(FeedTests
+set(FeedTestsSrc
 	FeedFilterTest.cpp
 	FeedFileTest.cpp
 	main.cpp
@@ -14,6 +14,13 @@ add_executable(FeedTests
 	${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp
 	${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp
 )
+
+if(WIN32)
+	set(FeedTestsSrc ${FeedTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp)
+endif()
+
+add_executable(FeedTests ${FeedTestsSrc})
+
 target_link_libraries(FeedTests PRIVATE ${LIBS})
 target_include_directories(FeedTests PRIVATE ${INCLUDES})