Prepare Vic2 to HoI4 release (#657)

* Fix text transformations(#608) (#653)

* Add unit tests for Vic2::Agreement

* Move military templates into their own class

* Add tests for the military mappings

* Fix merge error

* Add capability to translate from Win1252 to UTF-8

* Add capability to convert into win1252

* Convert Vic2 localisations from WIn1252 instead of ISO 8859-15.

* Linux fixes (#656)

* Add unit tests for Vic2::Agreement

* Move military templates into their own class

* Add tests for the military mappings

* Fix merge error

* Add capability to translate from Win1252 to UTF-8

* Add capability to convert into win1252

* Convert Vic2 localisations from WIn1252 instead of ISO 8859-15.

* Fixes for Linux version.

Vic2ToHoI4/CMakeLists.txt

@@ -24,10 +24,6 @@ set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/Date.cpp")
 set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/LinuxUtils.cpp")
 set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/Log.cpp")
 set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/newParser.cpp")
-set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/NewParserToOldParserConverters.cpp")
-set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/Object.cpp")
-set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/ParadoxParser8859_15.cpp")
-set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/ParadoxParserUTF8.cpp")
 set(COMMON_SOURCES ${COMMON_SOURCES} "../common_items/ParserHelpers.cpp")
 
 set(Boost_USE_STATIC_LIBS       OFF)

Vic2ToHoI4/Source/HOI4World/HoI4Buildings.h

@@ -73,6 +73,9 @@ class Building
 };
 
 
+std::ostream& operator << (std::ostream& out, const Building& building);
+
+
 typedef std::map<std::pair<int, int>, buildingPosition> defaultPositions;
 
 

Vic2ToHoI4/Source/HOI4World/HoI4Leader.h

@@ -56,6 +56,9 @@ class General
 };
 
 
+std::ofstream& operator << (std::ofstream& output, const General& instance);
+
+
 class Admiral
 {
 	public:
@@ -73,6 +76,9 @@ class Admiral
 		int skill = 1;
 };
 
+
+std::ofstream& operator << (std::ofstream& output, const Admiral& instance);
+
 }
 
 

Vic2ToHoI4/Source/HOI4World/HoI4Navy.h

@@ -53,6 +53,9 @@ class Ship
 };
 
 
+std::ofstream& operator << (std::ofstream& output, const Ship& instance);
+
+
 class Navy
 {
 	public:
@@ -71,6 +74,10 @@ class Navy
 		std::vector<Ship> ships;
 };
 
+
+std::ofstream& operator << (std::ofstream& output, const Navy& instance);
+
+
 }
 
 

Vic2ToHoI4/Source/HOI4World/HoI4World.cpp

@@ -178,7 +178,7 @@ void HoI4::World::convertCountry(pair<string, Vic2::Country*> country)
 			LOG(LogLevel::Warning) << "Could not set country name when converting country";
 		}
 
-		std::string countryFileName = Utils::convert8859_15ToUTF8(countryName) + ".txt";
+		std::string countryFileName = Utils::convertWin1252ToUTF8(countryName) + ".txt";
 		int pipe = countryFileName.find_first_of('|');
 		while (pipe != string::npos)
 		{

Vic2ToHoI4/Source/HOI4World/Names.cpp

@@ -117,8 +117,8 @@ culture::culture(std::istream& theStream)
 
 void::culture::convertNamesToUTF8()
 {
-	std::for_each(maleNames.begin(), maleNames.end(), [](std::string& name){ name = Utils::convert8859_15ToUTF8(name); });
-	std::for_each(surnames.begin(), surnames.end(), [](std::string& name){ name = Utils::convert8859_15ToUTF8(name); });
+	std::for_each(maleNames.begin(), maleNames.end(), [](std::string& name){ name = Utils::convertWin1252ToUTF8(name); });
+	std::for_each(surnames.begin(), surnames.end(), [](std::string& name){ name = Utils::convertWin1252ToUTF8(name); });
 }
 
 

Vic2ToHoI4/Source/Mappers/V2Localisations.cpp

@@ -90,8 +90,7 @@ void V2Localisations::processLine(string line)
 	{
 		pause = false;
 		string result = getNextLocalisation(line, division);
-		result = replaceBadCharacters(result);
-		auto UTF8Result = Utils::convert8859_15ToUTF8(result);
+		auto UTF8Result = Utils::convertWin1252ToUTF8(result);
 
 		if (language == "english")
 		{
@@ -117,52 +116,6 @@ string V2Localisations::getNextLocalisation(string line, int& division)
 }
 
 
-string V2Localisations::replaceBadCharacters(string localisation)
-{
-	// Ö gets translated to an invalid character sequence. Oe is accepted substitute in German.
-	int O = localisation.find_first_of('Ö');
-	while (O != string::npos)
-	{
-		localisation.replace(O, 1, "Oe");
-		O = localisation.find_first_of('Ö');
-	}
-
-	// dash characters other than 0x2D break HoI4
-	int dash = localisation.find_first_of('–');
-	while (dash != string::npos)
-	{
-		localisation.replace(dash, 1, "-");
-		dash = localisation.find_first_of('–');
-	}
-
-	// Problem with S-hacek
-	int shacek = localisation.find_first_of('š');	// character may not display, but is present
-	while (shacek != string::npos)
-	{
-		localisation.replace(shacek, 1, "š");
-		shacek = localisation.find_first_of('š');	// character may not display, but is present
-	}
-
-	// Problem with Z-hacek
-	int zhacek = localisation.find_first_of('ž');	// character may not display, but is present
-	while (zhacek != string::npos)
-	{
-		localisation.replace(zhacek, 1, "ž");
-		zhacek = localisation.find_first_of('ž');	// character may not display, but is present
-	}
-
-	// Problem with apostrophe (Spanish)
-	int apostrophe = localisation.find_first_of('’');	// character may not display, but is present
-	while (apostrophe != string::npos)
-	{
-		localisation.replace(apostrophe, 1, "’");
-		apostrophe = localisation.find_first_of('’');	// character may not display, but is present
-	}
-
-	return localisation;
-}
-
-
 void V2Localisations::ActuallyUpdateDomainCountry(const string& tag, const string& domainName)
 {
 	LanguageToLocalisationMap regionLocalisations;

Vic2ToHoI4/Source/Mappers/V2Localisations.h

@@ -66,7 +66,6 @@ class V2Localisations
 		void ReadFromFile(const string& fileName);
 		void processLine(string line);
 		string getNextLocalisation(string line, int& division);
-		string replaceBadCharacters(string localisation);
 
 		V2Localisations(const V2Localisations&) = delete;
 		V2Localisations& operator=(const V2Localisations&) = delete;

Vic2ToHoI4/Source/V2World/Army.cpp

@@ -32,7 +32,7 @@ Vic2::Regiment::Regiment(std::istream& theStream)
 {
 	registerKeyword(std::regex("name"), [this](const std::string& unused, std::istream& theStream){
 		commonItems::singleString nameString(theStream);
-		name = Utils::convert8859_15ToUTF8(nameString.getString());
+		name = Utils::convertWin1252ToUTF8(nameString.getString());
 	});
 	registerKeyword(std::regex("type"), [this](const std::string& unused, std::istream& theStream){
 		commonItems::singleString typeString(theStream);
@@ -84,7 +84,7 @@ Vic2::Army::Army(const std::string& type, std::istream& theStream):
 {
 	registerKeyword(std::regex("name"), [this](const std::string& unused, std::istream& theStream){
 		commonItems::singleString nameString(theStream);
-		name = Utils::convert8859_15ToUTF8(nameString.getString());
+		name = Utils::convertWin1252ToUTF8(nameString.getString());
 	});
 	registerKeyword(std::regex("location"), [this](const std::string& unused, std::istream& theStream){
 		commonItems::singleInt locationInt(theStream);

Vic2ToHoI4/Source/V2World/CommonCountryData.cpp

@@ -48,7 +48,7 @@ Vic2::commonCountryData::commonCountryData(const std::string& filename, const st
 				{
 					name = name.substr(1, name.length() - 2);
 				}
-				unitNames[*token].emplace_back(Utils::convert8859_15ToUTF8(name));
+				unitNames[*token].emplace_back(Utils::convertWin1252ToUTF8(name));
 			}
 
 			token = getNextTokenWithoutMatching(theStream);

Vic2ToHoI4/Source/V2toHOI4Converter.cpp

@@ -153,7 +153,7 @@ void setOutputName(const std::string& V2SaveFileName)
 	const int length = outputName.find_first_of(".");
 	if ((length == std::string::npos) || (".v2" != outputName.substr(length, outputName.length())))
 	{
-		std::exception theException("The save was not a Vic2 save. Choose a save ending in '.v2' and converter again.");
+		std::invalid_argument theException("The save was not a Vic2 save. Choose a save ending in '.v2' and convert again.");
 		throw theException;
 	}
 	outputName = outputName.substr(0, length);

common_items/LinuxUtils.cpp

@@ -1014,11 +1014,19 @@ namespace Utils
 		return ConvertString<string, string>("UTF−8", "ISO−8859−15", UTF8);
 	}
 
+
+	std::string convertUTF8ToWin1252(const std::string& UTF8)
+	{
+		using namespace std;
+		return ConvertString<string, string>("UTF−8", "CP1252", UTF8);
+	}
+
+
 	/*
 		Warning: The input string should not be encoded in UTF-16 but in the system dependent wchar_t encoding
 		see convertUTF8ToUTF16 for full explanation
 	*/
-	std::string convertUTF16ToUTF8(std::wstring UTF16)
+	std::string convertUTF16ToUTF8(const std::wstring& UTF16)
 	{
 		using namespace std;
 		return ConvertString<wstring, string>("wchar_t", "UTF-8", UTF16);
@@ -1035,31 +1043,44 @@ namespace Utils
 		This is an implementation of the original Windows-based API which uses UTF-16 LE as the system dependent wchar_t encoding
 		This behaviour is replicated on Linux but it uses the (system dependent) wchar_t encoding.
 	*/
-	std::wstring convert8859_15ToUTF16(std::string UTF8)
+	std::wstring convert8859_15ToUTF16(const std::string& UTF8)
 	{
 		using namespace std;
 		return ConvertString<string, wstring>("ISO−8859−15", "wchar_t", UTF8);
 	}
+
+	std::string convertWin1252ToUTF8(const std::string& Win1252)
+	{
+		using namespace std;
+		return ConvertString<string, string>("CP1252", "UTF-8", Win1252);
+	}
+
+
+	std::wstring convertWin1252ToUTF16(const std::string& Win1252)
+	{
+		using namespace std;
+		return ConvertString<string, wstring>("CP1252", "wchar_t", Win1252);
+	}
 
 	/*
 		Warning: Does not actually return an UTF-16 sequence.
 		This is an implementation of the original Windows-based API which uses UTF-16 LE as the system dependent wchar_t encoding
 		This behaviour is replicated on Linux but it uses the (system dependent) wchar_t encoding.
 	*/
-	std::wstring convertUTF8ToUTF16(std::string UTF8)
+	std::wstring convertUTF8ToUTF16(const std::string& UTF8)
 	{
 		using namespace std;
 		return ConvertString<string, wstring>("UTF-8", "wchar_t",UTF8);
 	}
 
 
-	std::string convertToUTF8(const std::wstring &input)
+	std::string convertToUTF8(const std::wstring& input)
 	{
 		using namespace std;
 		return ConvertString<wstring, string>("wchar_t", "UTF-8",input);
 	}
 
-	std::string normalizeUTF8Path(const std::string &utf_8_path){
+	std::string normalizeUTF8Path(const std::string& utf_8_path){
 		return utf_8_path;
 	};
 }

common_items/OSCompatibilityLayer.h

@@ -109,11 +109,15 @@ namespace Utils
 
 	std::string convertUTF8ToASCII(const std::string& UTF8);
 	std::string convertUTF8To8859_15(const std::string& UTF8);
-	std::string convertUTF16ToUTF8(std::wstring UTF16);
-	std::string convert8859_15ToASCII(std::string input);
+	std::string convertUTF8ToWin1252(const std::string& UTF8);
+	std::string convertUTF16ToUTF8(const std::wstring& UTF16);
+	std::string convert8859_15ToASCII(const std::string& input);
 	std::string convert8859_15ToUTF8(const std::string& input);
-	std::wstring convert8859_15ToUTF16(std::string UTF8);
-	std::wstring convertUTF8ToUTF16(std::string UTF8);
+	std::wstring convert8859_15ToUTF16(const std::string& input);
+	std::string convertWin1252ToASCII(const std::string& Win1252);
+	std::string convertWin1252ToUTF8(const std::string& Win1252);
+	std::wstring convertWin1252ToUTF16(const std::string& Win1252);
+	std::wstring convertUTF8ToUTF16(const std::string& UTF8);
 
 	// converts a string in the system dependent wchar_t encoding to UTF-8
 	std::string convertToUTF8(const std::wstring &input);

common_items/WinUtils.cpp

@@ -308,7 +308,24 @@ std::string convertUTF8To8859_15(const std::string& UTF8)
 }
 
 
-std::string convertUTF16ToUTF8(std::wstring UTF16)
+std::string convertUTF8ToWin1252(const std::string& UTF8)
+{
+	int requiredSize = WideCharToMultiByte(1252, 0, convertUTF8ToUTF16(UTF8).c_str(), -1, NULL, 0, "0", NULL);
+	char* asciiArray = new char[requiredSize];
+
+	if (0 == WideCharToMultiByte(1252, 0, convertUTF8ToUTF16(UTF8).c_str(), -1, asciiArray, requiredSize, "0", NULL))
+	{
+		LOG(LogLevel::Error) << "Could not translate string to ASCII - " << GetLastErrorString();
+	}
+	std::string returnable(asciiArray);
+
+	delete[] asciiArray;
+
+	return returnable;
+}
+
+
+std::string convertUTF16ToUTF8(const std::wstring& UTF16)
 {
 	const int requiredSize = WideCharToMultiByte(CP_UTF8, 0, UTF16.c_str(), -1, NULL, 0, NULL, NULL);
 	char* utf8array = new char[requiredSize];
@@ -325,7 +342,7 @@ std::string convertUTF16ToUTF8(std::wstring UTF16)
 }
 
 
-std::string convert8859_15ToASCII(std::string input)
+std::string convert8859_15ToASCII(const std::string& input)
 {
 	return Utils::convertUTF8ToASCII(Utils::convert8859_15ToUTF8(input));
 }
@@ -337,12 +354,41 @@ std::string convert8859_15ToUTF8(const std::string& input)
 }
 
 
-std::wstring convert8859_15ToUTF16(std::string string8859_15)
+std::wstring convert8859_15ToUTF16(const std::string& input)
+{
+	const int requiredSize = MultiByteToWideChar(28605 /* 8859-15*/, MB_PRECOMPOSED, input.c_str(), -1, NULL, 0);
+	wchar_t* wideKeyArray = new wchar_t[requiredSize];
+
+	if (0 == MultiByteToWideChar(28605 /* 8859-15*/, MB_PRECOMPOSED, input.c_str(), -1, wideKeyArray, requiredSize))
+	{
+		LOG(LogLevel::Error) << "Could not translate string to UTF-16 - " << GetLastErrorString();
+	}
+	std::wstring returnable(wideKeyArray);
+
+	delete[] wideKeyArray;
+
+	return returnable;
+}
+
+
+std::string convertWin1252ToASCII(const std::string& input)
+{
+	return Utils::convertUTF8ToASCII(Utils::convertWin1252ToUTF8(input));
+}
+
+
+std::string convertWin1252ToUTF8(const std::string& input)
+{
+	return convertUTF16ToUTF8(convertWin1252ToUTF16(input));
+}
+
+
+std::wstring convertWin1252ToUTF16(const std::string& input)
 {
-	const int requiredSize = MultiByteToWideChar(28605 /* 8859-15*/, MB_PRECOMPOSED, string8859_15.c_str(), -1, NULL, 0);
+	const int requiredSize = MultiByteToWideChar(1252, MB_PRECOMPOSED, input.c_str(), -1, NULL, 0);
 	wchar_t* wideKeyArray = new wchar_t[requiredSize];
 
-	if (0 == MultiByteToWideChar(28605 /* 8859-15*/, MB_PRECOMPOSED, string8859_15.c_str(), -1, wideKeyArray, requiredSize))
+	if (0 == MultiByteToWideChar(1252, MB_PRECOMPOSED, input.c_str(), -1, wideKeyArray, requiredSize))
 	{
 		LOG(LogLevel::Error) << "Could not translate string to UTF-16 - " << GetLastErrorString();
 	}
@@ -354,7 +400,7 @@ std::wstring convert8859_15ToUTF16(std::string string8859_15)
 }
 
 
-std::wstring convertUTF8ToUTF16(std::string UTF8)
+std::wstring convertUTF8ToUTF16(const std::string& UTF8)
 {
 	const int requiredSize = MultiByteToWideChar(CP_UTF8, 0, UTF8.c_str(), -1, NULL, 0);
 	if (requiredSize == 0)
@@ -374,11 +420,14 @@ std::wstring convertUTF8ToUTF16(std::string UTF8)
 	return returnable;
 }
 
-std::string convertToUTF8(const std::wstring &input){
+
+std::string convertToUTF8(const std::wstring& input)
+{
 	return convertUTF16ToUTF8(input);
 }
 
-std::string normalizeUTF8Path(const std::string &utf_8_path)
+
+std::string normalizeUTF8Path(const std::string& utf_8_path)
 {
 	std::string asciiPath = convertUTF8ToASCII(utf_8_path);
 	std::replace(asciiPath.begin(), asciiPath.end(), '/', '_');
@@ -394,6 +443,7 @@ std::string normalizeUTF8Path(const std::string &utf_8_path)
 	return asciiPath;
 };
 
+
 void WriteToConsole(LogLevel level, const std::string& logMessage)
 {
 	if (level == LogLevel::Debug)