Fix fallout from SimpleGraphic upgrade with wider Unicode support (#8412

)

* fix: enable Unicode separators and caret motions

As the runtime is going to support Unicode installation locations and
build directories, some UTF-8 text is going to reach the Lua side of
the project. This includes the script path, the user path, any paths
yielded from file searches and also imported character names from
accounts.

Care needs to be taken in many places where string operations are
performed as no longer does a byte necessarily correspond to a single
character and anything that truncates, reverses or otherwise slices
strings could need an audit.

This change fixes cursor movement in `EditControl`s with the arrow keys
as those historically used string matching and byte offsets. It also
ensures that the use of arbitrary Unicode codepoints as decimal and
thousands separators works correctly as the previous code used unaware
reversing and slicing.

* fix: turn update paths relative for wide installs

The updater is a fixed piece of older code that uses a Lua runtime that
only handles paths that are representable in the user's text codepage.

As the software may be installed in a location that cannot be expressed
in that way, to mitigate the problem we turn all the paths in the
update op-files into relative paths. That way as long as we never use
exotic codepoints in our own paths it should be able to apply them
cleanly and restart Path of Building afterward with a relative path.

The updater executable can ironically enough not be updated at all with
the related type of runtime hacks we introduced in SimpleGraphic as the
updater deadlocks in updating itself. We have to work around its
shortcomings in how we produce the op-files and possibly the update
application script that runs under that limited runtime.

* fix: convert GIFs masquerading as PNG to PNG

Upon removing support for several file formats like GIF and BLP from the
SimpleGraphic runtime, we noticed that there were some assets that had
incorrect file extensions and loaded only thanks to file format
detection ignoring extensions.

As the actual file format loader for GIF was removed, these stealth GIFs
are now losslessly converted to PNG.

* Add luautf8 to Dockerfile

---------

Co-authored-by: Wires77 <[email protected]>

Dockerfile

@@ -31,7 +31,8 @@ RUN --mount=type=cache,from=luarocks,source=/opt,target=/opt make -C /opt/luaroc
 # Install here to install lua rocks pkgs in pararell with compilation of emmylua and luajit
 RUN luarocks install busted 2.2.0-1;\
 	luarocks install cluacov 0.1.2-1;\
-	luarocks install luacov-coveralls 0.2.3-1
+	luarocks install luacov-coveralls 0.2.3-1;\
+	luarocks install luautf8 0.1.6-1
 
 RUN --mount=type=cache,from=emmyluadebugger,source=/opt,target=/opt make -C /opt/EmmyLuaDebugger/build/ install
 RUN --mount=type=cache,from=luajit,source=/opt,target=/opt make -C /opt/LuaJIT/ install

src/Classes/EditControl.lua

@@ -7,6 +7,7 @@ local m_max = math.max
 local m_min = math.min
 local m_floor = math.floor
 local protected_replace = "*"
+local utf8 = require('lua-utf8')
 
 local function lastLine(str)
 	local lastLineIndex = 1
@@ -541,18 +542,10 @@ function EditClass:OnKeyDown(key, doubleClick)
 		if self.caret > 1 then
 			if ctrl then
 			-- Skip leading space, then jump word
-				while self.buf:sub(self.caret-1, self.caret-1):match("[%s%p]") do
-					if self.caret > 1 then
-						self.caret = self.caret - 1
-					end
-				end
-				while self.buf:sub(self.caret-1, self.caret-1):match("%w") do
-					if self.caret > 1 then
-						self.caret = self.caret - 1
-					end
-				end
+				self.caret = self.caret - #utf8.match(self.buf:sub(1, self.caret-1), "[%s%p]*$")
+				self.caret = self.caret - #utf8.match(self.buf:sub(1, self.caret-1), "%w*$")
 			else
-				self.caret = self.caret - 1
+				self.caret = utf8.next(self.buf, self.caret, -1) or 0
 			end
 			self.lastUndoState.caret = self.caret
 			self:ScrollCaretIntoView()
@@ -562,19 +555,11 @@ function EditClass:OnKeyDown(key, doubleClick)
 		self.sel = shift and (self.sel or self.caret) or nil
 		if self.caret <= #self.buf then
 			if ctrl then
-			-- Jump word, then skip trailing space, 
-				while self.buf:sub(self.caret, self.caret):match("%w") do
-					if self.caret <= #self.buf then
-						self.caret = self.caret + 1
-					end
-				end
-				while self.buf:sub(self.caret, self.caret):match("[%s%p]") do
-					if self.caret <= #self.buf then
-						self.caret = self.caret + 1
-					end
-				end
+			-- Jump word, then skip trailing space,
+				self.caret = self.caret + #utf8.match(self.buf:sub(self.caret), "^%w*")
+				self.caret = self.caret + #utf8.match(self.buf:sub(self.caret), "^[%s%p]*")
 			else
-				self.caret = self.caret + 1
+				self.caret = utf8.next(self.buf, self.caret, 1) or #self.buf + 1
 			end
 			self.lastUndoState.caret = self.caret
 			self:ScrollCaretIntoView()

src/Modules/Common.lua

@@ -26,6 +26,7 @@ common.curl = require("lcurl.safe")
 common.xml = require("xml")
 common.base64 = require("base64")
 common.sha1 = require("sha1")
+local utf8 = require('lua-utf8')
 
 -- Try to load a library return nil if failed. https://stackoverflow.com/questions/34965863/lua-require-fallback-error-handling
 function prerequire(...)
@@ -657,20 +658,21 @@ function formatNumSep(str)
 		end
 		local x, y, minus, integer, fraction = str:find("(-?)(%d+)(%.?%d*)")
 		if main.showThousandsSeparators then
-			integer = integer:reverse():gsub("(%d%d%d)", "%1"..main.thousandsSeparator):reverse()
+			rev1kSep = utf8.reverse(main.thousandsSeparator)
+			integer = utf8.reverse(utf8.gsub(utf8.reverse(integer), "(%d%d%d)", "%1"..rev1kSep))
 			-- There will be leading separators if the number of digits are divisible by 3
 			-- This checks for their presence and removes them
 			-- Don't use patterns here because thousandsSeparator can be a pattern control character, and will crash if used
 			if main.thousandsSeparator ~= "" then
-				local thousandsSeparator = string.find(integer, main.thousandsSeparator, 1, 2)
+				local thousandsSeparator = utf8.find(integer, rev1kSep, 1, 2)
 				if thousandsSeparator and thousandsSeparator == 1 then
-					integer = integer:sub(2)
+					integer = utf8.sub(integer, 2)
 				end
 			end
 		else
-			integer = integer:reverse():gsub("(%d%d%d)", "%1"):reverse()
+			integer = utf8.reverse(utf8.gsub(utf8.reverse(integer), "(%d%d%d)", "%1"))
 		end
-		return colour..minus..integer..fraction:gsub("%.", main.decimalSeparator)
+		return colour..minus..integer..utf8.gsub(fraction, "%.", main.decimalSeparator)
 	end)
 end
 

src/UpdateCheck.lua

@@ -78,8 +78,8 @@ end
 
 ConPrintf("Checking for update...")
 
-local scriptPath = GetScriptPath()
-local runtimePath = GetRuntimePath()
+local scriptPath = "."
+local runtimePath = "."
 
 -- Load and process local manifest
 local localVer