Merge branch 'feature/rp-download-split-from-libcachemgr'

Not quite complete for Windows, and no extra security functionality on
Linux aside from running as a separate process, but it's good enough
for now.

NETWORK.md

@@ -17,6 +17,6 @@ The configuration file is located at:
 
 ## Domains accessed:
 
-* http://art.gametdb.com/ - Box, cover, and media scans for Nintendo
+* https://art.gametdb.com/ - Box, cover, and media scans for Nintendo
   GameCube, Wii, Wii U, DS, and 3DS games.
 * https://amiibo.life/ - Images of Nintendo amiibo products.

NEWS.md

@@ -23,6 +23,18 @@
     to another file browser window to extract them. This includes the icon,
     banner, and icons from ListView widgets.
     * Drag & drop support for Win32 and GTK+ may be implemented before release.
+  * Downloading functionality for online databases has been split out of the
+    DLL and into a separate executable. This will allow the downloading to be
+    handled in a lower privilege environment.
+    * Windows: On Vista and later, rp-download runs as a low-integrity process.
+      * TODO before release:
+        * Add more security options.
+        * Have rp-download drop privileges if run directly.
+        * If rp-download.exe isn't in the current directory, check the
+          architecture-specific subdirectory.
+    * Linux: TODO: Need to set up apparmor, possibly libseccomp.
+  * Windows: The online database code has been rewritten to use WinInet
+    directly instead of urlmon, which reduces overhead.
 
 * New parsers:
   * DidjTex: Leapster Didj .tex and .texs texture files. For .texs, currently
@@ -80,6 +92,8 @@
     This fixes handling of mtimes and content length.
   * UrlmonDownloader: Handle ERROR_INSUFFICIENT_BUFFER for the cache info
     entry correctly. Otherwise, we won't get the mtime.
+  * Fixed some CBC decryption issues on Windows XP. This mostly affected
+    decrypting Xbox 360 executables.
 
 * Other changes:
   * Removed the internal copy of libjpeg-turbo. On Windows, gdiplus is now

appveyor.yml

@@ -30,15 +30,8 @@ configuration:
   - Debug
   - Release
 
-environment:
-  NASM_URL: http://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-installer-x64.exe
-
-# Install NASM, then run CMake.
+# Run CMake.
 before_build:
-  # TODO: Remove after https://github.com/appveyor/ci/issues/1426 is fixed
-  - set "PATH=C:\Program Files\Git\mingw64\bin;%PATH%"
-  - curl -L -o nasminst.exe "%NASM_URL%"
-  - start /wait nasminst /S
   - cd %APPVEYOR_BUILD_FOLDER%
   - appveyor.cmd
 

debian/control

@@ -39,7 +39,7 @@ Vcs-Browser: https://github.com/GerbilSoft/rom-properties
 
 Package: rom-properties-all
 Architecture: any
-Depends: rom-properties-kde4, rom-properties-kde5, rom-properties-xfce, rom-properties-gnome, rom-properties-cinnamon, rom-properties-cli, rom-properties-stub, rom-properties-lang, rom-properties-thumbnailer-dbus, ${misc:Depends}
+Depends: rom-properties-kde4, rom-properties-kde5, rom-properties-xfce, rom-properties-gnome, rom-properties-cinnamon, rom-properties-cli, rom-properties-utils, rom-properties-lang, rom-properties-thumbnailer-dbus, ${misc:Depends}
 Description: ROM Properties Page shell extension
  This shell extension provides thumbnailing and property page functionality
  for ROM images, disc images, and save files for various game consoles,
@@ -82,31 +82,31 @@ Description: ROM Properties Page shell extension
 
 Package: rom-properties-gnome
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-stub, rom-properties-gtk3-common
+Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-utils, rom-properties-gtk3-common
 Recommends: rom-properties-lang
 Description: ROM Properties Page shell extension
  This shell extension provides thumbnailing and property page functionality
  for ROM images, disc images, and save files for various game consoles,
  including Nintendo GameCube and Wii.
  .
  This package contains the Nautilus (GNOME 3, Unity) version.
- The stub package is required for thumbnailing.
+ The utils package is required for thumbnailing.
 
 Package: rom-properties-cinnamon
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-stub, rom-properties-gtk3-common
+Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-utils, rom-properties-gtk3-common
 Recommends: rom-properties-lang
 Description: ROM Properties Page shell extension
  This shell extension provides thumbnailing and property page functionality
  for ROM images, disc images, and save files for various game consoles,
  including Nintendo GameCube and Wii.
  .
  This package contains the Nemo (Cinnamon) version.
- The stub package is required for thumbnailing.
+ The utils package is required for thumbnailing.
 
 Package: rom-properties-gtk3-common
 Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-stub
+Depends: ${shlibs:Depends}, ${misc:Depends}, rom-properties-utils
 Recommends: rom-properties-lang
 Description: ROM Properties Page shell extension
  This shell extension provides thumbnailing and property page functionality
@@ -127,17 +127,19 @@ Description: ROM Properties Page shell extension
  .
  This package contains the command line version.
 
-Package: rom-properties-stub
+Package: rom-properties-utils
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}
 Recommends: rom-properties-lang
+Replaces: rom-properties-stub
 Description: ROM Properties Page shell extension
  This shell extension provides thumbnailing and property page functionality
  for ROM images, disc images, and save files for various game consoles,
  including Nintendo GameCube and Wii.
  .
- This package contains the stub executable, which is used for the
- configuration UI and GNOME thumbnailer.
+ This package contains helper executables for invoking the configuration
+ UI, thumbnailing on some desktop environments, and downloading images
+ from online databases.
 
 Package: rom-properties-lang
 Architecture: all

debian/rom-properties-stub.install → debian/rom-properties-utils.install

@@ -1,3 +1,4 @@
 usr/bin/rp-stub
 usr/bin/rp-config
+usr/lib/*/libexec/rp-download
 usr/lib/*/libexec/rp-thumbnail

extlib/libmspack-xenia/xenia_lzx.c

@@ -27,6 +27,21 @@
 # define min(x, y) ((x) < (y) ? (x) : (y))
 #endif
 
+// bit_scan_forward
+#ifdef _MSC_VER
+#include <intrin.h>
+static __inline unsigned char bit_scan_forward(uint32_t v, uint32_t* out_first_set_index) {
+	return _BitScanForward((unsigned long*)(out_first_set_index), v) != 0;
+}
+#else /* !_MSC_VER */
+#include <strings.h>
+static inline unsigned char bit_scan_forward(uint32_t v, uint32_t* out_first_set_index) {
+	int i = ffs(v);
+	*out_first_set_index = i - 1;
+	return i != 0;
+}
+#endif /* _MSC_VER */
+
 typedef struct mspack_memory_file_t {
   struct mspack_system sys;
   void* buffer;
@@ -97,21 +112,16 @@ void mspack_memory_sys_destroy(struct mspack_system* sys) { free(sys); }
 int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest,
                    size_t dest_len, uint32_t window_size, void* window_data,
                    size_t window_data_len) {
-  uint32_t window_bits = 0;
-  uint32_t temp_sz = window_size;
-  size_t m;
   int result_code = 1;
+  uint32_t window_bits;
 
   struct mspack_system* sys;
   mspack_memory_file* lzxsrc;
   mspack_memory_file* lzxdst;
   struct lzxd_stream* lzxd;
 
-  for (m = 0; m < 32; m++, window_bits++) {
-    temp_sz >>= 1;
-    if (temp_sz == 0x00000000) {
-      break;
-    }
+  if (!bit_scan_forward(window_size, &window_bits)) {
+    return result_code;
   }
 
   sys = mspack_memory_sys_create();
@@ -123,10 +133,12 @@ int lzx_decompress(const void* lzx_data, size_t lzx_len, void* dest,
   if (lzxd) {
     if (window_data) {
       // zero the window and then copy window_data to the end of it
-      memset(lzxd->window, 0, window_data_len);
-      memcpy(lzxd->window + (window_size - window_data_len), window_data,
-                  window_data_len);
-      lzxd->ref_data_size = (uint32_t)window_data_len;
+      size_t padding_len = (size_t)window_size - window_data_len;
+      memset(&lzxd->window[0], 0, padding_len);
+      memcpy(&lzxd->window[padding_len], window_data, window_data_len);
+      // TODO(gibbed): should this be set regardless if source window data is
+      // available or not?
+      lzxd->ref_data_size = window_size;
     }
 
     result_code = lzxd_decompress(lzxd, (off_t)dest_len);

src/CMakeLists.txt

@@ -17,16 +17,24 @@ SET(CMAKE_CXX_FLAGS	"${CMAKE_CXX_FLAGS} ${RP_CXX11_COMPAT_HEADER}")
 INCLUDE(CheckHiddenVisibility)
 CHECK_HIDDEN_VISIBILITY()
 
+# Check for stdbool.h.
+INCLUDE(CheckIncludeFile)
+CHECK_INCLUDE_FILE("stdbool.h" HAVE_STDBOOL_H)
+
 # Write the version number to config.version.h.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/config.version.h.in" "${CMAKE_CURRENT_BINARY_DIR}/config.version.h")
 
+# Write stdbool.h.
+CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/stdboolx.h.in" "${CMAKE_CURRENT_BINARY_DIR}/stdboolx.h")
+
 # Source Code subdirectories.
 ADD_SUBDIRECTORY(libi18n)
 ADD_SUBDIRECTORY(librpthreads)
+ADD_SUBDIRECTORY(libcachecommon)
+ADD_SUBDIRECTORY(rp-download)
 ADD_SUBDIRECTORY(librpbase)
 ADD_SUBDIRECTORY(librptexture)
 ADD_SUBDIRECTORY(libromdata)
-ADD_SUBDIRECTORY(libcachemgr)
 
 IF(BUILD_CLI)
 	ADD_SUBDIRECTORY(rpcli)

src/gtk/cinnamon/CMakeLists.txt

@@ -80,7 +80,7 @@ IF(BUILD_CINNAMON)
 			$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
 			$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/src>
 		)
-	TARGET_LINK_LIBRARIES(rom-properties-cinnamon PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-cinnamon PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-cinnamon PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/gtk/gnome/CMakeLists.txt

@@ -89,7 +89,7 @@ IF(BUILD_GNOME)
 			$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
 			$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/src>
 		)
-	TARGET_LINK_LIBRARIES(rom-properties-gnome PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-gnome PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-gnome PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/gtk/mate/CMakeLists.txt

@@ -80,7 +80,7 @@ IF(BUILD_MATE)
 			$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src>
 			$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/src>
 		)
-	TARGET_LINK_LIBRARIES(rom-properties-mate PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-mate PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-mate PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/gtk/xfce/CMakeLists.txt

@@ -84,7 +84,7 @@ IF(BUILD_XFCE)
 		)
 	TARGET_INCLUDE_DIRECTORIES(rom-properties-xfce PUBLIC ${GTK2_INCLUDE_DIRS})
 
-	TARGET_LINK_LIBRARIES(rom-properties-xfce PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-xfce PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-xfce PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/gtk/xfce3/CMakeLists.txt

@@ -84,7 +84,7 @@ IF(BUILD_XFCE3)
 		)
 	TARGET_INCLUDE_DIRECTORIES(rom-properties-xfce3 PUBLIC ${GTK3_INCLUDE_DIRS})
 
-	TARGET_LINK_LIBRARIES(rom-properties-xfce3 PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-xfce3 PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-xfce3 PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/kde/kde4/CMakeLists.txt

@@ -59,7 +59,7 @@ IF(BUILD_KDE4)
 			$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>	# src
 			$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/../..>	# src
 		)
-	TARGET_LINK_LIBRARIES(rom-properties-kde4 PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-kde4 PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-kde4 PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/kde/kde5/CMakeLists.txt

@@ -129,7 +129,7 @@ IF(BUILD_KDE5)
 			$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../..>	# src
 			$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/../..>	# src
 		)
-	TARGET_LINK_LIBRARIES(rom-properties-kde5 PRIVATE cachemgr romdata rpbase)
+	TARGET_LINK_LIBRARIES(rom-properties-kde5 PRIVATE romdata rpbase)
 	IF(ENABLE_NLS)
 		TARGET_LINK_LIBRARIES(rom-properties-kde5 PRIVATE i18n)
 	ENDIF(ENABLE_NLS)

src/libcachecommon/CMakeLists.txt

@@ -0,0 +1,47 @@
+PROJECT(libcachecommon)
+# Common cache management functions used by both Cache Manager and rp-download.
+
+SET(libcachecommon_SRCS
+	CacheKeys.cpp
+	CacheDir.cpp
+	)
+SET(libcachecommon_H
+	CacheKeys.hpp
+	CacheDir.hpp
+	)
+
+######################
+# Build the library. #
+######################
+
+INCLUDE(SetMSVCDebugPath)
+
+ADD_LIBRARY(cachecommon STATIC
+	${libcachecommon_SRCS} ${libcachecommon_H}
+	)
+SET_MSVC_DEBUG_PATH(cachecommon)
+TARGET_INCLUDE_DIRECTORIES(cachecommon
+	PUBLIC  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+		$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+	PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
+		$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/..>
+	)
+# Exclude from ALL builds.
+SET_TARGET_PROPERTIES(cachecommon PROPERTIES EXCLUDE_FROM_ALL TRUE)
+TARGET_LINK_LIBRARIES(cachecommon PRIVATE rpthreads)
+IF(WIN32)
+	TARGET_LINK_LIBRARIES(cachecommon PRIVATE win32common)
+ELSE(WIN32)
+	TARGET_LINK_LIBRARIES(cachecommon PRIVATE unixcommon)
+ENDIF(WIN32)
+
+# Unix: Add -fpic/-fPIC in order to use these static libraries in plugins.
+IF(UNIX AND NOT APPLE)
+	SET(CMAKE_C_FLAGS	"${CMAKE_C_FLAGS} -fpic -fPIC")
+	SET(CMAKE_CXX_FLAGS	"${CMAKE_CXX_FLAGS} -fpic -fPIC")
+ENDIF(UNIX AND NOT APPLE)
+
+# Test suite.
+IF(BUILD_TESTING)
+	ADD_SUBDIRECTORY(tests)
+ENDIF(BUILD_TESTING)