Merge pull request #1206 from rommapp/feat/add-rahasher

feat: Add RAHasher and util to calculate RetroAchievements hashes

backend/adapters/services/rahasher.py

@@ -0,0 +1,127 @@
+import asyncio
+import re
+from pathlib import Path
+
+from logger.logger import log
+
+RAHASHER_VALID_HASH_REGEX = re.compile(r"^[0-9a-f]{32}$")
+
+# TODO: Centralize standarized platform slugs using StrEnum.
+PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = {
+    "3do": 43,
+    "cpc": 37,
+    "acpc": 37,
+    "apple2": 38,
+    "appleii": 38,
+    "arcade": 27,
+    "arcadia-2001": 73,
+    "arduboy": 71,
+    "atari-2600": 25,
+    "atari2600": 25,
+    "atari-7800": 51,
+    "atari7800": 51,
+    "atari-jaguar-cd": 77,
+    "colecovision": 44,
+    "dreamcast": 40,
+    "dc": 40,
+    "gameboy": 4,
+    "gb": 4,
+    "gameboy-advance": 5,
+    "gba": 5,
+    "gameboy-color": 6,
+    "gbc": 6,
+    "game-gear": 15,
+    "gamegear": 15,
+    "gamecube": 16,
+    "ngc": 14,
+    "genesis": 1,
+    "genesis-slash-megadrive": 16,
+    "intellivision": 45,
+    "jaguar": 17,
+    "lynx": 13,
+    "msx": 29,
+    "mega-duck-slash-cougar-boy": 69,
+    "nes": 7,
+    "famicom": 7,
+    "neo-geo-cd": 56,
+    "neo-geo-pocket": 14,
+    "neo-geo-pocket-color": 14,
+    "n64": 2,
+    "nintendo-ds": 18,
+    "nds": 18,
+    "nintendo-dsi": 78,
+    "odyssey-2": 23,
+    "pc-8000": 47,
+    "pc-8800-series": 47,
+    "pc-fx": 49,
+    "psp": 41,
+    "playstation": 12,
+    "ps": 12,
+    "ps2": 21,
+    "pokemon-mini": 24,
+    "saturn": 39,
+    "sega-32x": 10,
+    "sega32": 10,
+    "sega-cd": 9,
+    "segacd": 9,
+    "sega-master-system": 11,
+    "sms": 11,
+    "sg-1000": 33,
+    "snes": 3,
+    "turbografx-cd": 76,
+    "turbografx-16-slash-pc-engine-cd": 76,
+    "turbo-grafx": 8,
+    "turbografx16--1": 8,
+    "vectrex": 26,
+    "virtual-boy": 28,
+    "virtualboy": 28,
+    "watara-slash-quickshot-supervision": 63,
+    "wonderswan": 53,
+    "wonderswan-color": 53,
+}
+
+
+class RAHasherError(Exception): ...
+
+
+class RAHasherService:
+    """Service to calculate RetroAchievements hashes using RAHasher."""
+
+    async def calculate_hash(self, platform_slug: str, file_path: Path) -> str:
+        platform_id = PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID.get(platform_slug)
+        if not platform_id:
+            raise RAHasherError(
+                f"Platform not supported by RetroAchievements. {platform_slug=}"
+            )
+
+        args = (str(platform_id), str(file_path))
+        log.debug("Executing RAHasher with args: %s", args)
+
+        proc = await asyncio.create_subprocess_exec(
+            "RAHasher",
+            *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        return_code = await proc.wait()
+        if return_code != 1:
+            if proc.stderr is not None:
+                stderr = (await proc.stderr.read()).decode("utf-8")
+            else:
+                stderr = None
+            raise RAHasherError(f"RAHasher failed with code {return_code}. {stderr=}")
+
+        if proc.stdout is None:
+            raise RAHasherError("RAHasher did not return a hash.")
+
+        file_hash = (await proc.stdout.read()).decode("utf-8").strip()
+        if not file_hash:
+            raise RAHasherError(
+                f"RAHasher returned an empty hash. {platform_id=}, {file_path=}"
+            )
+        if not RAHASHER_VALID_HASH_REGEX.match(file_hash):
+            raise RAHasherError(
+                f"RAHasher returned an invalid hash: {file_hash=}, {platform_id=}, {file_path=}"
+            )
+
+        return file_hash

docker/Dockerfile

@@ -1,9 +1,18 @@
+# Stages:
+#   - front-build-stage: Build frontend
+#   - backend-build: Build backend environment
+#   - rahasher-build: Build RAHasher
+#   - nginx-build: Build nginx modules
+#   - production-stage: Setup frontend and backend
+#   - final-stage: Move everything to final stage
+
+# Versions:
 ARG ALPINE_VERSION=3.20
 ARG NGINX_VERSION=1.27.1
 ARG NODE_VERSION=20.16
 ARG PYTHON_VERSION=3.12
 
-# Build frontend
+
 FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage
 WORKDIR /front
 
@@ -13,7 +22,7 @@ RUN npm ci
 COPY ./frontend ./
 RUN npm run build
 
-# Build backend environment
+
 FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS backend-build
 
 # libffi-dev is needed to fix poetry dependencies for >= v1.8 on arm64
@@ -31,9 +40,33 @@ ENV POETRY_NO_INTERACTION=1 \
 WORKDIR /src
 
 COPY ./pyproject.toml ./poetry.lock /src/
-RUN poetry install --no-ansi --no-cache --only main
+RUN poetry install --no-ansi --no-cache
+
+
+# TODO: Upgrade Alpine to the same version as the other stages, when RAHasher is updated to work
+#       with it (seems like Alpine 3.18's g++ v12 is the latest version that works with RAHasher,
+#       while g++ v13 fails to compile it).
+FROM alpine:3.18 AS rahasher-build
+
+RUN apk add --no-cache \
+    g++ \
+    git \
+    linux-headers \
+    make \
+    zlib-dev
+
+# TODO: Change to a tagged version, once v1.7.2 or newer is released.
+#       Current pinned commit is needed to enable building RAHasher on ARM64.
+ARG RALIBRETRO_SHA=5b60a1d6d067238ece378b6250ae1ae8aeb90904
+
+# TODO: Remove `sed` command when RAHasher can be compiled without it.
+RUN git clone --recursive https://github.com/RetroAchievements/RALibretro.git && \
+    cd ./RALibretro && \
+    git checkout "${RALIBRETRO_SHA}" && \
+    sed -i '22a #include <ctime>' ./src/Util.h && \
+    make HAVE_CHD=1 -f ./Makefile.RAHasher
+
 
-# Build nginx modules
 FROM alpine:${ALPINE_VERSION} AS nginx-build
 
 RUN apk add --no-cache \
@@ -63,10 +96,11 @@ RUN git clone https://github.com/evanmiller/mod_zip.git && \
     make -f ./objs/Makefile modules && \
     chmod 644 ./objs/ngx_http_zip_module.so
 
-# Setup frontend and backend
+
 FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION} AS production-stage
 ARG WEBSERVER_FOLDER=/var/www/html
 
+COPY --from=rahasher-build /RALibretro/bin64/RAHasher /usr/bin/RAHasher
 COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/
 
 COPY --from=front-build-stage /front/dist ${WEBSERVER_FOLDER}
@@ -101,7 +135,7 @@ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf
 RUN addgroup -g 1000 -S romm && adduser -u 1000 -D -S -G romm romm && \
     mkdir /romm /redis-data && chown romm:romm /romm /redis-data
 
-# Move everything to final stage
+
 FROM scratch AS final-stage
 
 COPY --from=production-stage / /