diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index e085c2ece..946a0733c 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1 @@ -buy_me_a_coffee: zurdi15 +open_collective: romm diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d610a853..b3292e993 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,7 @@ jobs: tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} labels: | org.opencontainers.image.version={{version}} org.opencontainers.image.title="rommapp/romm" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d681f2ebd..ddb70fca6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -42,10 +42,10 @@ jobs: run: | pipx install poetry - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: "poetry" - name: Install dependencies diff --git a/.gitignore b/.gitignore index f570fd1f7..d6cae8462 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,13 @@ coverage /cypress/screenshots/ # Editor directories and files -.vscode +.vscode/* !.vscode/extensions.json !.vscode/settings.json !.vscode/tasks.json +.zed/* +!.zed/settings.json +pyrightconfig.json .idea *.suo *.ntvs* diff --git a/.python-version b/.python-version index 2c0733315..e4fba2183 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11 +3.12 diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 3fbf6ad2c..7b8d38035 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -2,53 +2,73 @@ # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml version: 0.1 cli: - version: 1.22.2 + version: 1.22.3 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) plugins: sources: - id: trunk - ref: v1.6.0 + ref: v1.6.1 uri: https://github.com/trunk-io/plugins # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - go@1.21.0 - node@18.12.1 - - python@3.11.6 + - python@3.12.2 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) lint: enabled: - markdownlint@0.41.0 - - eslint@9.6.0 + - eslint@9.9.0 - actionlint@1.7.1 - bandit@1.7.9 - - black@24.4.2 - - checkov@3.2.178 + - black@24.8.0 + - checkov@3.2.228 - git-diff-check - isort@5.13.2 - - mypy@1.10.1 - - osv-scanner@1.8.1 - - oxipng@9.1.1 - - prettier@3.3.2 - - ruff@0.5.1 + - mypy@1.11.1 + - osv-scanner@1.8.3 + - oxipng@9.1.2 + - prettier@3.3.3 + - ruff@0.6.0 - shellcheck@0.10.0 - shfmt@3.6.0 - svgo@3.3.2 - - taplo@0.8.1 - - trivy@0.52.2 - - trufflehog@3.79.0 + - taplo@0.9.3 + - trivy@0.54.1 + - trufflehog@3.81.9 - yamllint@1.35.1 ignore: - linters: [ALL] paths: - frontend/src/__generated__/** - docker/Dockerfile + - docker/nginx/js/** + files: + - name: vue + extensions: [vue] definitions: - name: eslint - files: [typescript, javascript] + files: + - javascript + - typescript commands: - name: lint run_from: ${root_or_parent_with_any_config} + - name: prettier + files: + - typescript + - yaml + - css + - postcss + - sass + - html + - markdown + - json + - javascript + - graphql + - vue + - prettier_supported_configs actions: disabled: - trunk-check-pre-push diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 000000000..c808db432 --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,16 @@ +{ + "languages": { + "Python": { + "tab_size": 4 + }, + "Vue.js": { + "tab_size": 2, + "formatter": { + "external": { + "command": "prettier", + "arguments": ["--stdin-filepath", "{buffer_path}"] + } + } + } + } +} diff --git a/backend/alembic/versions/0024_sibling_roms_db_view.py b/backend/alembic/versions/0024_sibling_roms_db_view.py new file mode 100644 index 000000000..f6c438936 --- /dev/null +++ b/backend/alembic/versions/0024_sibling_roms_db_view.py @@ -0,0 +1,68 @@ +"""empty message + +Revision ID: 0024_sibling_roms_db_view +Revises: 0023_make_columns_non_nullable +Create Date: 2024-08-08 12:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0024_sibling_roms_db_view" +down_revision = "0023_make_columns_non_nullable" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.create_index("idx_roms_igdb_id", ["igdb_id"]) + batch_op.create_index("idx_roms_moby_id", ["moby_id"]) + + connection = op.get_bind() + + connection.execute( + sa.text( + """ + CREATE VIEW sibling_roms AS + SELECT + r1.id AS rom_id, + r2.id AS sibling_rom_id, + r1.platform_id AS platform_id, + NOW() AS created_at, + NOW() AS updated_at, + CASE WHEN r1.igdb_id <=> r2.igdb_id THEN r1.igdb_id END AS igdb_id, + CASE WHEN r1.moby_id <=> r2.moby_id THEN r1.moby_id END AS moby_id + FROM + roms r1 + JOIN + roms r2 + ON + r1.platform_id = r2.platform_id + AND r1.id != r2.id + AND ( + (r1.igdb_id = r2.igdb_id AND r1.igdb_id IS NOT NULL AND r1.igdb_id != '') + OR + (r1.moby_id = r2.moby_id AND r1.moby_id IS NOT NULL AND r1.moby_id != '') + ); + """ + ), + ) + + +def downgrade() -> None: + connection = op.get_bind() + + connection.execute( + sa.text( + """ + DROP VIEW sibling_roms; + """ + ), + ) + + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.drop_index("idx_roms_igdb_id") + batch_op.drop_index("idx_roms_moby_id") diff --git a/backend/alembic/versions/0025_roms_hashes.py b/backend/alembic/versions/0025_roms_hashes.py new file mode 100644 index 000000000..0d14ade97 --- /dev/null +++ b/backend/alembic/versions/0025_roms_hashes.py @@ -0,0 +1,46 @@ +"""empty message + +Revision ID: 0025_roms_hashes +Revises: 0024_sibling_roms_db_view +Create Date: 2024-08-11 21:50:53.301352 + +""" + +import sqlalchemy as sa +from alembic import op +from config import IS_PYTEST_RUN, SCAN_TIMEOUT +from endpoints.sockets.scan import scan_platforms +from handler.redis_handler import high_prio_queue +from handler.scan_handler import ScanType + +# revision identifiers, used by Alembic. +revision = "0025_roms_hashes" +down_revision = "0024_sibling_roms_db_view" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.add_column(sa.Column("crc_hash", sa.String(length=100), nullable=True)) + batch_op.add_column(sa.Column("md5_hash", sa.String(length=100), nullable=True)) + batch_op.add_column( + sa.Column("sha1_hash", sa.String(length=100), nullable=True) + ) + + # Run a no-scan in the background on migrate + if not IS_PYTEST_RUN: + high_prio_queue.enqueue( + scan_platforms, [], ScanType.QUICK, [], [], job_timeout=SCAN_TIMEOUT + ) + + high_prio_queue.enqueue( + scan_platforms, [], ScanType.HASHES, [], [], job_timeout=SCAN_TIMEOUT + ) + + +def downgrade() -> None: + with op.batch_alter_table("roms", schema=None) as batch_op: + batch_op.drop_column("sha1_hash") + batch_op.drop_column("md5_hash") + batch_op.drop_column("crc_hash") diff --git a/backend/config/__init__.py b/backend/config/__init__.py index c28218d9a..d48134a00 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -6,10 +6,14 @@ load_dotenv() + +def str_to_bool(value: str) -> bool: + return value.lower() in ("true", "1") + + # GUNICORN DEV_PORT: Final = int(os.environ.get("VITE_BACKEND_DEV_PORT", "5000")) DEV_HOST: Final = "127.0.0.1" -ROMM_HOST: Final = os.environ.get("ROMM_HOST", DEV_HOST) GUNICORN_WORKERS: Final = int(os.environ.get("GUNICORN_WORKERS", 2)) # PATHS @@ -54,35 +58,41 @@ ROMM_AUTH_SECRET_KEY: Final = os.environ.get( "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32) ) -DISABLE_CSRF_PROTECTION = os.environ.get("DISABLE_CSRF_PROTECTION", "false") == "true" -DISABLE_DOWNLOAD_ENDPOINT_AUTH = ( - os.environ.get("DISABLE_DOWNLOAD_ENDPOINT_AUTH", "false") == "true" +DISABLE_CSRF_PROTECTION = str_to_bool( + os.environ.get("DISABLE_CSRF_PROTECTION", "false") +) +DISABLE_DOWNLOAD_ENDPOINT_AUTH = str_to_bool( + os.environ.get("DISABLE_DOWNLOAD_ENDPOINT_AUTH", "false") ) # SCANS SCAN_TIMEOUT: Final = int(os.environ.get("SCAN_TIMEOUT", 60 * 60 * 4)) # 4 hours # TASKS -ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = ( - os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") == "true" +ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = str_to_bool( + os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") ) RESCAN_ON_FILESYSTEM_CHANGE_DELAY: Final = int( os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5) # 5 minutes ) -ENABLE_SCHEDULED_RESCAN: Final = ( - os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") == "true" +ENABLE_SCHEDULED_RESCAN: Final = str_to_bool( + os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") ) SCHEDULED_RESCAN_CRON: Final = os.environ.get( "SCHEDULED_RESCAN_CRON", "0 3 * * *", # At 3:00 AM every day ) -ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = ( - os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true" +ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = str_to_bool( + os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") ) SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get( "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *", # At 4:00 AM every day ) +# EMULATION +DISABLE_EMULATOR_JS = str_to_bool(os.environ.get("DISABLE_EMULATOR_JS", "false")) +DISABLE_RUFFLE_RS = str_to_bool(os.environ.get("DISABLE_RUFFLE_RS", "false")) + # TESTING IS_PYTEST_RUN: Final = bool(os.environ.get("PYTEST_VERSION", False)) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 50339dedc..66c941bad 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -2,7 +2,6 @@ import sys from pathlib import Path from typing import Final -from urllib.parse import quote_plus import pydash import yaml @@ -21,6 +20,7 @@ ConfigNotWritableException, ) from logger.logger import log +from sqlalchemy import URL from yaml.loader import SafeLoader ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config" @@ -76,7 +76,7 @@ def __init__(self, config_file: str = ROMM_USER_CONFIG_FILE): sys.exit(5) @staticmethod - def get_db_engine() -> str: + def get_db_engine() -> URL: """Builds the database connection string depending on the defined database in the config.yml file Returns: @@ -90,9 +90,13 @@ def get_db_engine() -> str: ) sys.exit(3) - return ( - f"mariadb+mariadbconnector://{DB_USER}:%s@{DB_HOST}:{DB_PORT}/{DB_NAME}" - % quote_plus(DB_PASSWD) + return URL.create( + drivername="mariadb+mariadbconnector", + username=DB_USER, + password=DB_PASSWD, + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, ) # DEPRECATED diff --git a/backend/decorators/database.py b/backend/decorators/database.py index 230e6ae13..4e226b1f3 100644 --- a/backend/decorators/database.py +++ b/backend/decorators/database.py @@ -1,6 +1,7 @@ import functools from fastapi import HTTPException, status +from handler.database.base_handler import sync_session from logger.logger import log from sqlalchemy.exc import ProgrammingError @@ -12,7 +13,7 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) try: - with args[0].session.begin() as s: + with sync_session.begin() as s: kwargs["session"] = s return func(*args, **kwargs) except ProgrammingError as exc: diff --git a/backend/endpoints/feeds.py b/backend/endpoints/feeds.py index 561dc6589..ac761b98a 100644 --- a/backend/endpoints/feeds.py +++ b/backend/endpoints/feeds.py @@ -1,14 +1,22 @@ -from config import DISABLE_DOWNLOAD_ENDPOINT_AUTH, ROMM_HOST +from config import DISABLE_DOWNLOAD_ENDPOINT_AUTH from decorators.auth import protected_route from endpoints.responses.feeds import ( WEBRCADE_SLUG_TO_TYPE_MAP, WEBRCADE_SUPPORTED_PLATFORM_SLUGS, + TinfoilFeedFileSchema, TinfoilFeedSchema, + TinfoilFeedTitleDBSchema, + WebrcadeFeedCategorySchema, + WebrcadeFeedItemPropsSchema, + WebrcadeFeedItemSchema, WebrcadeFeedSchema, ) from fastapi import Request from handler.database import db_platform_handler, db_rom_handler +from handler.metadata import meta_igdb_handler +from handler.metadata.base_hander import SWITCH_TITLEDB_REGEX from models.rom import Rom +from starlette.datastructures import URLPath from utils.router import APIRouter router = APIRouter() @@ -21,6 +29,7 @@ ) def platforms_webrcade_feed(request: Request) -> WebrcadeFeedSchema: """Get webrcade feed endpoint + https://docs.webrcade.com/feeds/format/ Args: request (Request): Fastapi Request object @@ -31,41 +40,77 @@ def platforms_webrcade_feed(request: Request) -> WebrcadeFeedSchema: platforms = db_platform_handler.get_platforms() - return { - "title": "RomM Feed", - "longTitle": "Custom RomM Feed", - "description": "Custom feed from your RomM library", - "thumbnail": "https://raw.githubusercontent.com/rommapp/romm/f2dd425d87ad8e21bf47f8258ae5dcf90f56fbc2/frontend/assets/isotipo.svg", - "background": "https://raw.githubusercontent.com/rommapp/romm/release/.github/screenshots/gallery.png", - "categories": [ - { - "title": p.name, - "longTitle": f"{p.name} Games", - "background": f"{ROMM_HOST}/assets/webrcade/feed/{p.slug.lower()}-background.png", - "thumbnail": f"{ROMM_HOST}/assets/webrcade/feed/{p.slug.lower()}-thumb.png", - "description": "", - "items": [ - { - "title": rom.name, - "description": rom.summary, - "type": WEBRCADE_SLUG_TO_TYPE_MAP.get(p.slug, p.slug), - "thumbnail": f"{ROMM_HOST}/assets/romm/resources/{rom.path_cover_s}", - "background": f"{ROMM_HOST}/assets/romm/resources/{rom.path_cover_l}", - "props": { - "rom": f"{ROMM_HOST}/api/roms/{rom.id}/content/{rom.file_name}" - }, - } - for rom in db_rom_handler.get_roms(platform_id=p.id) - ], - } - for p in platforms - if p.slug in WEBRCADE_SUPPORTED_PLATFORM_SLUGS - ], - } + categories = [] + for p in platforms: + if p.slug not in WEBRCADE_SUPPORTED_PLATFORM_SLUGS: + continue + + category_items = [] + for rom in db_rom_handler.get_roms(platform_id=p.id): + category_item = WebrcadeFeedItemSchema( + title=rom.name or "", + description=rom.summary or "", + type=WEBRCADE_SLUG_TO_TYPE_MAP.get(p.slug, p.slug), + props=WebrcadeFeedItemPropsSchema( + rom=str( + request.url_for( + "get_rom_content", + id=rom.id, + file_name=rom.file_name, + ) + ), + ), + ) + if rom.path_cover_s: + category_item["thumbnail"] = str( + URLPath( + f"/assets/romm/resources/{rom.path_cover_s}" + ).make_absolute_url(request.base_url) + ) + if rom.path_cover_l: + category_item["background"] = str( + URLPath( + f"/assets/romm/resources/{rom.path_cover_l}" + ).make_absolute_url(request.base_url) + ) + category_items.append(category_item) + + categories.append( + WebrcadeFeedCategorySchema( + title=p.name, + longTitle=f"{p.name} Games", + background=str( + URLPath( + f"/assets/webrcade/feed/{p.slug.lower()}-background.png" + ).make_absolute_url(request.base_url) + ), + thumbnail=str( + URLPath( + f"/assets/webrcade/feed/{p.slug.lower()}-thumb.png" + ).make_absolute_url(request.base_url) + ), + items=category_items, + ) + ) + return WebrcadeFeedSchema( + title="RomM Feed", + longTitle="Custom RomM Feed", + description="Custom feed from your RomM library", + thumbnail="https://raw.githubusercontent.com/rommapp/romm/f2dd425d87ad8e21bf47f8258ae5dcf90f56fbc2/frontend/assets/isotipo.svg", + background="https://raw.githubusercontent.com/rommapp/romm/release/.github/resources/screenshots/gallery.png", + categories=categories, + ) -@protected_route(router.get, "/tinfoil/feed", ["roms.read"]) -def tinfoil_index_feed(request: Request, slug: str = "switch") -> TinfoilFeedSchema: + +@protected_route( + router.get, + "/tinfoil/feed", + [], +) +async def tinfoil_index_feed( + request: Request, slug: str = "switch" +) -> TinfoilFeedSchema: """Get tinfoil custom index feed endpoint https://blawar.github.io/tinfoil/custom_index/ @@ -77,16 +122,52 @@ def tinfoil_index_feed(request: Request, slug: str = "switch") -> TinfoilFeedSch TinfoilFeedSchema: Tinfoil feed object schema """ switch = db_platform_handler.get_platform_by_fs_slug(slug) - files: list[Rom] = db_rom_handler.get_roms(platform_id=switch.id) - - return { - "files": [ - { - "url": f"{ROMM_HOST}/api/roms/{file.id}/content/{file.file_name}", - "size": file.file_size_bytes, - } - for file in files + if not switch: + return TinfoilFeedSchema( + files=[], + directories=[], + error="Nintendo Switch platform not found", + ) + + roms: list[Rom] = db_rom_handler.get_roms(platform_id=switch.id) + + async def extract_titledb(roms: list[Rom]) -> dict[str, TinfoilFeedTitleDBSchema]: + titledb = {} + for rom in roms: + match = SWITCH_TITLEDB_REGEX.search(rom.file_name) + if match: + _search_term, index_entry = ( + await meta_igdb_handler._switch_titledb_format(match, rom.file_name) + ) + if index_entry: + titledb[str(index_entry["nsuId"])] = TinfoilFeedTitleDBSchema( + id=str(index_entry["nsuId"]), + name=index_entry["name"], + description=index_entry["description"], + size=index_entry["size"], + version=index_entry["version"] or 0, + region=index_entry["region"] or "US", + releaseDate=index_entry["releaseDate"] or 19700101, + rating=index_entry["rating"] or 0, + publisher=index_entry["publisher"] or "", + rank=0, + ) + + return titledb + + return TinfoilFeedSchema( + files=[ + TinfoilFeedFileSchema( + url=str( + request.url_for( + "get_rom_content", id=rom.id, file_name=rom.file_name + ) + ), + size=rom.file_size_bytes, + ) + for rom in roms ], - "directories": [], - "success": "RomM Switch Library", - } + directories=[], + success="RomM Switch Library", + titledb=await extract_titledb(roms), + ) diff --git a/backend/endpoints/heartbeat.py b/backend/endpoints/heartbeat.py index 7d1b37b26..e6e81f805 100644 --- a/backend/endpoints/heartbeat.py +++ b/backend/endpoints/heartbeat.py @@ -1,4 +1,6 @@ from config import ( + DISABLE_EMULATOR_JS, + DISABLE_RUFFLE_RS, ENABLE_RESCAN_ON_FILESYSTEM_CHANGE, ENABLE_SCHEDULED_RESCAN, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, @@ -55,4 +57,8 @@ def heartbeat() -> HeartbeatResponse: "MESSAGE": "Updates the Nintendo Switch TitleDB file", }, }, + "EMULATION": { + "DISABLE_EMULATOR_JS": DISABLE_EMULATOR_JS, + "DISABLE_RUFFLE_RS": DISABLE_RUFFLE_RS, + }, } diff --git a/backend/endpoints/responses/__init__.py b/backend/endpoints/responses/__init__.py index 29e7f685f..94a0a4bf4 100644 --- a/backend/endpoints/responses/__init__.py +++ b/backend/endpoints/responses/__init__.py @@ -1,4 +1,4 @@ -from typing_extensions import TypedDict +from typing import TypedDict class MessageResponse(TypedDict): diff --git a/backend/endpoints/responses/assets.py b/backend/endpoints/responses/assets.py index e274862c7..def6a753d 100644 --- a/backend/endpoints/responses/assets.py +++ b/backend/endpoints/responses/assets.py @@ -1,7 +1,7 @@ from datetime import datetime +from typing import TypedDict from pydantic import BaseModel -from typing_extensions import TypedDict class BaseAsset(BaseModel): diff --git a/backend/endpoints/responses/config.py b/backend/endpoints/responses/config.py index 4ff222c9d..4420a9354 100644 --- a/backend/endpoints/responses/config.py +++ b/backend/endpoints/responses/config.py @@ -1,4 +1,4 @@ -from typing_extensions import TypedDict +from typing import TypedDict class ConfigResponse(TypedDict): diff --git a/backend/endpoints/responses/feeds.py b/backend/endpoints/responses/feeds.py index c87c14017..00fc39b2e 100644 --- a/backend/endpoints/responses/feeds.py +++ b/backend/endpoints/responses/feeds.py @@ -1,38 +1,40 @@ -from typing_extensions import TypedDict - -WEBRCADE_SUPPORTED_PLATFORM_SLUGS = [ - "3do", - "arcade", - "atari2600", - "atari5200", - "atari7800", - "lynx", - "wonderswan", - "wonderswan-color", - "colecovision", - "turbografx16--1", - "turbografx-16-slash-pc-engine-cd", - "supergrafx", - "pc-fx", - "nes", - "n64", - "snes", - "gb", - "gba", - "gbc", - "virtualboy", - "sg1000", - "sms", - "genesis-slash-megadrive", - "segacd", - "gamegear", - "neo-geo-cd", - "neogeoaes", - "neogeomvs", - "neo-geo-pocket", - "neo-geo-pocket-color", - "ps", -] +from typing import NotRequired, TypedDict + +WEBRCADE_SUPPORTED_PLATFORM_SLUGS = frozenset( + ( + "3do", + "arcade", + "atari2600", + "atari5200", + "atari7800", + "colecovision", + "gamegear", + "gb", + "gba", + "gbc", + "genesis-slash-megadrive", + "lynx", + "n64", + "neo-geo-cd", + "neo-geo-pocket", + "neo-geo-pocket-color", + "neogeoaes", + "neogeomvs", + "nes", + "pc-fx", + "ps", + "segacd", + "sg1000", + "sms", + "snes", + "supergrafx", + "turbografx-16-slash-pc-engine-cd", + "turbografx16--1", + "virtualboy", + "wonderswan", + "wonderswan-color", + ) +) WEBRCADE_SLUG_TO_TYPE_MAP = { "atari2600": "2600", @@ -55,16 +57,67 @@ } +# Webrcade feed format +# Source: https://docs.webrcade.com/feeds/format/ + + +class WebrcadeFeedItemPropsSchema(TypedDict): + rom: str + + +class WebrcadeFeedItemSchema(TypedDict): + title: str + longTitle: NotRequired[str] + description: NotRequired[str] + type: str + thumbnail: NotRequired[str] + background: NotRequired[str] + props: WebrcadeFeedItemPropsSchema + + +class WebrcadeFeedCategorySchema(TypedDict): + title: str + longTitle: NotRequired[str] + background: NotRequired[str] + thumbnail: NotRequired[str] + description: NotRequired[str] + items: list[WebrcadeFeedItemSchema] + + class WebrcadeFeedSchema(TypedDict): title: str - longTitle: str + longTitle: NotRequired[str] + description: NotRequired[str] + thumbnail: NotRequired[str] + background: NotRequired[str] + categories: list[WebrcadeFeedCategorySchema] + + +# Tinfoil feed format +# Source: https://blawar.github.io/tinfoil/custom_index/ + + +class TinfoilFeedFileSchema(TypedDict): + url: str + size: int + + +class TinfoilFeedTitleDBSchema(TypedDict): + id: str + name: str + version: int + region: str + releaseDate: int + rating: int + publisher: str description: str - thumbnail: str - background: str - categories: list[dict] + size: int + rank: int class TinfoilFeedSchema(TypedDict): - files: list[dict] + files: list[TinfoilFeedFileSchema] directories: list[str] - success: str + titledb: NotRequired[dict[str, TinfoilFeedTitleDBSchema]] + success: NotRequired[str] + error: NotRequired[str] diff --git a/backend/endpoints/responses/firmware.py b/backend/endpoints/responses/firmware.py index 8faa3a306..b4a19cd60 100644 --- a/backend/endpoints/responses/firmware.py +++ b/backend/endpoints/responses/firmware.py @@ -1,7 +1,7 @@ from datetime import datetime +from typing import TypedDict from pydantic import BaseModel -from typing_extensions import TypedDict class FirmwareSchema(BaseModel): diff --git a/backend/endpoints/responses/heartbeat.py b/backend/endpoints/responses/heartbeat.py index ada1e424d..6a995dc16 100644 --- a/backend/endpoints/responses/heartbeat.py +++ b/backend/endpoints/responses/heartbeat.py @@ -1,4 +1,4 @@ -from typing_extensions import TypedDict +from typing import TypedDict class WatcherDict(TypedDict): @@ -22,6 +22,11 @@ class MetadataSourcesDict(TypedDict): STEAMGRIDDB_ENABLED: bool +class EmulationDict(TypedDict): + DISABLE_EMULATOR_JS: bool + DISABLE_RUFFLE_RS: bool + + class HeartbeatResponse(TypedDict): VERSION: str SHOW_SETUP_WIZARD: bool @@ -30,3 +35,4 @@ class HeartbeatResponse(TypedDict): ANY_SOURCE_ENABLED: bool METADATA_SOURCES: MetadataSourcesDict FS_PLATFORMS: list + EMULATION: EmulationDict diff --git a/backend/endpoints/responses/oauth.py b/backend/endpoints/responses/oauth.py index f9b126875..f70023a9b 100644 --- a/backend/endpoints/responses/oauth.py +++ b/backend/endpoints/responses/oauth.py @@ -1,6 +1,4 @@ -from typing import NotRequired - -from typing_extensions import TypedDict +from typing import NotRequired, TypedDict class TokenResponse(TypedDict): diff --git a/backend/endpoints/responses/rom.py b/backend/endpoints/responses/rom.py index 202d23707..200a9fef8 100644 --- a/backend/endpoints/responses/rom.py +++ b/backend/endpoints/responses/rom.py @@ -2,18 +2,15 @@ import re from datetime import datetime -from typing import NotRequired, get_type_hints +from typing import NotRequired, TypedDict, get_type_hints from endpoints.responses.assets import SaveSchema, ScreenshotSchema, StateSchema from endpoints.responses.collection import CollectionSchema from fastapi import Request -from fastapi.responses import StreamingResponse from handler.metadata.igdb_handler import IGDBMetadata from handler.metadata.moby_handler import MobyMetadata -from handler.socket_handler import socket_handler -from models.rom import Rom +from models.rom import Rom, RomFile from pydantic import BaseModel, Field, computed_field -from typing_extensions import TypedDict SORT_COMPARE_REGEX = re.compile(r"^([Tt]he|[Aa]|[Aa]nd)\s") @@ -108,29 +105,17 @@ class RomSchema(BaseModel): tags: list[str] multi: bool - files: list[str] + files: list[RomFile] + crc_hash: str | None + md5_hash: str | None + sha1_hash: str | None full_path: str created_at: datetime updated_at: datetime - rom_user: RomUserSchema | None = Field(default=None) - sibling_roms: list[RomSchema] = Field(default_factory=list) - class Config: from_attributes = True - @classmethod - def from_orm_with_request(cls, db_rom: Rom, request: Request) -> RomSchema: - rom = cls.model_validate(db_rom) - user_id = request.user.id - - rom.rom_user = RomUserSchema.for_user(user_id, db_rom) - rom.sibling_roms = [ - RomSchema.model_validate(r) for r in db_rom.get_sibling_roms() - ] - - return rom - @computed_field # type: ignore @property def sort_comparator(self) -> str: @@ -144,10 +129,24 @@ def sort_comparator(self) -> str: ) +class SimpleRomSchema(RomSchema): + sibling_roms: list[RomSchema] = Field(default_factory=list) + rom_user: RomUserSchema | None = Field(default=None) + + @classmethod + def from_orm_with_request(cls, db_rom: Rom, request: Request) -> SimpleRomSchema: + rom = cls.model_validate(db_rom) + user_id = request.user.id + + rom.rom_user = RomUserSchema.for_user(user_id, db_rom) + + return rom + + class DetailedRomSchema(RomSchema): merged_screenshots: list[str] - rom_user: RomUserSchema | None = Field(default=None) sibling_roms: list[RomSchema] = Field(default_factory=list) + rom_user: RomUserSchema | None = Field(default=None) user_saves: list[SaveSchema] = Field(default_factory=list) user_states: list[StateSchema] = Field(default_factory=list) user_screenshots: list[ScreenshotSchema] = Field(default_factory=list) @@ -161,9 +160,6 @@ def from_orm_with_request(cls, db_rom: Rom, request: Request) -> DetailedRomSche rom.rom_user = RomUserSchema.for_user(user_id, db_rom) rom.user_notes = RomUserSchema.notes_for_user(user_id, db_rom) - rom.sibling_roms = [ - RomSchema.model_validate(r) for r in db_rom.get_sibling_roms() - ] rom.user_saves = [ SaveSchema.model_validate(s) for s in db_rom.saves if s.user_id == user_id ] @@ -186,18 +182,3 @@ class UserNotesSchema(TypedDict): user_id: int username: str note_raw_markdown: str - - -class AddRomsResponse(TypedDict): - uploaded_roms: list[str] - skipped_roms: list[str] - - -class CustomStreamingResponse(StreamingResponse): - def __init__(self, *args, **kwargs) -> None: - self.emit_body = kwargs.pop("emit_body", None) - super().__init__(*args, **kwargs) - - async def stream_response(self, *args, **kwargs) -> None: - await super().stream_response(*args, **kwargs) - await socket_handler.socket_server.emit("download:complete", self.emit_body) diff --git a/backend/endpoints/responses/stats.py b/backend/endpoints/responses/stats.py index ee199ecaf..c7b4229be 100644 --- a/backend/endpoints/responses/stats.py +++ b/backend/endpoints/responses/stats.py @@ -1,4 +1,4 @@ -from typing_extensions import TypedDict +from typing import TypedDict class StatsReturn(TypedDict): diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 5fe7dd126..0168bc730 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -1,7 +1,6 @@ -from collections.abc import AsyncIterator -from datetime import datetime +import binascii +from base64 import b64encode from shutil import rmtree -from stat import S_IFREG from typing import Annotated from urllib.parse import quote @@ -13,84 +12,87 @@ ) from decorators.auth import protected_route from endpoints.responses import MessageResponse -from endpoints.responses.rom import ( - AddRomsResponse, - CustomStreamingResponse, - DetailedRomSchema, - RomSchema, - RomUserSchema, -) +from endpoints.responses.rom import DetailedRomSchema, RomUserSchema, SimpleRomSchema from exceptions.endpoint_exceptions import RomNotFoundInDatabaseException from exceptions.fs_exceptions import RomAlreadyExistsException -from fastapi import File, HTTPException, Query, Request, UploadFile, status -from fastapi.responses import FileResponse +from fastapi import HTTPException, Query, Request, UploadFile, status +from fastapi.responses import Response from handler.database import db_platform_handler, db_rom_handler from handler.filesystem import fs_resource_handler, fs_rom_handler from handler.filesystem.base_handler import CoverSize from handler.metadata import meta_igdb_handler, meta_moby_handler from logger.logger import log -from stream_zip import NO_COMPRESSION_32, ZIP_AUTO, AsyncMemberFile, async_stream_zip +from starlette.requests import ClientDisconnect +from streaming_form_data import StreamingFormDataParser +from streaming_form_data.targets import FileTarget, NullTarget +from utils.filesystem import sanitize_filename +from utils.hashing import crc32_to_hex +from utils.nginx import ZipContentLine, ZipResponse from utils.router import APIRouter router = APIRouter() @protected_route(router.post, "/roms", ["roms.write"]) -async def add_roms( - request: Request, - platform_id: int, - roms: list[UploadFile] = File(...), # noqa: B008 -) -> AddRomsResponse: - """Upload roms endpoint (one or more at the same time) +async def add_rom(request: Request): + """Upload single rom endpoint Args: request (Request): Fastapi Request object - platform_slug (str): Slug of the platform where to upload the roms - roms (list[UploadFile], optional): List of files to upload. Defaults to File(...). Raises: HTTPException: No files were uploaded - - Returns: - AddRomsResponse: Standard message response """ + platform_id = request.headers.get("x-upload-platform") + filename = request.headers.get("x-upload-filename") + if not platform_id or not filename: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No platform ID or filename provided", + ) from None - platform_fs_slug = db_platform_handler.get_platform(platform_id).fs_slug - log.info(f"Uploading roms to {platform_fs_slug}") - if roms is None: - log.error("No roms were uploaded") + db_platform = db_platform_handler.get_platform(int(platform_id)) + if not db_platform: raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="No roms were uploaded", - ) + status_code=status.HTTP_400_BAD_REQUEST, + detail="Platform not found", + ) from None + platform_fs_slug = db_platform.fs_slug roms_path = fs_rom_handler.build_upload_file_path(platform_fs_slug) + log.info(f"Uploading file to {platform_fs_slug}") - uploaded_roms = [] - skipped_roms = [] + file_location = Path(f"{roms_path}/{filename}") + parser = StreamingFormDataParser(headers=request.headers) + parser.register("x-upload-platform", NullTarget()) + parser.register(filename, FileTarget(str(file_location))) - for rom in roms: - if fs_rom_handler.file_exists(roms_path, rom.filename): - log.warning(f" - Skipping {rom.filename} since the file already exists") - skipped_roms.append(rom.filename) - continue - - log.info(f" - Uploading {rom.filename}") - file_location = f"{roms_path}/{rom.filename}" + if await file_location.exists(): + log.warning(f" - Skipping {filename} since the file already exists") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"File {filename} already exists", + ) from None - async with await open_file(file_location, "wb+") as f: - while True: - chunk = rom.file.read(1024) - if not chunk: - break - await f.write(chunk) + async def cleanup_partial_file(): + if await file_location.exists(): + await file_location.unlink() - uploaded_roms.append(rom.filename) + try: + async for chunk in request.stream(): + parser.data_received(chunk) + except ClientDisconnect: + log.error("Client disconnected during upload") + await cleanup_partial_file() + except Exception as exc: + log.error("Error uploading files", exc_info=exc) + await cleanup_partial_file() + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="There was an error uploading the file(s)", + ) from exc - return { - "uploaded_roms": uploaded_roms, - "skipped_roms": skipped_roms, - } + return Response(status_code=status.HTTP_201_CREATED) @protected_route(router.get, "/roms", ["roms.read"]) @@ -102,7 +104,7 @@ def get_roms( limit: int | None = None, order_by: str = "name", order_dir: str = "asc", -) -> list[RomSchema]: +) -> list[SimpleRomSchema]: """Get roms endpoint Args: @@ -110,7 +112,7 @@ def get_roms( id (int, optional): Rom internal id Returns: - list[RomSchema]: List of roms stored in the database + list[SimpleRomSchema]: List of roms stored in the database """ roms = db_rom_handler.get_roms( @@ -122,7 +124,7 @@ def get_roms( limit=limit, ) - return [RomSchema.from_orm_with_request(rom, request) for rom in roms] + return [SimpleRomSchema.from_orm_with_request(rom, request) for rom in roms] @protected_route( @@ -154,7 +156,12 @@ def get_rom(request: Request, id: int) -> DetailedRomSchema: "/roms/{id}/content/{file_name}", [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else ["roms.read"], ) -def head_rom_content(request: Request, id: int, file_name: str): +async def head_rom_content( + request: Request, + id: int, + file_name: str, + files: Annotated[list[str] | None, Query()] = None, +): """Head rom content endpoint Args: @@ -171,20 +178,39 @@ def head_rom_content(request: Request, id: int, file_name: str): if not rom: raise RomNotFoundInDatabaseException(id) - rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}" + files_to_check = files or [r["filename"] for r in rom.files] - return FileResponse( - path=rom_path if not rom.multi else f"{rom_path}/{rom.files[0]}", - filename=file_name, + if not rom.multi: + return Response( + media_type="application/octet-stream", + headers={ + "Content-Disposition": f'attachment; filename="{quote(rom.file_name)}"', + "X-Accel-Redirect": f"/library/{rom.full_path}", + }, + ) + + if len(files_to_check) == 1: + return Response( + media_type="application/octet-stream", + headers={ + "Content-Disposition": f'attachment; filename="{quote(files_to_check[0])}"', + "X-Accel-Redirect": f"/library/{rom.full_path}/{files_to_check[0]}", + }, + ) + + return Response( + media_type="application/zip", headers={ - "Content-Disposition": f'attachment; filename="{quote(rom.name)}.zip"', - "Content-Type": "application/zip", - "Content-Length": str(rom.file_size_bytes), + "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"', }, ) -@protected_route(router.get, "/roms/{id}/content/{file_name}", ["roms.read"]) +@protected_route( + router.get, + "/roms/{id}/content/{file_name}", + [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else ["roms.read"], +) async def get_rom_content( request: Request, id: int, @@ -202,7 +228,7 @@ async def get_rom_content( FileResponse: Returns one file for single file roms Yields: - CustomStreamingResponse: Streams a file for multi-part roms + ZipResponse: Returns a response for nginx to serve a Zip file for multi-part roms """ rom = db_rom_handler.get_rom(id) @@ -211,61 +237,49 @@ async def get_rom_content( raise RomNotFoundInDatabaseException(id) rom_path = f"{LIBRARY_BASE_PATH}/{rom.full_path}" - files_to_download = files or rom.files or [] + files_to_download = sorted(files or [r["filename"] for r in rom.files]) if not rom.multi: - return FileResponse(path=rom_path, filename=rom.file_name) + return Response( + media_type="application/octet-stream", + headers={ + "Content-Disposition": f'attachment; filename="{quote(rom.file_name)}"', + "X-Accel-Redirect": f"/library/{rom.full_path}", + }, + ) if len(files_to_download) == 1: - return FileResponse( - path=f"{rom_path}/{files_to_download[0]}", filename=files_to_download[0] + return Response( + media_type="application/octet-stream", + headers={ + "Content-Disposition": f'attachment; filename="{quote(files_to_download[0])}"', + "X-Accel-Redirect": f"/library/{rom.full_path}/{files_to_download[0]}", + }, ) - # Builds a generator of tuples for each member file - async def local_files() -> AsyncIterator[AsyncMemberFile]: - async def contents(filename: str) -> AsyncIterator[bytes]: - try: - async with await open_file(f"{rom_path}/{filename}", "rb") as f: - while chunk := await f.read(65536): - yield chunk - except FileNotFoundError: - log.error(f"File {rom_path}/{filename} not found!") - raise - - async def m3u_file() -> AsyncIterator[bytes]: - for file in files_to_download: - yield str.encode(f"{file}\n") - - now = datetime.now() - - for f in files_to_download: - file_size = (await Path(f"{rom_path}/{f}").stat()).st_size - yield ( - f, - now, - S_IFREG | 0o600, - ZIP_AUTO(file_size, level=0), - contents(f), - ) - - yield ( - f"{file_name}.m3u", - now, - S_IFREG | 0o600, - NO_COMPRESSION_32, - m3u_file(), + content_lines = [ + ZipContentLine( + # TODO: Use calculated CRC-32 if available. + crc32=None, + size_bytes=(await Path(f"{rom_path}/{f}").stat()).st_size, + encoded_location=quote(f"/library-zip/{rom.full_path}/{f}"), + filename=f, ) + for f in files_to_download + ] + + m3u_encoded_content = "\n".join([f for f in files_to_download]).encode() + m3u_base64_content = b64encode(m3u_encoded_content).decode() + m3u_line = ZipContentLine( + crc32=crc32_to_hex(binascii.crc32(m3u_encoded_content)), + size_bytes=len(m3u_encoded_content), + encoded_location=f"/decode?value={m3u_base64_content}", + filename=f"{file_name}.m3u", + ) - zipped_chunks = async_stream_zip(local_files()) - - # Streams the zip file to the client - return CustomStreamingResponse( - zipped_chunks, - media_type="application/zip", - headers={ - "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"' - }, - emit_body={"id": rom.id}, + return ZipResponse( + content_lines=content_lines + [m3u_line], + filename=f"{quote(file_name)}.zip", ) @@ -276,6 +290,7 @@ async def update_rom( rename_as_source: bool = False, remove_cover: bool = False, artwork: UploadFile | None = None, + unmatch_metadata: bool = False, ) -> DetailedRomSchema: """Update rom endpoint @@ -284,6 +299,7 @@ async def update_rom( id (Rom): Rom internal id rename_as_source (bool, optional): Flag to rename rom file as matched IGDB game. Defaults to False. artwork (UploadFile, optional): Custom artork to set as cover. Defaults to File(None). + unmatch_metadata: Remove the metadata matches for this game. Defaults to False. Raises: HTTPException: If a rom already have that name when enabling the rename_as_source flag @@ -299,6 +315,31 @@ async def update_rom( if not rom: raise RomNotFoundInDatabaseException(id) + if unmatch_metadata: + db_rom_handler.update_rom( + id, + { + "igdb_id": None, + "sgdb_id": None, + "moby_id": None, + "name": rom.file_name, + "summary": "", + "url_screenshots": [], + "path_screenshots": [], + "path_cover_s": "", + "path_cover_l": "", + "url_cover": "", + "slug": "", + "igdb_metadata": {}, + "moby_metadata": {}, + "revision": "", + }, + ) + + return DetailedRomSchema.from_orm_with_request( + db_rom_handler.get_rom(id), request + ) + cleaned_data = { "igdb_id": data.get("igdb_id", None), "moby_id": data.get("moby_id", None), @@ -335,19 +376,25 @@ async def update_rom( } ) - fs_safe_file_name = data.get("file_name", rom.file_name).strip().replace("/", "-") - fs_safe_name = cleaned_data["name"].strip().replace("/", "-") - - if rename_as_source: - fs_safe_file_name = rom.file_name.replace( - rom.file_name_no_tags or rom.file_name_no_ext, fs_safe_name - ) + new_file_name = data.get("file_name", rom.file_name) try: - if rom.file_name != fs_safe_file_name: + if rename_as_source: + new_file_name = rom.file_name.replace( + rom.file_name_no_tags or rom.file_name_no_ext, + data.get("name", rom.name), + ) + new_file_name = sanitize_filename(new_file_name) + fs_rom_handler.rename_file( + old_name=rom.file_name, + new_name=new_file_name, + file_path=rom.file_path, + ) + elif rom.file_name != new_file_name: + new_file_name = sanitize_filename(new_file_name) fs_rom_handler.rename_file( old_name=rom.file_name, - new_name=fs_safe_file_name, + new_name=new_file_name, file_path=rom.file_path, ) except RomAlreadyExistsException as exc: @@ -358,12 +405,12 @@ async def update_rom( cleaned_data.update( { - "file_name": fs_safe_file_name, + "file_name": new_file_name, "file_name_no_tags": fs_rom_handler.get_file_name_with_no_tags( - fs_safe_file_name + new_file_name ), "file_name_no_ext": fs_rom_handler.get_file_name_with_no_extension( - fs_safe_file_name + new_file_name ), } ) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index a8c3c062c..2af19497c 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -7,7 +7,7 @@ import socketio # type: ignore from config import SCAN_TIMEOUT from endpoints.responses.platform import PlatformSchema -from endpoints.responses.rom import RomSchema +from endpoints.responses.rom import SimpleRomSchema from exceptions.fs_exceptions import ( FirmwareNotFoundException, FolderStructureNotMatchException, @@ -21,6 +21,7 @@ fs_resource_handler, fs_rom_handler, ) +from handler.filesystem.roms_handler import FSRom from handler.metadata.igdb_handler import IGDB_API_ENABLED from handler.metadata.moby_handler import MOBY_API_ENABLED from handler.redis_handler import high_prio_queue, redis_client, redis_url @@ -81,6 +82,7 @@ def _should_scan_rom(scan_type: ScanType, rom: Rom, roms_ids: list): return ( (scan_type in {ScanType.NEW_PLATFORMS, ScanType.QUICK} and not rom) or (scan_type == ScanType.COMPLETE) + or (scan_type == ScanType.HASHES) or ( rom and ( @@ -224,7 +226,7 @@ async def _identify_platform( # Scanning firmware try: - fs_firmware = fs_firmware_handler.get_firmware(platform) + fs_firmware = fs_firmware_handler.get_firmware(platform.fs_slug) except FirmwareNotFoundException: fs_firmware = [] @@ -241,7 +243,7 @@ async def _identify_platform( # Scanning roms try: - fs_roms = fs_rom_handler.get_roms(platform) + fs_roms = fs_rom_handler.get_roms(platform.fs_slug) except RomsNotFoundException as e: log.error(e) return scan_stats @@ -302,7 +304,7 @@ async def _identify_firmware( async def _identify_rom( platform: Platform, - fs_rom: dict, + fs_rom: FSRom, scan_type: ScanType, roms_ids: list[str], metadata_sources: list[str], @@ -317,11 +319,17 @@ async def _identify_rom( rom = db_rom_handler.get_rom_by_filename(platform.id, fs_rom["file_name"]) if not _should_scan_rom(scan_type=scan_type, rom=rom, roms_ids=roms_ids): + # Just to update the filesystem data + rom.file_name = fs_rom["file_name"] + rom.multi = fs_rom["multi"] + rom.files = fs_rom["files"] + db_rom_handler.add_rom(rom) + return scan_stats scanned_rom = await scan_rom( platform=platform, - rom_attrs=fs_rom, + fs_rom=fs_rom, scan_type=scan_type, rom=rom, metadata_sources=metadata_sources, @@ -333,6 +341,10 @@ async def _identify_rom( _added_rom = db_rom_handler.add_rom(scanned_rom) + # Return early if we're only scanning for hashes + if scan_type == ScanType.HASHES: + return scan_stats + path_cover_s, path_cover_l = await fs_resource_handler.get_cover( overwrite=True, entity=_added_rom, @@ -361,7 +373,7 @@ async def _identify_rom( { "platform_name": platform.name, "platform_slug": platform.slug, - **RomSchema.model_validate(_added_rom).model_dump( + **SimpleRomSchema.model_validate(_added_rom).model_dump( exclude={"created_at", "updated_at", "rom_user"} ), }, @@ -386,6 +398,14 @@ async def scan_handler(_sid: str, options: dict): roms_ids = options.get("roms_ids", []) metadata_sources = options.get("apis", []) + # Uncomment this to run scan in the current process + # await scan_platforms( + # platform_ids=platform_ids, + # scan_type=scan_type, + # roms_ids=roms_ids, + # metadata_sources=metadata_sources, + # ) + return high_prio_queue.enqueue( scan_platforms, platform_ids, diff --git a/backend/handler/database/base_handler.py b/backend/handler/database/base_handler.py index 6d1d44e31..6d3fda8ba 100644 --- a/backend/handler/database/base_handler.py +++ b/backend/handler/database/base_handler.py @@ -2,8 +2,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +sync_engine = create_engine(ConfigManager.get_db_engine(), pool_pre_ping=True) +sync_session = sessionmaker(bind=sync_engine, expire_on_commit=False) -class DBBaseHandler: - def __init__(self) -> None: - self.engine = create_engine(ConfigManager.get_db_engine(), pool_pre_ping=True) - self.session = sessionmaker(bind=self.engine, expire_on_commit=False) + +class DBBaseHandler: ... diff --git a/backend/handler/database/platforms_handler.py b/backend/handler/database/platforms_handler.py index f2c0aee5f..de3976fa1 100644 --- a/backend/handler/database/platforms_handler.py +++ b/backend/handler/database/platforms_handler.py @@ -1,49 +1,36 @@ -import functools - from decorators.database import begin_session from models.platform import Platform from models.rom import Rom from sqlalchemy import Select, delete, or_, select -from sqlalchemy.orm import Query, Session, selectinload +from sqlalchemy.orm import Session from .base_handler import DBBaseHandler -def with_roms(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - session = kwargs.get("session") - if session is None: - raise ValueError("session is required") - - kwargs["query"] = select(Platform).options( - selectinload(Platform.roms).load_only(Rom.id) - ) - return func(*args, **kwargs) - - return wrapper - - class DBPlatformsHandler(DBBaseHandler): @begin_session - @with_roms def add_platform( - self, platform: Platform, query: Query = None, session: Session = None + self, + platform: Platform, + session: Session, ) -> Platform: platform = session.merge(platform) session.flush() - return session.scalar(query.filter_by(id=platform.id).limit(1)) + new_platform = session.scalar( + select(Platform).filter_by(id=platform.id).limit(1) + ) + if not new_platform: + raise ValueError("Could not find newlyewly created platform") + + return new_platform @begin_session - @with_roms - def get_platform( - self, id: int, *, query: Query = None, session: Session = None - ) -> Platform | None: - return session.scalar(query.filter_by(id=id).limit(1)) + def get_platform(self, id: int, *, session: Session) -> Platform | None: + return session.scalar(select(Platform).filter_by(id=id).limit(1)) @begin_session - def get_platforms(self, *, session: Session = None) -> Select[tuple[Platform]]: + def get_platforms(self, *, session: Session) -> Select[tuple[Platform]]: return ( session.scalars(select(Platform).order_by(Platform.name.asc())) # type: ignore[attr-defined] .unique() @@ -51,28 +38,28 @@ def get_platforms(self, *, session: Session = None) -> Select[tuple[Platform]]: ) @begin_session - @with_roms def get_platform_by_fs_slug( - self, fs_slug: str, query: Query = None, session: Session = None + self, fs_slug: str, session: Session ) -> Platform | None: - return session.scalar(query.filter_by(fs_slug=fs_slug).limit(1)) + return session.scalar(select(Platform).filter_by(fs_slug=fs_slug).limit(1)) @begin_session - def delete_platform(self, id: int, session: Session = None) -> int: + def delete_platform(self, id: int, session: Session) -> None: # Remove all roms from that platforms first session.execute( delete(Rom) .where(Rom.platform_id == id) .execution_options(synchronize_session="evaluate") ) - return session.execute( + + session.execute( delete(Platform) .where(Platform.id == id) .execution_options(synchronize_session="evaluate") ) @begin_session - def purge_platforms(self, fs_platforms: list[str], session: Session = None) -> int: + def purge_platforms(self, fs_platforms: list[str], session: Session) -> int: return session.execute( delete(Platform) .where(or_(Platform.fs_slug.not_in(fs_platforms), Platform.slug.is_(None))) # type: ignore[attr-defined] diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index f9acc12a0..d15d92e3c 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -23,6 +23,7 @@ def wrapper(*args, **kwargs): selectinload(Rom.states), selectinload(Rom.screenshots), selectinload(Rom.rom_users), + selectinload(Rom.sibling_roms), ) return func(*args, **kwargs) @@ -38,7 +39,9 @@ def wrapper(*args, **kwargs): f"{func} is missing required kwarg 'session' with type 'Session'" ) - kwargs["query"] = select(Rom).options(selectinload(Rom.rom_users)) + kwargs["query"] = select(Rom).options( + selectinload(Rom.rom_users), selectinload(Rom.sibling_roms) + ) return func(*args, **kwargs) return wrapper @@ -153,32 +156,6 @@ def get_rom_by_filename_no_ext( query.filter_by(file_name_no_ext=file_name_no_ext).limit(1) ) - @begin_session - @with_simple - def get_sibling_roms( - self, rom: Rom, query: Query = None, session: Session = None - ) -> list[Rom]: - return session.scalars( - query.where( - and_( - Rom.platform_id == rom.platform_id, - Rom.id != rom.id, - or_( - and_( - Rom.igdb_id == rom.igdb_id, - Rom.igdb_id.isnot(None), - Rom.igdb_id != "", - ), - and_( - Rom.moby_id == rom.moby_id, - Rom.moby_id.isnot(None), - Rom.moby_id != "", - ), - ), - ) - ) - ).all() - @begin_session def get_rom_collections( self, rom: Rom, session: Session = None diff --git a/backend/handler/filesystem/firmware_handler.py b/backend/handler/filesystem/firmware_handler.py index 24ccc38a6..4aa00c13f 100644 --- a/backend/handler/filesystem/firmware_handler.py +++ b/backend/handler/filesystem/firmware_handler.py @@ -11,8 +11,8 @@ ) from fastapi import UploadFile from logger.logger import log -from models.platform import Platform from utils.filesystem import iter_files +from utils.hashing import crc32_to_hex from .base_handler import FSHandler @@ -27,7 +27,7 @@ def remove_file(self, file_name: str, file_path: str): except IsADirectoryError: shutil.rmtree(f"{LIBRARY_BASE_PATH}/{file_path}/{file_name}") - def get_firmware(self, platform: Platform): + def get_firmware(self, platform_fs_slug: str): """Gets all filesystem firmware for a platform Args: @@ -35,13 +35,13 @@ def get_firmware(self, platform: Platform): Returns: list with all the filesystem firmware for a platform found in the LIBRARY_BASE_PATH """ - firmware_path = self.get_firmware_fs_structure(platform.fs_slug) + firmware_path = self.get_firmware_fs_structure(platform_fs_slug) firmware_file_path = f"{LIBRARY_BASE_PATH}/{firmware_path}" try: fs_firmware_files = [f for _, f in iter_files(firmware_file_path)] except IndexError as exc: - raise FirmwareNotFoundException(platform.fs_slug) from exc + raise FirmwareNotFoundException(platform_fs_slug) from exc return [f for f in self._exclude_files(fs_firmware_files, "single")] @@ -51,13 +51,20 @@ def get_firmware_file_size(self, firmware_path: str, file_name: str): def calculate_file_hashes(self, firmware_path: str, file_name: str): with open(f"{LIBRARY_BASE_PATH}/{firmware_path}/{file_name}", "rb") as f: - data = f.read() + crc_c = 0 + md5_h = hashlib.md5(usedforsecurity=False) + sha1_h = hashlib.sha1(usedforsecurity=False) + + # Read in chunks to avoid memory issues + while chunk := f.read(8192): + md5_h.update(chunk) + sha1_h.update(chunk) + crc_c = binascii.crc32(chunk, crc_c) + return { - "crc_hash": (binascii.crc32(data) & 0xFFFFFFFF) - .to_bytes(4, byteorder="big") - .hex(), - "md5_hash": hashlib.md5(data, usedforsecurity=False).hexdigest(), - "sha1_hash": hashlib.sha1(data, usedforsecurity=False).hexdigest(), + "crc_hash": crc32_to_hex(crc_c), + "md5_hash": md5_h.hexdigest(), + "sha1_hash": sha1_h.hexdigest(), } def file_exists(self, path: str, file_name: str): diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index 5aa786b0e..01f870f45 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -1,13 +1,23 @@ +import binascii +import bz2 +import hashlib import os import re import shutil +import tarfile +import zipfile +from collections.abc import Iterator from pathlib import Path +from typing import Any, Final, TypedDict +import magic +import py7zr from config import LIBRARY_BASE_PATH from config.config_manager import config_manager as cm from exceptions.fs_exceptions import RomAlreadyExistsException, RomsNotFoundException -from models.platform import Platform +from models.rom import RomFile from utils.filesystem import iter_directories, iter_files +from utils.hashing import crc32_to_hex from .base_handler import ( LANGUAGES_BY_SHORTCODE, @@ -18,12 +28,106 @@ FSHandler, ) +# list of known compressed file MIME types +COMPRESSED_MIME_TYPES: Final = [ + "application/zip", + "application/x-tar", + "application/x-gzip", + "application/x-7z-compressed", + "application/x-bzip2", +] + +# list of known file extensions that are compressed +COMPRESSED_FILE_EXTENSIONS = [ + ".zip", + ".tar", + ".gz", + ".7z", + ".bz2", +] + +FILE_READ_CHUNK_SIZE = 1024 * 8 + + +class FSRom(TypedDict): + multi: bool + file_name: str + files: list[RomFile] + + +def is_compressed_file(file_path: str) -> bool: + mime = magic.Magic(mime=True) + file_type = mime.from_file(file_path) + + return file_type in COMPRESSED_MIME_TYPES or file_path.endswith( + tuple(COMPRESSED_FILE_EXTENSIONS) + ) + + +def read_basic_file(file_path: Path) -> Iterator[bytes]: + with open(file_path, "rb") as f: + while chunk := f.read(FILE_READ_CHUNK_SIZE): + yield chunk + + +def read_zip_file(file_path: Path) -> Iterator[bytes]: + try: + with zipfile.ZipFile(file_path, "r") as z: + for file in z.namelist(): + with z.open(file, "r") as f: + while chunk := f.read(FILE_READ_CHUNK_SIZE): + yield chunk + except zipfile.BadZipFile: + for chunk in read_basic_file(file_path): + yield chunk + + +def read_tar_file(file_path: Path, mode: str = "r") -> Iterator[bytes]: + try: + with tarfile.open(file_path, mode) as f: + for member in f.getmembers(): + # Ignore metadata files created by macOS + if member.name.startswith("._"): + continue + + with f.extractfile(member) as ef: # type: ignore + while chunk := ef.read(FILE_READ_CHUNK_SIZE): + yield chunk + except tarfile.ReadError: + for chunk in read_basic_file(file_path): + yield chunk + + +def read_gz_file(file_path: Path) -> Iterator[bytes]: + return read_tar_file(file_path, "r:gz") + + +def read_7z_file(file_path: Path) -> Iterator[bytes]: + try: + with py7zr.SevenZipFile(file_path, "r") as f: + for _name, bio in f.readall().items(): + while chunk := bio.read(FILE_READ_CHUNK_SIZE): + yield chunk + except py7zr.Bad7zFile: + for chunk in read_basic_file(file_path): + yield chunk + + +def read_bz2_file(file_path: Path) -> Iterator[bytes]: + try: + with bz2.BZ2File(file_path, "rb") as f: + while chunk := f.read(FILE_READ_CHUNK_SIZE): + yield chunk + except EOFError: + for chunk in read_basic_file(file_path): + yield chunk + class FSRomsHandler(FSHandler): def __init__(self) -> None: pass - def remove_file(self, file_name: str, file_path: str): + def remove_file(self, file_name: str, file_path: str) -> None: try: os.remove(f"{LIBRARY_BASE_PATH}/{file_path}/{file_name}") except IsADirectoryError: @@ -74,7 +178,7 @@ def parse_tags(self, file_name: str) -> tuple: other_tags.append(tag) return regs, rev, langs, other_tags - def _exclude_multi_roms(self, roms) -> list[str]: + def _exclude_multi_roms(self, roms: list[str]) -> list[str]: excluded_names = cm.get_config().EXCLUDED_MULTI_FILES filtered_files: list = [] @@ -84,16 +188,96 @@ def _exclude_multi_roms(self, roms) -> list[str]: return [f for f in roms if f not in filtered_files] - def get_rom_files(self, rom: str, roms_path: str) -> list[str]: - rom_files: list[str] = [] + def _build_rom_file(self, path: Path) -> RomFile: + return RomFile( + filename=path.name, + size=os.stat(path).st_size, + last_modified=os.path.getmtime(path), + ) + + def get_rom_files(self, rom: str, roms_path: str) -> list[RomFile]: + rom_files: list[RomFile] = [] - for path, _, files in os.walk(f"{roms_path}/{rom}"): - for f in self._exclude_files(files, "multi_parts"): - rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", "")) + # Check if rom is a multi-part rom + if os.path.isdir(f"{roms_path}/{rom}"): + multi_files = os.listdir(f"{roms_path}/{rom}") + for file in self._exclude_files(multi_files, "multi_parts"): + path = Path(roms_path, rom, file) + rom_files.append(self._build_rom_file(path)) + else: + path = Path(roms_path, rom) + rom_files.append(self._build_rom_file(path)) return rom_files - def get_roms(self, platform: Platform): + def _calculate_rom_hashes( + self, file_path: Path, crc_c: int, md5_h: Any, sha1_h: Any + ) -> tuple[int, Any, Any]: + mime = magic.Magic(mime=True) + file_type = mime.from_file(file_path) + extension = Path(file_path).suffix.lower() + + def update_hashes(chunk: bytes): + md5_h.update(chunk) + sha1_h.update(chunk) + nonlocal crc_c + crc_c = binascii.crc32(chunk, crc_c) + + if extension == ".zip" or file_type == "application/zip": + for chunk in read_zip_file(file_path): + update_hashes(chunk) + + elif extension == ".tar" or file_type == "application/x-tar": + for chunk in read_tar_file(file_path): + update_hashes(chunk) + + elif extension == ".gz" or file_type == "application/x-gzip": + for chunk in read_gz_file(file_path): + update_hashes(chunk) + + elif extension == ".7z" or file_type == "application/x-7z-compressed": + for chunk in read_7z_file(file_path): + update_hashes(chunk) + + elif extension == ".bz2" or file_type == "application/x-bzip2": + for chunk in read_bz2_file(file_path): + update_hashes(chunk) + + else: + for chunk in read_basic_file(file_path): + update_hashes(chunk) + + return crc_c, md5_h, sha1_h + + def get_rom_hashes(self, rom: str, roms_path: str) -> dict[str, str]: + roms_file_path = f"{LIBRARY_BASE_PATH}/{roms_path}" + + crc_c = 0 + md5_h = hashlib.md5(usedforsecurity=False) + sha1_h = hashlib.sha1(usedforsecurity=False) + + # Check if rom is a multi-part rom + if os.path.isdir(f"{roms_file_path}/{rom}"): + multi_files = os.listdir(f"{roms_file_path}/{rom}") + for file in self._exclude_files(multi_files, "multi_parts"): + path = Path(roms_file_path, rom, file) + # Pass the raw hashes to the next iteration + crc_c, md5_h, sha1_h = self._calculate_rom_hashes( + path, crc_c, md5_h, sha1_h + ) + else: + path = Path(roms_file_path, rom) + crc_c, md5_h, sha1_h = self._calculate_rom_hashes( + path, crc_c, md5_h, sha1_h + ) + + return { + "crc_hash": crc32_to_hex(crc_c), + "md5_hash": md5_h.hexdigest(), + "sha1_hash": sha1_h.hexdigest(), + } + + def get_roms(self, platform_fs_slug: str) -> list[FSRom]: """Gets all filesystem roms for a platform Args: @@ -101,18 +285,18 @@ def get_roms(self, platform: Platform): Returns: list with all the filesystem roms for a platform found in the LIBRARY_BASE_PATH """ - roms_path = self.get_roms_fs_structure(platform.fs_slug) + roms_path = self.get_roms_fs_structure(platform_fs_slug) roms_file_path = f"{LIBRARY_BASE_PATH}/{roms_path}" try: fs_single_roms = [f for _, f in iter_files(roms_file_path)] except IndexError as exc: - raise RomsNotFoundException(platform.fs_slug) from exc + raise RomsNotFoundException(platform_fs_slug) from exc try: fs_multi_roms = [d for _, d in iter_directories(roms_file_path)] except IndexError as exc: - raise RomsNotFoundException(platform.fs_slug) from exc + raise RomsNotFoundException(platform_fs_slug) from exc fs_roms: list[dict] = [ {"multi": False, "file_name": rom} @@ -123,34 +307,15 @@ def get_roms(self, platform: Platform): ] return [ - dict( - rom, + FSRom( + multi=rom["multi"], + file_name=rom["file_name"], files=self.get_rom_files(rom["file_name"], roms_file_path), ) for rom in fs_roms ] - def get_rom_file_size( - self, - roms_path: str, - file_name: str, - multi: bool, - multi_files: list[str] | None = None, - ): - if multi_files is None: - multi_files = [] - - files = ( - [f"{LIBRARY_BASE_PATH}/{roms_path}/{file_name}"] - if not multi - else [ - f"{LIBRARY_BASE_PATH}/{roms_path}/{file_name}/{file}" - for file in multi_files - ] - ) - return sum([os.stat(file).st_size for file in files]) - - def file_exists(self, path: str, file_name: str): + def file_exists(self, path: str, file_name: str) -> bool: """Check if file exists in filesystem Args: @@ -161,7 +326,7 @@ def file_exists(self, path: str, file_name: str): """ return bool(os.path.exists(f"{LIBRARY_BASE_PATH}/{path}/{file_name}")) - def rename_file(self, old_name: str, new_name: str, file_path: str): + def rename_file(self, old_name: str, new_name: str, file_path: str) -> None: if new_name != old_name: if self.file_exists(path=file_path, file_name=new_name): raise RomAlreadyExistsException(new_name) @@ -171,6 +336,6 @@ def rename_file(self, old_name: str, new_name: str, file_path: str): f"{LIBRARY_BASE_PATH}/{file_path}/{new_name}", ) - def build_upload_file_path(self, fs_slug: str): + def build_upload_file_path(self, fs_slug: str) -> str: file_path = self.get_roms_fs_structure(fs_slug) return f"{LIBRARY_BASE_PATH}/{file_path}" diff --git a/backend/handler/filesystem/tests/test_fs.py b/backend/handler/filesystem/tests/test_fs.py index c825dfbd4..360539046 100644 --- a/backend/handler/filesystem/tests/test_fs.py +++ b/backend/handler/filesystem/tests/test_fs.py @@ -29,7 +29,7 @@ def test_get_roms_fs_structure(): def test_get_roms(): platform = Platform(name="Nintendo 64", slug="n64", fs_slug="n64") - roms = fs_rom_handler.get_roms(platform=platform) + roms = fs_rom_handler.get_roms(platform_fs_slug=platform.fs_slug) assert len(roms) == 2 assert roms[0]["file_name"] == "Paper Mario (USA).z64" @@ -39,28 +39,6 @@ def test_get_roms(): assert roms[1]["multi"] -def test_rom_size(): - rom_size = fs_rom_handler.get_rom_file_size( - roms_path=fs_rom_handler.get_roms_fs_structure(fs_slug="n64"), - file_name="Paper Mario (USA).z64", - multi=False, - ) - - assert rom_size == 1024 - - rom_size = fs_rom_handler.get_rom_file_size( - roms_path=fs_rom_handler.get_roms_fs_structure(fs_slug="n64"), - file_name="Super Mario 64 (J) (Rev A)", - multi=True, - multi_files=[ - "Super Mario 64 (J) (Rev A) [Part 1].z64", - "Super Mario 64 (J) (Rev A) [Part 2].z64", - ], - ) - - assert rom_size == 2048 - - def test_exclude_files(): from config.config_manager import ConfigManager diff --git a/backend/handler/metadata/base_hander.py b/backend/handler/metadata/base_hander.py index f163b8fdf..9bfce872a 100644 --- a/backend/handler/metadata/base_hander.py +++ b/backend/handler/metadata/base_hander.py @@ -2,6 +2,7 @@ import os import re import unicodedata +from itertools import batched from typing import Final from handler.redis_handler import async_cache, sync_cache @@ -11,7 +12,6 @@ SWITCH_TITLEDB_INDEX_KEY, update_switch_titledb_task, ) -from utils.iterators import batched def conditionally_set_cache( diff --git a/backend/handler/metadata/igdb_handler.py b/backend/handler/metadata/igdb_handler.py index 5d2f7eff4..87181485f 100644 --- a/backend/handler/metadata/igdb_handler.py +++ b/backend/handler/metadata/igdb_handler.py @@ -1,7 +1,7 @@ import functools import re import time -from typing import Final, NotRequired +from typing import Final, NotRequired, TypedDict import httpx import pydash @@ -9,7 +9,6 @@ from fastapi import HTTPException, status from handler.redis_handler import sync_cache from logger.logger import log -from typing_extensions import TypedDict from unidecode import unidecode as uc from utils.context import ctx_httpx_client diff --git a/backend/handler/metadata/moby_handler.py b/backend/handler/metadata/moby_handler.py index a623638e5..28774e987 100644 --- a/backend/handler/metadata/moby_handler.py +++ b/backend/handler/metadata/moby_handler.py @@ -1,7 +1,7 @@ import asyncio import http import re -from typing import Final, NotRequired +from typing import Final, NotRequired, TypedDict from urllib.parse import quote import httpx @@ -10,7 +10,6 @@ from config import MOBYGAMES_API_KEY from fastapi import HTTPException, status from logger.logger import log -from typing_extensions import TypedDict from unidecode import unidecode as uc from utils.context import ctx_httpx_client @@ -621,6 +620,7 @@ class SlugToMobyId(TypedDict): "sega-cd": {"id": 20, "name": "SEGA CD"}, "segacd": {"id": 20, "name": "SEGA CD"}, # IGDB "sega-master-system": {"id": 26, "name": "SEGA Master System"}, + "sms": {"id": 26, "name": "SEGA Master System"}, # IGDB "sega-pico": {"id": 103, "name": "SEGA Pico"}, "sega-saturn": {"id": 23, "name": "SEGA Saturn"}, "saturn": {"id": 23, "name": "SEGA Saturn"}, # IGDB diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index b574d0d91..13bc0937e 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -1,3 +1,4 @@ +import asyncio from enum import Enum from typing import Any @@ -5,6 +6,7 @@ from config.config_manager import config_manager as cm from handler.database import db_platform_handler from handler.filesystem import fs_asset_handler, fs_firmware_handler, fs_rom_handler +from handler.filesystem.roms_handler import FSRom from handler.metadata import meta_igdb_handler, meta_moby_handler from handler.metadata.igdb_handler import IGDBPlatform, IGDBRom from handler.metadata.moby_handler import MobyGamesPlatform, MobyGamesRom @@ -22,6 +24,7 @@ class ScanType(Enum): UNIDENTIFIED = "unidentified" PARTIAL = "partial" COMPLETE = "complete" + HASHES = "hashes" async def _get_main_platform_igdb_id(platform: Platform): @@ -156,7 +159,7 @@ def scan_firmware( async def scan_rom( platform: Platform, - rom_attrs: dict, + fs_rom: FSRom, scan_type: ScanType, rom: Rom | None = None, metadata_sources: list[str] | None = None, @@ -166,22 +169,21 @@ async def scan_rom( roms_path = fs_rom_handler.get_roms_fs_structure(platform.fs_slug) - log.info(f"\t · {rom_attrs['file_name']}") + log.info(f"\t · {fs_rom['file_name']}") - if rom_attrs.get("multi", False): - for file in rom_attrs["files"]: - log.info(f"\t\t · {file}") + if fs_rom.get("multi", False): + for file in fs_rom["files"]: + log.info(f"\t\t · {file['filename']}") # Set default properties - rom_attrs.update( - { - "id": rom.id if rom else None, - "platform_id": platform.id, - "name": rom_attrs["file_name"], - "url_cover": "", - "url_screenshots": [], - } - ) + rom_attrs = { + **fs_rom, + "id": rom.id if rom else None, + "platform_id": platform.id, + "name": fs_rom["file_name"], + "url_cover": "", + "url_screenshots": [], + } # Update properties from existing rom if not a complete rescan if rom and scan_type != ScanType.COMPLETE: @@ -204,12 +206,7 @@ async def scan_rom( ) # Update properties that don't require metadata - file_size = fs_rom_handler.get_rom_file_size( - multi=rom_attrs["multi"], - file_name=rom_attrs["file_name"], - multi_files=rom_attrs["files"], - roms_path=roms_path, - ) + file_size = sum([file["size"] for file in rom_attrs["files"]]) regs, rev, langs, other_tags = fs_rom_handler.parse_tags(rom_attrs["file_name"]) rom_attrs.update( { @@ -233,42 +230,59 @@ async def scan_rom( } ) - igdb_handler_rom: IGDBRom = IGDBRom(igdb_id=None) - moby_handler_rom: MobyGamesRom = MobyGamesRom(moby_id=None) - - if ( - "igdb" in metadata_sources - and platform.igdb_id - and ( - not rom - or scan_type == ScanType.COMPLETE - or (scan_type == ScanType.PARTIAL and not rom.igdb_id) - or (scan_type == ScanType.UNIDENTIFIED and not rom.igdb_id) - ) - ): - main_platform_igdb_id = await _get_main_platform_igdb_id(platform) - igdb_handler_rom = await meta_igdb_handler.get_rom( - rom_attrs["file_name"], main_platform_igdb_id - ) + # Calculating hashes is expensive, so we only do it if necessary + if not rom or scan_type == ScanType.COMPLETE or scan_type == ScanType.HASHES: + rom_hashes = fs_rom_handler.get_rom_hashes(rom_attrs["file_name"], roms_path) + rom_attrs.update(**rom_hashes) - if ( - "moby" in metadata_sources - and platform.moby_id - and ( - not rom - or scan_type == ScanType.COMPLETE - or (scan_type == ScanType.PARTIAL and not rom.moby_id) - or (scan_type == ScanType.UNIDENTIFIED and not rom.moby_id) - ) - ): - moby_handler_rom = await meta_moby_handler.get_rom( - rom_attrs["file_name"], platform_moby_id=platform.moby_id - ) + # If no metadata scan is required + if scan_type == ScanType.HASHES: + return Rom(**rom_attrs) + + async def fetch_igdb_rom(): + if ( + "igdb" in metadata_sources + and platform.igdb_id + and ( + not rom + or scan_type == ScanType.COMPLETE + or (scan_type == ScanType.PARTIAL and not rom.igdb_id) + or (scan_type == ScanType.UNIDENTIFIED and not rom.igdb_id) + ) + ): + main_platform_igdb_id = await _get_main_platform_igdb_id(platform) + return await meta_igdb_handler.get_rom( + rom_attrs["file_name"], main_platform_igdb_id + ) + + return IGDBRom(igdb_id=None) + + async def fetch_moby_rom(): + if ( + "moby" in metadata_sources + and platform.moby_id + and ( + not rom + or scan_type == ScanType.COMPLETE + or (scan_type == ScanType.PARTIAL and not rom.moby_id) + or (scan_type == ScanType.UNIDENTIFIED and not rom.moby_id) + ) + ): + return await meta_moby_handler.get_rom( + rom_attrs["file_name"], platform_moby_id=platform.moby_id + ) + + return MobyGamesRom(moby_id=None) + + # Run both metadata fetches concurrently + igdb_handler_rom, moby_handler_rom = await asyncio.gather( + fetch_igdb_rom(), fetch_moby_rom() + ) # Reversed to prioritize IGDB rom_attrs.update({**moby_handler_rom, **igdb_handler_rom}) - # Return early if not found in IGDB or MobyGames + # If not found in IGDB or MobyGames if not igdb_handler_rom.get("igdb_id") and not moby_handler_rom.get("moby_id"): log.warning( emoji.emojize(f"\t {rom_attrs['file_name']} not found :cross_mark:") diff --git a/backend/handler/tests/cassettes/test_fastapi/test_scan_rom.yaml b/backend/handler/tests/cassettes/test_fastapi/test_scan_rom.yaml index 9924c89fc..4ca969dd0 100644 --- a/backend/handler/tests/cassettes/test_fastapi/test_scan_rom.yaml +++ b/backend/handler/tests/cassettes/test_fastapi/test_scan_rom.yaml @@ -17,7 +17,7 @@ interactions: Content-Length: - "796" User-Agent: - - python-requests/2.32.3 + - python-requests/2.32.0 method: POST uri: https://api.igdb.com/v4/games response: @@ -33,7 +33,7 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8954458a8d1ac1c8-BUD + - 8a4509ef2f06ab0c-YYZ Connection: - keep-alive Content-Length: @@ -41,12 +41,12 @@ interactions: Content-Type: - application/json Date: - - Mon, 17 Jun 2024 16:03:47 GMT + - Tue, 16 Jul 2024 21:20:53 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=Pkqts9nrlZzzqm8MAWz11FD9VAhWJG81SmXWzhFZVXQ-1718640227-1.0.1.1-IlZEHZANhqRkqtx6l9gZD7ncaSkhbRdt7pvVairBZqrcfrq7bZZIIFOLHNRRYBZn.9wEBmD.UmLrxmhaSg2y7A; - path=/; expires=Mon, 17-Jun-24 16:33:47 GMT; domain=.igdb.com; HttpOnly; Secure; + - __cf_bm=K63xoXgTxGJm.n8D27NJG_ly4hN5z2J6yYDTzgLbD80-1721164853-1.0.1.1-p2nx3KwSJSjeNAWPDw0cRQAUg6YHgO_Y3Z0jYU6sB2Yr5fkj8Db6FRqWdlVSS5EN_6KieLIwqUxDUg4J45h50g; + path=/; expires=Tue, 16-Jul-24 21:50:53 GMT; domain=.igdb.com; HttpOnly; Secure; SameSite=None Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload @@ -55,17 +55,17 @@ interactions: alt-svc: - h3=":443"; ma=86400 via: - - 1.1 ebc0709f2918acef5e26208dffcb618c.cloudfront.net (CloudFront) + - 1.1 1acedc07a77a02e11785c2290515f2e4.cloudfront.net (CloudFront) x-amz-apigw-id: - - ZhOvkGSYPHcEk1A= + - bBiYcG8dvHcEJyg= x-amz-cf-id: - - 0lKqIn-0Z3y8symJxCZXC5axLtvm5dfu9qs8-s50-DR6KG1wAXCVhA== + - 5BP2CkODtFEThqTb1WSzRtjpsi2tscDJRx4NmPXzCpmEQod5maIjnQ== x-amz-cf-pop: - - FRA56-P8 + - ORD51-C2 x-amzn-errortype: - AccessDeniedException x-amzn-requestid: - - 406ba6c1-1be5-4eee-8f8d-03579da25c54 + - b6b00df1-1438-4aa1-bda3-0cd0f7c23296 x-cache: - Error from cloudfront status: @@ -83,13 +83,13 @@ interactions: Content-Length: - "0" User-Agent: - - python-requests/2.32.3 + - python-requests/2.32.0 method: POST uri: https://id.twitch.tv/oauth2/token?client_id=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&client_secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&grant_type=client_credentials response: body: string: - '{"access_token":"yg0ucvq4spk9v1eict05z4bpkctaso","expires_in":5268473,"token_type":"bearer"} + '{"access_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","expires_in":5624809,"token_type":"bearer"} ' headers: @@ -102,11 +102,11 @@ interactions: Content-Type: - application/json Date: - - Mon, 17 Jun 2024 16:03:48 GMT + - Tue, 16 Jul 2024 21:20:54 GMT Server: - nginx X-Ctxlog-Logid: - - 1-66705e64-4daa4ef9086314ea51864a45 + - 1-6696e436-783213912ac0e5067f1e9592 status: code: 200 message: OK @@ -120,7 +120,7 @@ interactions: Accept-Encoding: - gzip, deflate Authorization: - - Bearer yg0ucvq4spk9v1eict05z4bpkctaso + - Bearer xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Client-ID: - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Connection: @@ -128,7 +128,7 @@ interactions: Content-Length: - "796" User-Agent: - - python-requests/2.32.3 + - python-requests/2.32.0 method: POST uri: https://api.igdb.com/v4/games response: @@ -206,7 +206,7 @@ interactions: the Mushroom Kingdom\\u0027s diverse locales and biomes, meeting its inhabitants, fighthing unruly enemies and recruiting an array of companions in order to once again save Princess Peach from the clutches of the evil Koopa King Bowser.\",\n - \ \"total_rating\": 88.63712520126188,\n \"collections\": [\n {\n + \ \"total_rating\": 88.63753217601834,\n \"collections\": [\n {\n \ \"id\": 240,\n \"name\": \"Super Mario\"\n },\n {\n \ \"id\": 593,\n \"name\": \"Paper Mario\"\n }\n ]\n \ }\n]" @@ -214,20 +214,20 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8954459468d768bb-BUD + - 8a4509f43a89abb8-YYZ Connection: - keep-alive Content-Length: - - "1529" + - "1528" Content-Type: - application/json Date: - - Mon, 17 Jun 2024 16:03:48 GMT + - Tue, 16 Jul 2024 21:20:54 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=MZGQ45rHyOlT6dLcQvaxjOTTbOgnNg4SGP9mJGrJCJ0-1718640228-1.0.1.1-5d1QCqvlQBVVBcfqZHDeacIHc5d8UKwR1QBiaLjeF7wu5qdTPIFI9uGU.9LSOBjbiqUwW73rQd2EbPuqduXO3Q; - path=/; expires=Mon, 17-Jun-24 16:33:48 GMT; domain=.igdb.com; HttpOnly; Secure; + - __cf_bm=PsncMQhPTWpR1oHjMpk15qRUP7WbtGRS.NGfZOY65l8-1721164854-1.0.1.1-cYy4SO3EJA2a4jqwmm1x1Zl3Z9o7Y8cIfRAv5mv0GnC8rjX2I1KEmbMp3aurQ3iwuGOS_LwNuMReKOD74TwXbA; + path=/; expires=Tue, 16-Jul-24 21:50:54 GMT; domain=.igdb.com; HttpOnly; Secure; SameSite=None Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload @@ -236,19 +236,19 @@ interactions: alt-svc: - h3=":443"; ma=86400 via: - - 1.1 1e0f88a39289286be3e03ff93487da80.cloudfront.net (CloudFront) + - 1.1 412b0215b557780a6efcc1651037dc90.cloudfront.net (CloudFront) x-amz-apigw-id: - - ZhOvxGLJPHcEiyw= + - bBiYjGocvHcES9Q= x-amz-cf-id: - - BRUeRhAc8MkId4W5Ytgwib4jTq6EIo4VgQKkZoAw-vECfwX38KZw4A== + - qNRcxGGRlOiahu_AQFHB6TMBbOASu2LA0t9PmPndvxSI4nkv96V3dA== x-amz-cf-pop: - - FRA56-P8 + - YTO50-P1 x-amzn-remapped-content-length: - - "1529" + - "1528" x-amzn-remapped-date: - - Mon, 17 Jun 2024 16:03:48 GMT + - Tue, 16 Jul 2024 21:20:54 GMT x-amzn-requestid: - - f83c4ba8-e83b-4b27-b769-3de078d84ba2 + - 2659af3f-8bbb-4d7b-9e1c-dc972fc6367f x-cache: - Miss from cloudfront x-count: @@ -266,7 +266,7 @@ interactions: Accept-Encoding: - gzip, deflate Authorization: - - Bearer yg0ucvq4spk9v1eict05z4bpkctaso + - Bearer xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Client-ID: - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Connection: @@ -274,7 +274,7 @@ interactions: Content-Length: - "121" User-Agent: - - python-requests/2.32.3 + - python-requests/2.32.0 method: POST uri: https://api.igdb.com/v4/search response: @@ -306,18 +306,18 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 89544596b8d6684c-BUD + - 8a4509f668ce7119-YYZ Connection: - keep-alive Content-Type: - application/json Date: - - Mon, 17 Jun 2024 16:03:49 GMT + - Tue, 16 Jul 2024 21:20:55 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=S3S9rrHhBOvzvmVCHIeVraPhJyoqg7eFaasmWKy4WrE-1718640229-1.0.1.1-MynYCqaygKWR4apMN6El6uR.QP9ouU93PdAsZ4IXfYQYAFER1Z9ek7QyBfLq60KBam5Te3U.RvZcUdnUrDIw0g; - path=/; expires=Mon, 17-Jun-24 16:33:49 GMT; domain=.igdb.com; HttpOnly; Secure; + - __cf_bm=KkAmqxhQPavV7fCKkDnuvzC9DyvenDp7tA7GNRkkVAc-1721164855-1.0.1.1-Ae5ASfcHkdEuszHbLuVn2rOsIOFO4lhkF0wJz_zkUMyHZGyjfMfbxc23IZI6vc_liGEP2Xgy2dI6I4xyduE1pA; + path=/; expires=Tue, 16-Jul-24 21:50:55 GMT; domain=.igdb.com; HttpOnly; Secure; SameSite=None Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload @@ -328,19 +328,19 @@ interactions: alt-svc: - h3=":443"; ma=86400 via: - - 1.1 193d38535c6cb246e365763e9c32e672.cloudfront.net (CloudFront) + - 1.1 19d1514f5f81da4dca6349d0f75a352c.cloudfront.net (CloudFront) x-amz-apigw-id: - - ZhOv1HtuPHcETtw= + - bBiYnGaWvHcEMVw= x-amz-cf-id: - - 5tJYJzvDTLMBuOc9kX5sQbOwUU8M31kbfrCwEwn6Kmrf2Y6e7LFZEg== + - qgoGtG02JXbCB4cXLi0sKJRw1f7uX6caTDqPIzUFUAiiaaTYr0DPmw== x-amz-cf-pop: - - FRA56-P8 + - YUL62-C2 x-amzn-remapped-content-length: - "1421" x-amzn-remapped-date: - - Mon, 17 Jun 2024 16:03:49 GMT + - Tue, 16 Jul 2024 21:20:54 GMT x-amzn-requestid: - - 55032156-444c-4347-af24-51c20426ee8b + - b12a507c-48c8-4e57-8028-af00f0ec2ecf x-cache: - Miss from cloudfront x-count: @@ -358,7 +358,7 @@ interactions: Accept-Encoding: - gzip, deflate Authorization: - - Bearer yg0ucvq4spk9v1eict05z4bpkctaso + - Bearer xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Client-ID: - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Connection: @@ -366,7 +366,7 @@ interactions: Content-Length: - "739" User-Agent: - - python-requests/2.32.3 + - python-requests/2.32.0 method: POST uri: https://api.igdb.com/v4/games response: @@ -444,7 +444,7 @@ interactions: the Mushroom Kingdom\\u0027s diverse locales and biomes, meeting its inhabitants, fighthing unruly enemies and recruiting an array of companions in order to once again save Princess Peach from the clutches of the evil Koopa King Bowser.\",\n - \ \"total_rating\": 88.63712520126188,\n \"collections\": [\n {\n + \ \"total_rating\": 88.63753217601834,\n \"collections\": [\n {\n \ \"id\": 240,\n \"name\": \"Super Mario\"\n },\n {\n \ \"id\": 593,\n \"name\": \"Paper Mario\"\n }\n ]\n \ }\n]" @@ -452,20 +452,20 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8954459c292fc1bc-BUD + - 8a4509fc9e8da246-YYZ Connection: - keep-alive Content-Length: - - "1529" + - "1528" Content-Type: - application/json Date: - - Mon, 17 Jun 2024 16:03:50 GMT + - Tue, 16 Jul 2024 21:20:56 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=ou6Xt5NppxZER6H5U_Hf7JhkNeFmqvfi_r26YZDDiDA-1718640230-1.0.1.1-8odH4zUHWZ2rEQs6Es07QiilEn4uihIVSrpuO7nEvnsZyiCrpTXBUJeocFATmAs2nlDkU1rzUo3ywc0a4Vza1w; - path=/; expires=Mon, 17-Jun-24 16:33:50 GMT; domain=.igdb.com; HttpOnly; Secure; + - __cf_bm=5bPHYIegBMi1xJ02b0hjaxBDqtvwnry9PO980Xd7lNc-1721164856-1.0.1.1-cu9HyVrxbU2z.sX3r4j0oAGX.uCZMrSGh9G3DRsu7t2.CCaDCEZcRTqdbASeR2V9LLJseMkdn7TF7FBTjNY2DA; + path=/; expires=Tue, 16-Jul-24 21:50:56 GMT; domain=.igdb.com; HttpOnly; Secure; SameSite=None Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload @@ -474,19 +474,19 @@ interactions: alt-svc: - h3=":443"; ma=86400 via: - - 1.1 b7c8b552077b93dc0acaa0b82d11fa62.cloudfront.net (CloudFront) + - 1.1 490c6f54e6cd81b80f07ff6be833267e.cloudfront.net (CloudFront) x-amz-apigw-id: - - ZhOv9EwOPHcEDOQ= + - bBiYxEaCPHcEErA= x-amz-cf-id: - - 6Sn4tTPxhiFmPanVHta__qD8AW6cJVXrSXtZvgRED-1CCTbimEhZzw== + - syxc_2spiC05CJNtd7tJxBxPfHKt0FzGAI0rbGnWlRK5_zFEK10LWA== x-amz-cf-pop: - - FRA56-P8 + - YTO50-P1 x-amzn-remapped-content-length: - - "1529" + - "1528" x-amzn-remapped-date: - - Mon, 17 Jun 2024 16:03:49 GMT + - Tue, 16 Jul 2024 21:20:55 GMT x-amzn-requestid: - - c60d2696-7381-4b4d-ac0f-18b1c9292fb5 + - 5674e595-eafc-4003-91be-0267d28f76c9 x-cache: - Miss from cloudfront x-count: diff --git a/backend/handler/tests/test_fastapi.py b/backend/handler/tests/test_fastapi.py index e757ddee5..6b17b5d6c 100644 --- a/backend/handler/tests/test_fastapi.py +++ b/backend/handler/tests/test_fastapi.py @@ -1,7 +1,7 @@ import pytest from handler.scan_handler import ScanType, scan_platform, scan_rom from models.platform import Platform -from models.rom import Rom +from models.rom import Rom, RomFile from utils.context import initialize_context @@ -30,13 +30,17 @@ async def test_scan_rom(): platform = Platform(fs_slug="n64", igdb_id=4) async with initialize_context(): + files = [ + RomFile( + filename="Paper Mario (USA).z64", + size=1024, + last_modified=1620000000, + ) + ] + rom = await scan_rom( platform, - { - "file_name": "Paper Mario (USA).z64", - "multi": False, - "files": ["Paper Mario (USA).z64"], - }, + {"file_name": "Paper Mario (USA).z64", "multi": False, "files": files}, ScanType.QUICK, ) @@ -45,6 +49,6 @@ async def test_scan_rom(): assert rom.name == "Paper Mario" assert rom.igdb_id == 3340 assert rom.file_size_bytes == 1024 - assert rom.files == ["Paper Mario (USA).z64"] + assert rom.files == files assert rom.tags == [] assert not rom.multi diff --git a/backend/main.py b/backend/main.py index 8f3eb38b1..c810f870d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -32,7 +32,6 @@ ) from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.middleware.gzip import GZipMiddleware from handler.auth.base_handler import ALGORITHM from handler.auth.hybrid_auth import HybridAuthBackend from handler.auth.middleware import CustomCSRFMiddleware, SessionMiddleware @@ -68,9 +67,6 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: exempt_urls=[re.compile(r"^/token.*"), re.compile(r"^/ws")], ) -# Enable GZip compression for responses -app.add_middleware(GZipMiddleware, minimum_size=1024) - # Handles both basic and oauth authentication app.add_middleware( AuthenticationMiddleware, diff --git a/backend/models/firmware.py b/backend/models/firmware.py index cb79190f2..06b2c73f5 100644 --- a/backend/models/firmware.py +++ b/backend/models/firmware.py @@ -60,11 +60,10 @@ def is_verified(self) -> bool: ) if cache_entry: cache_json = json.loads(cache_entry) - return ( - self.file_size_bytes == int(cache_json.get("size", 0)) - and self.md5_hash == cache_json.get("md5") - and self.sha1_hash == cache_json.get("sha1") - and self.crc_hash == cache_json.get("crc") + return self.file_size_bytes == int(cache_json.get("size", 0)) and ( + self.md5_hash == cache_json.get("md5") + or self.sha1_hash == cache_json.get("sha1") + or self.crc_hash == cache_json.get("crc") ) return False diff --git a/backend/models/fixtures/known_bios_files.json b/backend/models/fixtures/known_bios_files.json index 8b9b3f163..34e844225 100644 --- a/backend/models/fixtures/known_bios_files.json +++ b/backend/models/fixtures/known_bios_files.json @@ -299,7 +299,7 @@ "md5": "66223be1497460f1e60885eeb35e03cc", "sha1": "db6b504744281369794e26ba71a6e385cf6227fa" }, - "odyssey-2-slash-videopac-g7000o2rom.bin": { + "odyssey-2-slash-videopac-g7000:o2rom.bin": { "size": "1024", "crc": "8016a315", "md5": "562d5ebf9e030a40d6fabfc2f33139fd", @@ -875,12 +875,24 @@ "md5": "e10c53c2f8b90bab96ead2d368858623", "sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c" }, + "dc:boot.bin": { + "size": "2097152", + "crc": "89f2b1a1", + "md5": "e10c53c2f8b90bab96ead2d368858623", + "sha1": "8951d1bb219ab2ff8583033d2119c899cc81f18c" + }, "dc:dc_flash.bin": { "size": "131072", "crc": "c611b498", "md5": "0a93f7940c455905bea6e392dfde92a4", "sha1": "94d44d7f9529ec1642ba3771ed3c5f756d5bc872" }, + "dc:flash.bin": { + "size": "131072", + "crc": "c611b498", + "md5": "0a93f7940c455905bea6e392dfde92a4", + "sha1": "94d44d7f9529ec1642ba3771ed3c5f756d5bc872" + }, "dc:naomi_boot.bin": { "size": "2097152", "crc": "d2a1c6bf", @@ -1189,21 +1201,21 @@ }, "zxs:plus3-0.rom": { "size": "16384", - "crc": "7f4a5482", - "md5": "bc123f625e245c225f92ef05933ed134", - "sha1": "649fbd233490bf58b35350b0123d36caaaa011eb" + "crc": "a10230c0", + "md5": "3abdc20e72890a750dd3c745d286dfba", + "sha1": "a837f66977040f7b51ed053a2483c10f3d070ab7" }, "zxs:plus3-1.rom": { "size": "16384", - "crc": "5aeb4675", - "md5": "617364264c587d20c9fc4746c29679f2", - "sha1": "f12198108cbb14de4f03c6695bc16d08c85ee214" + "crc": "09b9c3ca", + "md5": "8361a1d9c8bcef89c0c39293776564ad", + "sha1": "6a4364f25513e4079f048f2de131a896d30edc64" }, "zxs:plus3-2.rom": { "size": "16384", - "crc": "504755cb", - "md5": "c363e95dcd0a90e6e7f847e6e47e0179", - "sha1": "773633dce5ba323a9e00d9d0f9e4d8c295df7c87" + "crc": "a60285a0", + "md5": "f36c5c2d1f2a682caadeaa6f947db0da", + "sha1": "0a747cc0b827a94b4fd74cfd818ca792437a38f7" }, "zxs:plus3-3.rom": { "size": "16384", @@ -1547,6 +1559,12 @@ "md5": "b9d9a0286c33dc6b7237bb13cd46fdee", "sha1": "8d5de56a79954f29e9006929ba3fed9b6a418c1d" }, + "ps:scph9002(7502).bin": { + "size": "524288", + "crc": "318178bf", + "md5": "b9d9a0286c33dc6b7237bb13cd46fdee", + "sha1": "8d5de56a79954f29e9006929ba3fed9b6a418c1d" + }, "psp:ppge_atlas.zim": { "size": "666530", "crc": "7b57fa78", @@ -1564,5 +1582,545 @@ "crc": "a93f1c4b", "md5": "a17e0e0150155400d8cced329563d9c8", "sha1": "718c1a00d38e0810a1ad0ffde79f73447f846f01" + }, + "acpc:cpc464.rom": { + "size": "32768", + "crc": "40852f25", + "md5": "a993f85b88ac4350cf4d41554e87fe4f", + "sha1": "56d39c463da60968d93e58b4ba0e675829412a20" + }, + "acpc:cpc664.rom": { + "size": "32768", + "crc": "9ab5a036", + "md5": "5a384a2310f472c7857888371c00ed66", + "sha1": "073a7665527b5bd8a148747a3947dbd3328682c8" + }, + "acpc:cpc6128.rom": { + "size": "32768", + "crc": "9e827fe1", + "md5": "b96280dc6c95a48857b4b8eb931533ae", + "sha1": "5977adbad3f7c1e0e082cd02fe76a700d9860c30" + }, + "acpc:cpc_amsdos.rom": { + "size": "16384", + "crc": "1fe22ecd", + "md5": "25629dfe870d097469c217b95fdc1c95", + "sha1": "39102c8e9cb55fcc0b9b62098780ed4a3cb6a4bb" + }, + "enterprise:hun.rom": { + "size": "16384", + "crc": "596ab6d6", + "md5": "22167938f142c222f40992839aa21a06", + "sha1": "325a5e28c2a0d896711f8829e7ff14fed5dd4103" + }, + "enterprise:brd.rom": { + "size": "16384", + "crc": "6999d6a3", + "md5": "6af0402906944fd134004b85097c8524", + "sha1": "f34f0c330b44dbf2548329bea954d5991dec30ca" + }, + "enterprise:exos20.rom": { + "size": "32768", + "crc": "d421795f", + "md5": "5ad3baaad3b5156d6b60b34229a676fb", + "sha1": "6033a0535136c40c47137e4d1cd9273c06d5fdff" + }, + "enterprise:exos21.rom": { + "size": "32768", + "crc": "982a3b44", + "md5": "f36f24cbb87745fbd2714e4df881db09", + "sha1": "55315b20fecb4441a07ee4bc5dc7153f396e0a2e" + }, + "enterprise:zt19uk.rom": { + "size": "32768", + "crc": "d6deedf1", + "md5": "228540b6be83ae2acd7569c8ff0f91d0", + "sha1": "b7af62f0bc95fdca4b31d236f8327dafc80f83b7" + }, + "enterprise:basic20.rom": { + "size": "16384", + "crc": "1228de34", + "md5": "8e18edce4a7acb2c33cc0ab18f988482", + "sha1": "61d0987b906146e21b94f265d5b51b4938c986a9" + }, + "enterprise:basic21.rom": { + "size": "16384", + "crc": "55f96251", + "md5": "e972fe42b398c9ff1d93ff014786aec6", + "sha1": "03bbb386cf530e804363acdfc1d13e64cf28af2e" + }, + "enterprise:exdos13.rom": { + "size": "32768", + "crc": "e0135929", + "md5": "ddff70c014d1958dc75378b6c9aab6f8", + "sha1": "cb43ab3676b93c279f1ed8ffcb0d4dcd4b34e631" + }, + "enterprise:epd19hft.rom": { + "size": "32768", + "crc": "bd503eeb", + "md5": "12cfc9c7e48c8a16c2e09edbd926d467", + "sha1": "8f28fe73d13e94dd1da02519908ecc6eebe104f6" + }, + "enterprise:zt18hfnt.rom": { + "size": "32768", + "crc": "76c9dbf6", + "md5": "3082dc488d32f30a612761b99074199b", + "sha1": "283be1ce417a759a3368bb4bbe72f692fd43ca6d" + }, + "enterprise:epfileio.rom": { + "size": "16384", + "crc": "60c79925", + "md5": "a68ebcbc73a4d2178d755b7755bf18fe", + "sha1": "2f9077bcd89b1ec42dbdcd55d335bdbaf361eff3" + }, + "enterprise:exos24uk.rom": { + "size": "65536", + "crc": "c099a5e3", + "md5": "55af78f877a21ca45eb2df68a74fcc60", + "sha1": "cf12e971623a54bf8c4f891ca3a36d969f205c49" + }, + "ps2:ps2-0100jd-20000117.bin": { + "size": "4194304", + "crc": "5a04500c", + "md5": "32f2e4d5ff5ee11072a6bc45530f5765", + "sha1": "5b33170323ed6344e2363fed8115dc3918bb96a4" + }, + "ps2:ps2-0100j-20000117.bin": { + "size": "4194304", + "crc": "b7ef81a9", + "md5": "acf4730ceb38ac9d8c7d8e21f2614600", + "sha1": "aea061e6e263fdcc1c4fdbd68553ef78dae74263" + }, + "ps2:ps2-0101jd-20000217.bin": { + "size": "4194304", + "crc": "4f8b4205", + "md5": "acf9968c8f596d2b15f42272082513d1", + "sha1": "16f4a284d0e760ee13a2aff2f7dda928255e3080" + }, + "ps2:ps2-0101j-20000217.bin": { + "size": "4194304", + "crc": "211dfb6a", + "md5": "b1459d7446c69e3e97e6ace3ae23dd1c", + "sha1": "916e02431bcd73140504da3355c9598143b77e11" + }, + "ps2:ps2-0101xd-20000224.bin": { + "size": "4194304", + "crc": "2fef9faf", + "md5": "d3f1853a16c2ec18f3cd1ae655213308", + "sha1": "4440b246bfde7bb31002c584a76c6ef384908e84" + }, + "ps2:ps2-0110ad-20000727.bin": { + "size": "4194304", + "crc": "795578c1", + "md5": "63e6fd9b3c72e0d7b920e80cf76645cd", + "sha1": "339c646cf0699268552df5b05f18f0a03a9f55ff" + }, + "ps2:ps2-0110a-20000727.bin": { + "size": "4194304", + "crc": "9678ad6a", + "md5": "a20c97c02210f16678ca3010127caf36", + "sha1": "20f6ce6693cf97e9494f8f0227f2b7988ffaf961" + }, + "ps2:ps2-0120a-20000902.bin": { + "size": "4194304", + "crc": "1ae71e5d", + "md5": "8db2fbbac7413bf3e7154c1e0715e565", + "sha1": "dbc2318a1029347b5af3a0c74b0bdf88d19efee6" + }, + "ps2:ps2-0120ed-20000902.bin": { + "size": "4194304", + "crc": "25495aa7", + "md5": "91c87cb2f2eb6ce529a2360f80ce2457", + "sha1": "3bb1eecd618ab5c973c7bc53671a4475a02e1d5b" + }, + "ps2:ps2-0120ed-20000902-20030110.bin": { + "size": "4194304", + "crc": "e2f78425", + "md5": "3016b3dd42148a67e2c048595ca4d7ce", + "sha1": "1b73dec999fcc2b92fa958110ff6bfe4d0af276e" + }, + "ps2:ps2-0120e-20000902.bin": { + "size": "4194304", + "crc": "7b08c33b", + "md5": "b7fa11e87d51752a98b38e3e691cbf17", + "sha1": "274c05fec654913a3f698d4b0d592085866a2cbd" + }, + "ps2:ps2-0120j-20001027-185015.bin": { + "size": "4194304", + "crc": "9b096622", + "md5": "f63bc530bd7ad7c026fcd6f7bd0d9525", + "sha1": "e481079eca752225555f0c26d14c9d0f94d9a8e9" + }, + "ps2:ps2-0120j-20001027-191435.bin": { + "size": "4194304", + "crc": "c1ccf3f6", + "md5": "cee06bd68c333fc5768244eae77e4495", + "sha1": "a9f5d8ed56cfff18add1b599010493461fa02448" + }, + "ps2:ps2-0150ad-20001228-20030520.bin": { + "size": "4194304", + "crc": "0380c2ce", + "md5": "0bf988e9c7aaa4c051805b0fa6eb3387", + "sha1": "7284b9d16df9935afc384318e024c87ef0574fe5" + }, + "ps2:ps2-0150a-20001228.bin": { + "size": "4194304", + "crc": "bce74746", + "md5": "8accc3c49ac45f5ae2c5db0adc854633", + "sha1": "5af5b5077d84a9c037ebe12bfab8a38b31d8a543" + }, + "ps2:ps2-0150ed-20001228-20030520.bin": { + "size": "4194304", + "crc": "60bc0031", + "md5": "6f9a6feb749f0533aaae2cc45090b0ed", + "sha1": "d004326c9d8060812b4433c3f07646b04854d6c4" + }, + "ps2:ps2-0150e-20001228.bin": { + "size": "4194304", + "crc": "1559fd43", + "md5": "838544f12de9b0abc90811279ee223c8", + "sha1": "e22ef231faf3661edd92f2ee449a71297c82a092" + }, + "ps2:ps2-0150jd-20010118.bin": { + "size": "4194304", + "crc": "0b3ec2bc", + "md5": "bb6bbc850458fff08af30e969ffd0175", + "sha1": "334e029fc7fd50222a399c50384ff42732652259" + }, + "ps2:ps2-0150j-20010118.bin": { + "size": "4194304", + "crc": "4fc3b495", + "md5": "815ac991d8bc3b364696bead3457de7d", + "sha1": "d6f365a0f07cd04ed28108e6ec5076e2f81e5f72" + }, + "ps2:ps2-0160a-20010427.bin": { + "size": "4194304", + "crc": "4008ac18", + "md5": "b107b5710042abe887c0f6175f6e94bb", + "sha1": "7331a40b4b4feb1b3f0f77b013b6d38483577baa" + }, + "ps2:ps2-0160j-20010427.bin": { + "size": "4194304", + "crc": "c268ef47", + "md5": "ab55cceea548303c22c72570cfd4dd71", + "sha1": "e525a0c900e37acf0ae5a655d82a0abcb07c6f1f" + }, + "ps2:ps2-0160a-20010704.bin": { + "size": "4194304", + "crc": "c506c693", + "md5": "18bcaadb9ff74ed3add26cdf709fff2e", + "sha1": "ce92e8e8c88665f2f645a9522e337823d47a914a" + }, + "ps2:ps2-0160e-20010704.bin": { + "size": "4194304", + "crc": "f1ac735f", + "md5": "491209dd815ceee9de02dbbc408c06d6", + "sha1": "3cbd048e437c785b5a05a0feced00117a8a42545" + }, + "ps2:ps2-0160a-20011004.bin": { + "size": "4194304", + "crc": "a01ec625", + "md5": "7200a03d51cacc4c14fcdfdbc4898431", + "sha1": "d257bce6ecaf3bafb704c75a1b4741b910bd2d49" + }, + "ps2:ps2-0160e-20011004.bin": { + "size": "4194304", + "crc": "82aa5055", + "md5": "8359638e857c8bc18c3c18ac17d9cc3c", + "sha1": "ee34c3a87c53c75ca2a37d77b0042ca24d07831f" + }, + "ps2:ps2-0160h-20010730.bin": { + "size": "4194304", + "crc": "75f83c67", + "md5": "352d2ff9b3f68be7e6fa7e6dd8389346", + "sha1": "ba15dcf7aac13864c08222037e9321d7468c87d1" + }, + "ps2:ps2-0160a-20020207.bin": { + "size": "4194304", + "crc": "a19e0bf5", + "md5": "d5ce2c7d119f563ce04bc04dbc3a323e", + "sha1": "f9a5d629a036b99128f7cb530c6e3ca016e9c8b7" + }, + "ps2:ps2-0160e-20020319.bin": { + "size": "4194304", + "crc": "2fe21e4d", + "md5": "0d2228e6fd4fb639c9c39d077a9ec10c", + "sha1": "bff2902bd0ce9729a060581132541e9fd1a9fab6" + }, + "ps2:ps2-0160j-20020426.bin": { + "size": "4194304", + "crc": "c9363baf", + "md5": "72da56fccb8fcd77bba16d1b6f479914", + "sha1": "003628c137dae577ff3b04b93ca1787b0c944702" + }, + "ps2:ps2-0160e-20020426.bin": { + "size": "4194304", + "crc": "dad0baec", + "md5": "5b1f47fbeb277c6be2fccdd6344ff2fd", + "sha1": "d106b757ae2544dfe63f7e1924e59d5ad44c0c29" + }, + "ps2:ps2-0160h-20020426.bin": { + "size": "4194304", + "crc": "3355623e", + "md5": "315a4003535dfda689752cb25f24785c", + "sha1": "e3a74125c426bcacabca00b513fab928665c8846" + }, + "ps2:ps2-0170j-20030206.bin": { + "size": "4194304", + "crc": "9457f64e", + "md5": "312ad4816c232a9606e56f946bc0678a", + "sha1": "d812ac65c357d392396ca9edee812dc41bed8bde" + }, + "ps2:ps2-0170ed-20030227.bin": { + "size": "4194304", + "crc": "970a9c56", + "md5": "666018ffec65c5c7e04796081295c6c7", + "sha1": "e220bb282378c1f48ea1b585b3675e51a6dca572" + }, + "ps2:ps2-0170e-20030227.bin": { + "size": "4194304", + "crc": "51b5fb8b", + "md5": "6e69920fa6eef8522a1d688a11e41bc6", + "sha1": "ad15bd7eabd5bd81ba011516a5be44947d6641aa" + }, + "ps2:ps2-0170ad-20030325.bin": { + "size": "4194304", + "crc": "0e1ece79", + "md5": "eb960de68f0c0f7f9fa083e9f79d0360", + "sha1": "c5bc6e893b4c43d528142e56c96073024de64157" + }, + "ps2:ps2-0170a-20030325.bin": { + "size": "4194304", + "crc": "9a99e3f4", + "md5": "8aa12ce243210128c5074552d3b86251", + "sha1": "d269d1ed513227f3ef7133c76cf1b3a64f97b15d" + }, + "ps2:ps2-0180cd-20030224.bin": { + "size": "4194304", + "crc": "8c1a04cf", + "md5": "240d4c5ddd4b54069bdc4a3cd2faf99d", + "sha1": "2de87767008fc4a303af64a46251156e965d9065" + }, + "ps2:ps2-0180j-20031028.bin": { + "size": "4194304", + "crc": "585fd27c", + "md5": "1c6cd089e6c83da618fbf2a081eb4888", + "sha1": "aa4a35c14ee342cf7a03b1dde294ca10e64889e1" + }, + "ps2:ps2-0190j-20030623.bin": { + "size": "4194304", + "crc": "7c10a967", + "md5": "463d87789c555a4a7604e97d7db545d1", + "sha1": "6a6ecfe6c10e42eff1ca056349def799b5629067" + }, + "ps2:ps2-0190a-20030623.bin": { + "size": "4194304", + "crc": "b3e87709", + "md5": "35461cecaa51712b300b2d6798825048", + "sha1": "c74d92a2952a2912b6698cbcf7742adac8f784d3" + }, + "ps2:ps2-0190e-20030623.bin": { + "size": "4194304", + "crc": "1752a52e", + "md5": "bd6415094e1ce9e05daabe85de807666", + "sha1": "18b9ba833c469c4683676cc20da5124080d980bb" + }, + "ps2:ps2-0190h-20030623.bin": { + "size": "4194304", + "crc": "41391dd3", + "md5": "2e70ad008d4ec8549aada8002fdf42fb", + "sha1": "caa18ed04854a91e68f2d61a782560edd6373bbf" + }, + "ps2:ps2-0190r-20030623.bin": { + "size": "4194304", + "crc": "25f6212a", + "md5": "b53d51edc7fc086685e31b811dc32aad", + "sha1": "34a81db03ab617fbfdd7f9b861692dd2ecd57b82" + }, + "ps2:ps2-0190c-20030623.bin": { + "size": "4194304", + "crc": "d2347ee7", + "md5": "1b6e631b536247756287b916f9396872", + "sha1": "92d9eb4b11cef97bb69a275b2851b72f7b0023d6" + }, + "ps2:ps2-0190j-20030822.bin": { + "size": "4194304", + "crc": "79d60546", + "md5": "00da1b177096cfd2532c8fa22b43e667", + "sha1": "0ea98a25a32145dda514de2f0d4bfbbd806bd00c" + }, + "ps2:ps2-0190e-20030822.bin": { + "size": "4194304", + "crc": "3afd1d1e", + "md5": "afde410bd026c16be605a1ae4bd651fd", + "sha1": "6e1f0eb4aec51a6288b3d802d3bcdb477cf52104" + }, + "ps2:ps2-0190a-20040329.bin": { + "size": "4194304", + "crc": "9ba4c32f", + "md5": "81f4336c1de607dd0865011c0447052e", + "sha1": "b68c05f5cd86bf03cb38a643a723b7a97b759531" + }, + "ps2:ps2-0200ed-20040614.bin": { + "size": "4194304", + "crc": "881c9aa9", + "md5": "63ead1d74893bf7f36880af81f68a82d", + "sha1": "902f4680b258abd40c0922f6b0d581cbd8f8a73e" + }, + "ps2:ps2-0200h-20040614.bin": { + "size": "4194304", + "crc": "b57201bf", + "md5": "3e3e030c0f600442fa05b94f87a1e238", + "sha1": "7f8e812cab7c7393c85eac6c42661e1fd0a642df" + }, + "ps2:ps2-0210j-20040917.bin": { + "size": "4194304", + "crc": "55710d11", + "md5": "1ad977bb539fc9448a08ab276a836bbc", + "sha1": "bbb1af3085e77599691ec430d147810157da934f" + }, + "ps2:ps2-0220j-20050620.bin": { + "size": "4194304", + "crc": "d27fc41d", + "md5": "eb4f40fcf4911ede39c1bbfe91e7a89a", + "sha1": "7ffa75d142cb8eeea6c777dbcf263143655275d5" + }, + "ps2:ps2-0220ad-20050620.bin": { + "size": "4194304", + "crc": "181f1bda", + "md5": "9959ad7a8685cad66206e7752ca23f8b", + "sha1": "7c7efdfcec7705f4e84bb47f45322104e39eed09" + }, + "ps2:ps2-0220a-20050620.bin": { + "size": "4194304", + "crc": "d305a97a", + "md5": "929a14baca1776b00869f983aa6e14d2", + "sha1": "48d0445dffd1e879c7ae752c5166ec3101921555" + }, + "ps2:ps2-0220e-20050620.bin": { + "size": "4194304", + "crc": "e2862e39", + "md5": "573f7d4a430c32b3cc0fd0c41e104bbd", + "sha1": "929a85e974faf4b40d0a7785023b758402c43bd9" + }, + "ps2:ps2-0220h-20050620.bin": { + "size": "4194304", + "crc": "e27c4a6c", + "md5": "df63a604e8bff5b0599bd1a6c2721bd0", + "sha1": "0a071d1b46607a7694770407606d8599f62a372b" + }, + "ps2:ps2-0220j-20060210.bin": { + "size": "4194304", + "crc": "1303918e", + "md5": "5b1ba4bb914406fae75ab8e38901684d", + "sha1": "0edf1fbb772a8e6a79ae00e977450e3ade25c4f3" + }, + "ps2:ps2-0220a-20060210.bin": { + "size": "4194304", + "crc": "1279fce9", + "md5": "cb801b7920a7d536ba07b6534d2433ca", + "sha1": "92e488d5b2705e4cca83d4d1efbc421012faf83e" + }, + "ps2:ps2-0220e-20060210.bin": { + "size": "4194304", + "crc": "23fa7baa", + "md5": "af60e6d1a939019d55e5b330d24b1c25", + "sha1": "28ad756d0cfd1e7b2e2de3de5d9e14207ee89761" + }, + "ps2:ps2-0220h-20060210.bin": { + "size": "4194304", + "crc": "23001fff", + "md5": "549a66d0c698635ca9fa3ab012da7129", + "sha1": "fce2a24e5e0400cc6d98c08f426405d19173813e" + }, + "ps2:ps2-0220j-20060905.bin": { + "size": "4194304", + "crc": "1d6d879b", + "md5": "5de9d0d730ff1e7ad122806335332524", + "sha1": "3baf847c1c217aa71ac6d298389c88edb3db32e2" + }, + "ps2:ps2-0220ad-20060905.bin": { + "size": "4194304", + "crc": "431d9b7f", + "md5": "21fe4cad111f7dc0f9af29477057f88d", + "sha1": "4191b5842f31a9985b5428bc9d2b733ce3abb583" + }, + "ps2:ps2-0220a-20060905.bin": { + "size": "4194304", + "crc": "1c17eafc", + "md5": "40c11c063b3b9409aa5e4058e984e30c", + "sha1": "8361d615cc895962e0f0838489337574dbdc9173" + }, + "ps2:ps2-0220e-20060905.bin": { + "size": "4194304", + "crc": "2d946dbf", + "md5": "80bbb237a6af9c611df43b16b930b683", + "sha1": "da5aacead2fb55807d6d4e70b1f10f4fdcfd3281" + }, + "ps2:ps2-0220h-20060905.bin": { + "size": "4194304", + "crc": "2d6e09ea", + "md5": "c37bce95d32b2be480f87dd32704e664", + "sha1": "a5a2ee0dd9a86ca35b94e97ca92476a584f755bf" + }, + "ps2:ps2-0230j-20080220.bin": { + "size": "4194304", + "crc": "2912faa5", + "md5": "80ac46fa7e77b8ab4366e86948e54f83", + "sha1": "fbd54bfc020af34008b317dcb80b812dd29b3759" + }, + "ps2:ps2-0230a-20080220.bin": { + "size": "4194304", + "crc": "286897c2", + "md5": "21038400dc633070a78ad53090c53017", + "sha1": "f9229fe159d0353b9f0632f3fdc66819c9030458" + }, + "ps2:ps2-0230e-20080220.bin": { + "size": "4194304", + "crc": "19eb1081", + "md5": "dc69f0643a3030aaa4797501b483d6c4", + "sha1": "9915b5ba56798f4027ac1bd8d10abe0c1c9c326a" + }, + "ps2:ps2-0230h-20080220.bin": { + "size": "4194304", + "crc": "191174d4", + "md5": "30d56e79d89fbddf10938fa67fe3f34e", + "sha1": "a277b456849697abec11285c6b35bc734598c220" + }, + "ps2:ps2-0250e-20100415.bin": { + "size": "4194304", + "crc": "7e75fc28", + "md5": "93ea3bcee4252627919175ff1b16a1d9", + "sha1": "b9cb5775af29cd4d1ec5521e8231f8b6636e2e44" + }, + "ps2:ps2-0250j-20100415.bin": { + "size": "4194304", + "crc": "4e8c160c", + "md5": "d3e81e95db25f5a86a7b7474550a2155", + "sha1": "4b5ef16b67e3b523d28ed2406106cb80470a06d0" + }, + "tvc:tvcfileio.rom": { + "size": "8192", + "crc": "950e32fd", + "md5": "a2cf86ba8e7fc58b242137fe59036832", + "sha1": "98889c3a56b11dedf077f866ed2e12d51b604113" + }, + "tvc:tvc22_ext.rom": { + "size": "8192", + "crc": "05e1c3a8", + "md5": "5ce95a26ceed5bec73995d83568da9cf", + "sha1": "abf119cf947ea32defd08b29a8a25d75f6bd4987" + }, + "tvc:tvc22_sys.rom": { + "size": "16384", + "crc": "79fa818c", + "md5": "8c54285f541930cde766069942bad0f2", + "sha1": "f2572ee83d09fc08f4de4a62f101c8bb301a9505" + }, + "tvc:tvc_dos12d.rom": { + "size": "16384", + "crc": "1466aed4", + "md5": "88dc7876d584f90e4106f91444ab23b7", + "sha1": "072c6160d4e7d406f5d8f5b1b66066c797d35561" } } diff --git a/backend/models/rom.py b/backend/models/rom.py index 06c8a3548..0f196a251 100644 --- a/backend/models/rom.py +++ b/backend/models/rom.py @@ -1,11 +1,19 @@ from __future__ import annotations from functools import cached_property -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypedDict from config import FRONTEND_RESOURCES_PATH from models.base import BaseModel -from sqlalchemy import JSON, BigInteger, ForeignKey, String, Text, UniqueConstraint +from sqlalchemy import ( + JSON, + BigInteger, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, +) from sqlalchemy.dialects.mysql.json import JSON as MySQLJSON from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -16,6 +24,12 @@ from models.user import User +class RomFile(TypedDict): + filename: str + size: int + last_modified: float | None + + class Rom(BaseModel): __tablename__ = "roms" @@ -59,13 +73,21 @@ class Rom(BaseModel): ) multi: Mapped[bool] = mapped_column(default=False) - files: Mapped[list[str] | None] = mapped_column(JSON, default=[]) + files: Mapped[list[RomFile] | None] = mapped_column(JSON, default=[]) + crc_hash: Mapped[str | None] = mapped_column(String(100)) + md5_hash: Mapped[str | None] = mapped_column(String(100)) + sha1_hash: Mapped[str | None] = mapped_column(String(100)) platform_id: Mapped[int] = mapped_column( ForeignKey("platforms.id", ondelete="CASCADE") ) platform: Mapped[Platform] = relationship(lazy="immediate") + sibling_roms: Mapped[list[Rom]] = relationship( + secondary="sibling_roms", + primaryjoin="Rom.id == SiblingRom.rom_id", + secondaryjoin="Rom.id == SiblingRom.sibling_rom_id", + ) saves: Mapped[list[Save]] = relationship(back_populates="rom") states: Mapped[list[State]] = relationship(back_populates="rom") @@ -98,12 +120,6 @@ def merged_screenshots(self) -> list[str]: f"{FRONTEND_RESOURCES_PATH}/{s}" for s in self.path_screenshots ] - # This is an expensive operation so don't call it on a list of roms - def get_sibling_roms(self) -> list[Rom]: - from handler.database import db_rom_handler - - return db_rom_handler.get_sibling_roms(self) - def get_collections(self) -> list[Collection]: from handler.database import db_rom_handler @@ -176,3 +192,14 @@ class RomUser(BaseModel): @property def user__username(self) -> str: return self.user.username + + +class SiblingRom(BaseModel): + __tablename__ = "sibling_roms" + + rom_id: Mapped[int] = mapped_column(Integer, primary_key=True) + sibling_rom_id: Mapped[int] = mapped_column(Integer, primary_key=True) + + __table_args__ = ( + UniqueConstraint("rom_id", "sibling_rom_id", name="unique_sibling_roms"), + ) diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py index 1529e8c9e..89d46d880 100644 --- a/backend/tasks/update_switch_titledb.py +++ b/backend/tasks/update_switch_titledb.py @@ -1,4 +1,5 @@ import json +from itertools import batched from typing import Final from config import ( @@ -9,7 +10,6 @@ from logger.logger import log from tasks.tasks import RemoteFilePullTask from utils.context import initialize_context -from utils.iterators import batched SWITCH_TITLEDB_INDEX_KEY: Final = "romm:switch_titledb" SWITCH_PRODUCT_ID_KEY: Final = "romm:switch_product_id" diff --git a/backend/utils/filesystem.py b/backend/utils/filesystem.py index 3db3fabde..1cc0bfd54 100644 --- a/backend/utils/filesystem.py +++ b/backend/utils/filesystem.py @@ -1,4 +1,5 @@ import os +import re from collections.abc import Iterator from pathlib import Path @@ -27,3 +28,36 @@ def iter_directories(path: str, recursive: bool = False) -> Iterator[tuple[Path, yield Path(root), directory if not recursive: break + + +INVALID_CHARS_HYPHENS = re.compile(r"[\\/:|]") +INVALUD_CHARS_EMPTY = re.compile(r'[*?"<>]') + + +def sanitize_filename(filename): + """ + Replace invalid characters in the filename to make it valid across common filesystems + + Args: + - filename (str): The filename to sanitize. + + Returns: + - str: The sanitized filename. + """ + # Replace some invalid characters with hyphen + sanitized_filename = INVALID_CHARS_HYPHENS.sub("-", filename) + + # Remove other invalid characters + sanitized_filename = INVALUD_CHARS_EMPTY.sub("", sanitized_filename) + + # Ensure null bytes are not included (ZFS allows any characters except null bytes) + sanitized_filename = sanitized_filename.replace("\0", "") + + # Remove leading/trailing whitespace + sanitized_filename = sanitized_filename.strip() + + # Ensure the filename is not empty + if not sanitized_filename: + raise ValueError("Filename cannot be empty after sanitization") + + return sanitized_filename diff --git a/backend/utils/hashing.py b/backend/utils/hashing.py new file mode 100644 index 000000000..a47340474 --- /dev/null +++ b/backend/utils/hashing.py @@ -0,0 +1,2 @@ +def crc32_to_hex(value: int) -> str: + return (value & 0xFFFFFFFF).to_bytes(4, byteorder="big").hex() diff --git a/backend/utils/iterators.py b/backend/utils/iterators.py deleted file mode 100644 index 324503f09..000000000 --- a/backend/utils/iterators.py +++ /dev/null @@ -1,17 +0,0 @@ -import sys - -if sys.version_info >= (3, 12): - from itertools import batched # noqa: F401 -else: - from collections.abc import Iterable, Iterator - from itertools import islice - from typing import TypeVar - - T = TypeVar("T") - - def batched(iterable: Iterable[T], n: int) -> Iterator[tuple[T, ...]]: - if n < 1: - raise ValueError("n must be at least one") - iterator = iter(iterable) - while batch := tuple(islice(iterator, n)): - yield batch diff --git a/backend/utils/nginx.py b/backend/utils/nginx.py new file mode 100644 index 000000000..56d6d4758 --- /dev/null +++ b/backend/utils/nginx.py @@ -0,0 +1,49 @@ +import dataclasses +from collections.abc import Collection +from typing import Any + +from fastapi.responses import Response + + +@dataclasses.dataclass(frozen=True) +class ZipContentLine: + """Dataclass for lines returned in the response body, for usage with the `mod_zip` module. + + Reference: + https://github.com/evanmiller/mod_zip?tab=readme-ov-file#usage + """ + + crc32: str | None + size_bytes: int + encoded_location: str + filename: str + + def __str__(self) -> str: + crc32 = self.crc32 or "-" + return f"{crc32} {self.size_bytes} {self.encoded_location} {self.filename}" + + +class ZipResponse(Response): + """Response class for returning a ZIP archive with multiple files, using the `mod_zip` module.""" + + def __init__( + self, + *, + content_lines: Collection[ZipContentLine], + filename: str, + **kwargs: Any, + ): + if kwargs.get("content"): + raise ValueError( + "Argument 'content' must not be provided, as it is generated from 'content_lines'" + ) + + kwargs["content"] = "\n".join(str(line) for line in content_lines) + kwargs.setdefault("headers", {}).update( + { + "Content-Disposition": f'attachment; filename="{filename}"', + "X-Archive-Files": "zip", + } + ) + + super().__init__(**kwargs) diff --git a/backend/utils/router.py b/backend/utils/router.py index 9abb9bf45..e9bfd1023 100644 --- a/backend/utils/router.py +++ b/backend/utils/router.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from fastapi import APIRouter as FastAPIRouter from fastapi.types import DecoratedCallable @@ -29,7 +30,9 @@ def api_route( ) def decorator(func: DecoratedCallable) -> DecoratedCallable: + # Path without trailing slash is registered first, for router's `url_path_for` to prefer it. + result = add_path(func) add_alternate_path(func) - return add_path(func) + return result return decorator diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f2c3a966..2ee544689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ -ARG ALPINE_VERSION=3.19 -ARG NGINX_VERSION=1.27.0 -ARG NODE_VERSION=lts -ARG PYTHON_VERSION=3.11 +ARG ALPINE_VERSION=3.20 +ARG NGINX_VERSION=1.27.1 +ARG NODE_VERSION=20.16 +ARG PYTHON_VERSION=3.12 # Build frontend FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage @@ -33,13 +33,46 @@ WORKDIR /src COPY ./pyproject.toml ./poetry.lock /src/ RUN poetry install --no-ansi --no-cache --only main +# Build nginx modules +FROM alpine:${ALPINE_VERSION} AS nginx-build + +RUN apk add --no-cache \ + gcc \ + git \ + libc-dev \ + make \ + pcre-dev \ + zlib-dev + +ARG NGINX_VERSION +# The specified commit SHA is the latest commit on the `master` branch at the time of writing. +# It includes a fix to correctly calculate CRC-32 checksums when using upstream subrequests. +# TODO: Move to a tagged release of `mod_zip`, once a version newer than 1.3.0 is released. +ARG NGINX_MOD_ZIP_SHA=8e65b82c82c7890f67a6107271c127e9881b6313 + +# Clone both nginx and `ngx_http_zip_module` repositories, needed to compile the module from source. +# This is needed to be able to dinamically load it as a module in the final image. `nginx` Docker +# images do not have a simple way to include third-party modules. +RUN git clone https://github.com/evanmiller/mod_zip.git && \ + cd ./mod_zip && \ + git checkout "${NGINX_MOD_ZIP_SHA}" && \ + cd ../ && \ + git clone --branch "release-${NGINX_VERSION}" --depth 1 https://github.com/nginx/nginx.git && \ + cd ./nginx && \ + ./auto/configure --with-compat --add-dynamic-module=../mod_zip/ && \ + make -f ./objs/Makefile modules && \ + chmod 644 ./objs/ngx_http_zip_module.so + # Setup frontend and backend -FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION}-slim AS production-stage +FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION} AS production-stage ARG WEBSERVER_FOLDER=/var/www/html +COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/ + COPY --from=front-build-stage /front/dist ${WEBSERVER_FOLDER} COPY ./frontend/assets/default ${WEBSERVER_FOLDER}/assets/default COPY ./frontend/assets/emulatorjs ${WEBSERVER_FOLDER}/assets/emulatorjs +COPY ./frontend/assets/ruffle ${WEBSERVER_FOLDER}/assets/ruffle COPY ./frontend/assets/scrappers ${WEBSERVER_FOLDER}/assets/scrappers COPY ./frontend/assets/platforms ${WEBSERVER_FOLDER}/assets/platforms COPY ./frontend/assets/webrcade/feed ${WEBSERVER_FOLDER}/assets/webrcade/feed @@ -53,12 +86,15 @@ RUN apk add --no-cache \ mariadb-connector-c \ python3 \ tzdata \ - redis + redis \ + libmagic \ + p7zip COPY ./backend /backend # Setup init script and config files COPY ./docker/init_scripts/* / +COPY ./docker/nginx/js/ /etc/nginx/js/ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf # User permissions diff --git a/docker/init_scripts/init b/docker/init_scripts/init index b071b9aec..fb44bc8dd 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -52,6 +52,7 @@ start_bin_gunicorn() { --bind=0.0.0.0:5000 \ --bind=unix:/tmp/gunicorn.sock \ --pid=/tmp/gunicorn.pid \ + --forwarded-allow-ips="*" \ --workers "${GUNICORN_WORKERS:=2}" \ main:app & } diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index 5565cad7b..59420be46 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -1,3 +1,6 @@ +load_module modules/ngx_http_js_module.so; +load_module modules/ngx_http_zip_module.so; + worker_processes auto; pid /tmp/nginx.pid; @@ -14,9 +17,14 @@ http { scgi_temp_path /tmp/scgi; sendfile on; + client_body_buffer_size 128k; client_max_body_size 0; + client_header_buffer_size 1k; + large_client_header_buffers 4 16k; + send_timeout 60s; + keepalive_timeout 65s; tcp_nopush on; - # types_hash_max_size 2048; + tcp_nodelay on; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -24,6 +32,8 @@ http { ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE ssl_prefer_server_ciphers on; + js_import /etc/nginx/js/decode.js; + map $time_iso8601 $date { ~([^+]+)T $1; } @@ -41,6 +51,13 @@ http { error_log /dev/stderr; gzip on; + gzip_proxied any; + gzip_vary on; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_min_length 1024; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; # include /etc/nginx/conf.d/*.conf; # include /etc/nginx/sites-enabled/*; @@ -87,5 +104,41 @@ http { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } + + # Internally redirect download requests + location /library { + internal; + alias /romm/library; + } + + # This location, and the related server at port 8081, are used to serve files when + # using the `mod_zip` module. This is because the `mod_zip` module does not support + # calculating CRC-32 values when using subrequests pointing directly to internal + # locations that access the filesystem. + # TODO: If that gets fixed, this workaround can be removed, and the `/library` location + # can be used directly (also removing the server at port 8081). + # Related issue: https://github.com/evanmiller/mod_zip/issues/90 + location /library-zip { + internal; + rewrite ^/library-zip/(.*)$ /library/$1 break; + proxy_pass http://localhost:8081; + # Proxy buffering must be disabled, for the module to correctly calculate CRC-32 values. + proxy_buffering off; + } + + # Internal decoding endpoint, used to decode base64 encoded data + location /decode { + internal; + js_content decode.decodeBase64; + } + } + + server { + listen 8081; + server_name localhost; + + location /library { + alias /romm/library; + } } } diff --git a/docker/nginx/js/decode.js b/docker/nginx/js/decode.js new file mode 100644 index 000000000..b371fb179 --- /dev/null +++ b/docker/nginx/js/decode.js @@ -0,0 +1,19 @@ +// Decode a Base64 encoded string received as a query parameter named 'value', +// and return the decoded value in the response body. +function decodeBase64(r) { + var encodedValue = r.args.value; + + if (!encodedValue) { + r.return(400, "Missing 'value' query parameter"); + return; + } + + try { + var decodedValue = atob(encodedValue); + r.return(200, decodedValue); + } catch (e) { + r.return(400, "Invalid Base64 encoding"); + } +} + +export default { decodeBase64 }; diff --git a/env.template b/env.template index 63035ca63..76dd91517 100644 --- a/env.template +++ b/env.template @@ -2,7 +2,6 @@ ROMM_BASE_PATH=/path/to/romm_mock VITE_BACKEND_DEV_PORT=5000 # Gunicorn (optional) -ROMM_HOST=localhost GUNICORN_WORKERS=4 # (2 × CPU cores) + 1 # IGDB credentials @@ -39,3 +38,7 @@ ENABLE_SCHEDULED_RESCAN=true SCHEDULED_RESCAN_CRON=0 3 * * * ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * * + +# In-browser emulation +DISABLE_EMULATOR_JS=false +DISABLE_RUFFLE_RS=false diff --git a/frontend/assets/platforms/pico.ico b/frontend/assets/platforms/pico.ico new file mode 100644 index 000000000..e886acaab Binary files /dev/null and b/frontend/assets/platforms/pico.ico differ diff --git a/frontend/assets/platforms/sega-master-system.ico b/frontend/assets/platforms/sega-master-system.ico new file mode 100644 index 000000000..a64f3177c Binary files /dev/null and b/frontend/assets/platforms/sega-master-system.ico differ diff --git a/frontend/assets/ruffle/powered_by_ruffle.png b/frontend/assets/ruffle/powered_by_ruffle.png new file mode 100644 index 000000000..9c0694fe1 Binary files /dev/null and b/frontend/assets/ruffle/powered_by_ruffle.png differ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 46fb8fb30..4146fc6c8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,10 +11,11 @@ "license": "GPL-3.0-only", "dependencies": { "@mdi/font": "7.0.96", + "@ruffle-rs/ruffle": "^0.1.0-nightly.2024.7.29", "axios": "^1.7.4", "core-js": "^3.37.1", "cronstrue": "^2.50.0", - "emulatorjs": "github:emulatorjs/emulatorjs#v4.0.12", + "emulatorjs": "github:emulatorjs/emulatorjs#v4.1.1", "file-saver": "^2.0.5", "js-cookie": "^3.0.5", "jszip": "^3.10.1", @@ -2582,6 +2583,11 @@ } } }, + "node_modules/@ruffle-rs/ruffle": { + "version": "0.1.0-nightly.2024.7.29", + "resolved": "https://registry.npmjs.org/@ruffle-rs/ruffle/-/ruffle-0.1.0-nightly.2024.7.29.tgz", + "integrity": "sha512-hChElD2KhZgwP8jkGik4Dl92c1HWAAgu81w+6VBZ0I6MFXJOCEiECjvoj1jrEgAs/pVdordLrAOBqw+v2UDhpA==" + }, "node_modules/@socket.io/component-emitter": { "version": "3.1.2", "license": "MIT" @@ -3219,7 +3225,8 @@ }, "node_modules/async": { "version": "2.6.4", - "license": "MIT", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", "dependencies": { "lodash": "^4.17.14" } @@ -3251,9 +3258,9 @@ } }, "node_modules/axios": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", - "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.5.tgz", + "integrity": "sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -3311,7 +3318,8 @@ }, "node_modules/basic-auth": { "version": "2.0.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", "dependencies": { "safe-buffer": "5.1.2" }, @@ -3618,7 +3626,8 @@ }, "node_modules/corser": { "version": "2.0.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", "engines": { "node": ">= 0.4.0" } @@ -3837,9 +3846,8 @@ }, "node_modules/emulatorjs": { "name": "@emulatorjs/emulatorjs", - "version": "4.0.11", - "resolved": "git+ssh://git@github.com/emulatorjs/emulatorjs.git#62c0424e68c3382932237b36b0b03b56e7171c1e", - "license": "GPL-3.0", + "version": "4.0.12", + "resolved": "git+ssh://git@github.com/emulatorjs/emulatorjs.git#a7b59f19a3e9415c2a42121461da3df03b25d3b5", "dependencies": { "http-server": "^14.1.1" } @@ -4250,7 +4258,8 @@ }, "node_modules/eventemitter3": { "version": "4.0.7", - "license": "MIT" + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" }, "node_modules/fast-deep-equal": { "version": "3.1.3", @@ -4746,7 +4755,8 @@ }, "node_modules/html-encoding-sniffer": { "version": "3.0.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "dependencies": { "whatwg-encoding": "^2.0.0" }, @@ -4756,7 +4766,8 @@ }, "node_modules/http-proxy": { "version": "1.18.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", "dependencies": { "eventemitter3": "^4.0.0", "follow-redirects": "^1.0.0", @@ -4768,7 +4779,8 @@ }, "node_modules/http-server": { "version": "14.1.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", "dependencies": { "basic-auth": "^2.0.1", "chalk": "^4.1.2", @@ -4793,7 +4805,8 @@ }, "node_modules/iconv-lite": { "version": "0.6.3", - "license": "MIT", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -5502,7 +5515,8 @@ }, "node_modules/mime": { "version": "1.6.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", "bin": { "mime": "cli.js" }, @@ -5551,7 +5565,8 @@ }, "node_modules/mkdirp": { "version": "0.5.6", - "license": "MIT", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dependencies": { "minimist": "^1.2.6" }, @@ -5675,7 +5690,8 @@ }, "node_modules/opener": { "version": "1.5.2", - "license": "(WTFPL OR MIT)", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", "bin": { "opener": "bin/opener-bin.js" } @@ -5914,7 +5930,8 @@ }, "node_modules/portfinder": { "version": "1.0.32", - "license": "MIT", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.32.tgz", + "integrity": "sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg==", "dependencies": { "async": "^2.6.4", "debug": "^3.2.7", @@ -5926,7 +5943,8 @@ }, "node_modules/portfinder/node_modules/debug": { "version": "3.2.7", - "license": "MIT", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "dependencies": { "ms": "^2.1.1" } @@ -6035,8 +6053,9 @@ } }, "node_modules/qs": { - "version": "6.12.1", - "license": "BSD-3-Clause", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dependencies": { "side-channel": "^1.0.6" }, @@ -6188,7 +6207,8 @@ }, "node_modules/requires-port": { "version": "1.0.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" }, "node_modules/resolve": { "version": "1.22.8", @@ -6338,7 +6358,8 @@ }, "node_modules/safer-buffer": { "version": "2.1.2", - "license": "MIT" + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/sass": { "version": "1.77.1", @@ -6358,7 +6379,8 @@ }, "node_modules/secure-compare": { "version": "3.0.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==" }, "node_modules/semver": { "version": "7.6.2", @@ -6969,6 +6991,8 @@ }, "node_modules/union": { "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", "dependencies": { "qs": "^6.4.0" }, @@ -7043,7 +7067,8 @@ }, "node_modules/url-join": { "version": "4.0.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==" }, "node_modules/util-deprecate": { "version": "1.0.2", @@ -7331,7 +7356,8 @@ }, "node_modules/whatwg-encoding": { "version": "2.0.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", "dependencies": { "iconv-lite": "0.6.3" }, diff --git a/frontend/package.json b/frontend/package.json index f216e0ff6..c0f3e7c68 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -29,10 +29,11 @@ }, "dependencies": { "@mdi/font": "7.0.96", + "@ruffle-rs/ruffle": "^0.1.0-nightly.2024.7.29", "axios": "^1.7.4", "core-js": "^3.37.1", "cronstrue": "^2.50.0", - "emulatorjs": "github:emulatorjs/emulatorjs#v4.0.12", + "emulatorjs": "github:emulatorjs/emulatorjs#v4.1.1", "file-saver": "^2.0.5", "js-cookie": "^3.0.5", "jszip": "^3.10.1", diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 5e38ff7a3..001e2c1ff 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,5 +1,6 @@ diff --git a/frontend/src/components/Management/Dialog/CreatePlatformVersion.vue b/frontend/src/components/Management/Dialog/CreatePlatformVersion.vue index e447fd20b..c375864d2 100644 --- a/frontend/src/components/Management/Dialog/CreatePlatformVersion.vue +++ b/frontend/src/components/Management/Dialog/CreatePlatformVersion.vue @@ -42,10 +42,10 @@ emitter?.on( }); fsSlugToCreate.value = fsSlug; selectedPlatform.value = supportedPlatforms.value?.find( - (platform) => platform.slug == slug + (platform) => platform.slug == slug, ); show.value = true; - } + }, ); // Functions @@ -60,7 +60,7 @@ function addVersionPlatform() { if (selectedPlatform.value) { configStore.addPlatformBinding( fsSlugToCreate.value, - selectedPlatform.value.slug + selectedPlatform.value.slug, ); } }) @@ -136,6 +136,7 @@ function closeDialog() { :key="item.raw.slug" :size="35" :slug="item.raw.slug" + :name="item.raw.name" /> @@ -147,6 +148,7 @@ function closeDialog() { :size="35" :key="item.raw.slug" :slug="item.raw.slug" + :name="item.raw.name" /> diff --git a/frontend/src/components/common/Game/Card/ActionBar.vue b/frontend/src/components/common/Game/Card/ActionBar.vue index 618f54621..eaffbdbff 100644 --- a/frontend/src/components/common/Game/Card/ActionBar.vue +++ b/frontend/src/components/common/Game/Card/ActionBar.vue @@ -1,20 +1,34 @@ diff --git a/frontend/src/components/common/Game/Dialog/UploadRom.vue b/frontend/src/components/common/Game/Dialog/UploadRom.vue index 886139fd8..5e210eaf7 100644 --- a/frontend/src/components/common/Game/Dialog/UploadRom.vue +++ b/frontend/src/components/common/Game/Dialog/UploadRom.vue @@ -6,6 +6,7 @@ import romApi from "@/services/api/rom"; import socket from "@/services/socket"; import storeHeartbeat from "@/stores/heartbeat"; import { type Platform } from "@/stores/platforms"; +import storeUpload from "@/stores/upload"; import storeScanning from "@/stores/scanning"; import type { Events } from "@/types/emitter"; import { formatBytes } from "@/utils"; @@ -16,11 +17,12 @@ import { useDisplay } from "vuetify"; // Props const { xs, mdAndUp, smAndUp } = useDisplay(); const show = ref(false); -const romsToUpload = ref([]); +const filesToUpload = ref([]); const scanningStore = storeScanning(); const selectedPlatform = ref(null); const supportedPlatforms = ref(); const heartbeat = storeHeartbeat(); +const uploadStore = storeUpload(); const HEADERS = [ { title: "Name", @@ -50,9 +52,7 @@ emitter?.on("showUploadRomDialog", (platformWhereUpload) => { }) .catch(({ response, message }) => { emitter?.emit("snackbarShow", { - msg: `Unable to upload roms: ${ - response?.data?.detail || response?.statusText || message - }`, + msg: `Unable to upload roms: ${response?.data?.detail || response?.statusText || message}`, icon: "mdi-close-circle", color: "red", timeout: 4000, @@ -64,7 +64,6 @@ emitter?.on("showUploadRomDialog", (platformWhereUpload) => { async function uploadRoms() { if (!selectedPlatform.value) return; show.value = false; - scanningStore.set(true); if (selectedPlatform.value.id == -1) { await platformApi @@ -93,36 +92,40 @@ async function uploadRoms() { } const platformId = selectedPlatform.value.id; - emitter?.emit("snackbarShow", { - msg: `Uploading ${romsToUpload.value.length} roms to ${selectedPlatform.value.name}...`, - icon: "mdi-loading mdi-spin", - color: "romm-accent-1", - }); await romApi .uploadRoms({ - romsToUpload: romsToUpload.value, + filesToUpload: filesToUpload.value, platformId: platformId, }) - .then(({ data }) => { - const { uploaded_roms, skipped_roms } = data; + .then((responses: PromiseSettledResult[]) => { + const successfulUploads = responses.filter( + (d) => d.status == "fulfilled", + ); + const failedUploads = responses.filter((d) => d.status == "rejected"); + + if (failedUploads.length == 0) { + uploadStore.clearAll(); + } - if (uploaded_roms.length == 0) { + if (successfulUploads.length == 0) { return emitter?.emit("snackbarShow", { msg: `All files skipped, nothing to upload.`, icon: "mdi-close-circle", color: "orange", - timeout: 2000, + timeout: 5000, }); } emitter?.emit("snackbarShow", { - msg: `${uploaded_roms.length} files uploaded successfully (and ${skipped_roms.length} skipped). Starting scan...`, + msg: `${successfulUploads.length} files uploaded successfully (and ${failedUploads.length} skipped/failed). Starting scan...`, icon: "mdi-check-bold", color: "green", - timeout: 2000, + timeout: 3000, }); + scanningStore.set(true); + if (!socket.connected) socket.connect(); setTimeout(() => { socket.emit("scan", { @@ -134,15 +137,13 @@ async function uploadRoms() { }) .catch(({ response, message }) => { emitter?.emit("snackbarShow", { - msg: `Unable to upload roms: ${ - response?.data?.detail || response?.statusText || message - }`, + msg: `Unable to upload roms: ${response?.data?.detail || response?.statusText || message}`, icon: "mdi-close-circle", color: "red", timeout: 4000, }); }); - romsToUpload.value = []; + filesToUpload.value = []; selectedPlatform.value = null; } @@ -152,17 +153,19 @@ function triggerFileInput() { } function removeRomFromList(romName: string) { - romsToUpload.value = romsToUpload.value.filter((rom) => rom.name !== romName); + filesToUpload.value = filesToUpload.value.filter( + (rom) => rom.name !== romName, + ); } function closeDialog() { show.value = false; - romsToUpload.value = []; + filesToUpload.value = []; selectedPlatform.value = null; } function updateDataTablePages() { - pageCount.value = Math.ceil(romsToUpload.value.length / itemsPerPage.value); + pageCount.value = Math.ceil(filesToUpload.value.length / itemsPerPage.value); } watch(itemsPerPage, async () => { updateDataTablePages(); @@ -201,6 +204,7 @@ watch(itemsPerPage, async () => { :key="item.raw.slug" :size="35" :slug="item.raw.slug" + :name="item.raw.name" /> @@ -212,6 +216,7 @@ watch(itemsPerPage, async () => { :size="35" :key="item.raw.slug" :slug="item.raw.slug" + :name="item.raw.name" /> @@ -232,7 +237,7 @@ watch(itemsPerPage, async () => { {