From 81c045da0db7423cff1c33f03414933d5ba66563 Mon Sep 17 00:00:00 2001 From: Zurdi <34356590+zurdi15@users.noreply.github.com> Date: Wed, 26 Apr 2023 09:07:30 +0200 Subject: [PATCH 01/45] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1fda9e5c9..dd0b784d1 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ RomM will try to find the structure 1 and if it doesn't exists, RomM will try to ## ⚙️ Config.yml file -RomM can be configured through a yml file. This is used to exclude platforms and/or roms to be scanned. +RomM can be configured through a yml file. For a configuration change to take effect, RomM must be restarted. From d0aaea695cb2db5d72253ea564ffc7323af9f424 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 11:20:00 +0200 Subject: [PATCH 02/45] igdb api calls refactored --- backend/src/endpoints/rom.py | 2 +- backend/src/endpoints/search.py | 2 +- backend/src/handler/igdb_handler.py | 177 ++++++++++++---------------- backend/src/utils/fastapi.py | 7 +- backend/src/utils/fs.py | 2 +- 5 files changed, 81 insertions(+), 109 deletions(-) diff --git a/backend/src/endpoints/rom.py b/backend/src/endpoints/rom.py index 58b1c5493..91f02d6ec 100644 --- a/backend/src/endpoints/rom.py +++ b/backend/src/endpoints/rom.py @@ -39,7 +39,7 @@ async def updateRom(req: Request, p_slug: str, id: int) -> dict: log.error(error) raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error) updated_rom['file_name_no_tags'] = get_file_name_with_no_tags(updated_rom['file_name']) - updated_rom.update(fs.get_cover_details(True, p_slug, updated_rom['file_name'], updated_rom['url_cover'])) + updated_rom.update(fs.get_cover(True, p_slug, updated_rom['file_name'], updated_rom['url_cover'])) dbh.update_rom(id, updated_rom) return {'data': dbh.get_rom(id), 'msg': f"{updated_rom['file_name']} updated successfully!"} diff --git a/backend/src/endpoints/search.py b/backend/src/endpoints/search.py index 655027728..8e90e6e06 100644 --- a/backend/src/endpoints/search.py +++ b/backend/src/endpoints/search.py @@ -16,7 +16,7 @@ async def search_rom_igdb(req: Request, igdb_id: str = None) -> dict: log.info(emoji.emojize(":magnifying_glass_tilted_right: IGDB Searching")) if igdb_id: log.info(f"Searching by id: {igdb_id}") - matched_roms = igdbh.get_matched_roms_by_id(igdb_id) + matched_roms = igdbh.get_matched_rom_by_id(igdb_id) else: log.info(emoji.emojize(f":video_game: {rom['p_slug']}: {COLORS['orange']}{rom['file_name']}{COLORS['reset']}")) matched_roms = igdbh.get_matched_roms(rom['file_name'], rom['p_igdb_id'], rom['p_slug']) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index dd6f67f10..1b2dcda7d 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -1,17 +1,21 @@ import sys import functools -import unidecode +from unidecode import unidecode as uc from time import time import requests from config import CLIENT_ID, CLIENT_SECRET, DEFAULT_URL_COVER_L -from utils import get_file_name_with_no_tags +from utils import get_file_name_with_no_tags as get_search_term from logger.logger import log class IGDBHandler(): def __init__(self) -> None: + base_url: str = 'https://api.igdb.com/v4' + self.platform_url: str = f'{base_url}/platforms/' + self.games_url: str = f'{base_url}/games/' + self.covers_url: str = f'{base_url}/covers/' self.twitch_auth: TwitchAuth = TwitchAuth() self.headers = { 'Client-ID': self.twitch_auth.client_id, @@ -26,17 +30,32 @@ def wrapper(*args): args[0].headers['Authorization'] = f'Bearer {args[0].twitch_auth.get_oauth_token()}' return func(*args) return wrapper + + + def _search_rom_by_category(self, search_term: str, p_igdb_id: str, category: int = None) -> dict: + category_query: str = f"& category={category}" if category else "" + try: + return requests.post(self.games_url, headers=self.headers, + data=f"search \"{search_term}\"; \ + fields id, slug, name, summary; \ + where platforms=[{p_igdb_id}] {category_query};").json()[0] + except IndexError: + return {} + + + def _search_cover(self, rom_id: str) -> str: + res: dict = requests.post(self.covers_url, headers=self.headers, + data=f"fields url; where game={rom_id};").json()[0] + return f"https:{res['url']}" if 'url' in res.keys() else "" @check_twitch_token - def get_platform_details(self, slug: str) -> tuple: - igdb_id: str = "" - name: str = "" + def get_platform(self, slug: str) -> tuple: try: - res_details: dict = requests.post("https://api.igdb.com/v4/platforms/", headers=self.headers, - data=f"fields id, name; where slug=\"{slug}\";").json()[0] - igdb_id = res_details['id'] - name = res_details['name'] + res: dict = requests.post(self.platform_url, headers=self.headers, + data=f"fields id, name; where slug=\"{slug}\";").json()[0] + igdb_id = res['id'] if 'id' in res.keys() else "" + name = res['name'] if 'name' in res.keys() else "" except IndexError: log.warning(f"{slug} not found in IGDB") if not name: name = slug @@ -44,111 +63,62 @@ def get_platform_details(self, slug: str) -> tuple: @check_twitch_token - def get_rom_details(self, file_name: str, p_igdb_id: int, r_igdb_id_search: str) -> dict: - search_term: str = unidecode.unidecode(get_file_name_with_no_tags(file_name)) - r_igdb_id: str = "" - r_slug: str = "" - r_name: str = "" - summary: str = "" - url_cover: str = "" - - if r_igdb_id_search: - res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"fields id, slug, name, summary; where id={r_igdb_id_search};").json()[0] - r_igdb_id = res_details['id'] - r_slug = res_details['slug'] - r_name = res_details['name'] - try: - summary = res_details['summary'] - except KeyError: - pass - - else: #TODO: improve API calls to make only one - if p_igdb_id: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{search_term}\";fields id, slug, name, summary; where platforms=[{p_igdb_id}] & category=0;").json()[0] - r_igdb_id = res_details['id'] - r_slug = res_details['slug'] - r_name = res_details['name'] - try: - summary = res_details['summary'] - except KeyError: - pass - except IndexError: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}] & category=10;").json()[0] - r_igdb_id = res_details['id'] - r_slug = res_details['slug'] - r_name = res_details['name'] - try: - summary = res_details['summary'] - except KeyError: - pass - except IndexError: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json()[0] - r_igdb_id = res_details['id'] - r_slug = res_details['slug'] - r_name = res_details['name'] - try: - summary = res_details['summary'] - except KeyError: - pass - except IndexError: - log.warning(f"{file_name} not found in IGDB") - if r_igdb_id: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/covers/", headers=self.headers, - data=f"fields url; where game={r_igdb_id};").json()[0] - url_cover: str = f"https:{res_details['url']}" - except IndexError: - log.warning(f"{r_name} cover not found in IGDB") - if not r_name: r_name = search_term - return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': url_cover} + def get_rom_by_id(self, r_igdb_id: str) -> dict: + res: dict = requests.post(self.games_url, headers=self.headers, + data=f"fields slug, name, summary; where id=[{r_igdb_id}];").json()[0] + r_slug = res['slug'] if 'slug' in res.keys() else "" + r_name = res['name'] if 'name' in res.keys() else "" + summary = res['summary'] if 'summary' in res.keys() else "" + return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id)} + + @check_twitch_token + def get_rom(self, file_name: str, p_igdb_id: int) -> dict: + search_term: str = uc(get_search_term(file_name)) + res = (self._search_rom_by_category(search_term, p_igdb_id, 0) or + self._search_rom_by_category(search_term, p_igdb_id, 10) or + self._search_rom_by_category(search_term, p_igdb_id)) + + r_igdb_id = res['id'] if 'id' in res.keys() else "" + r_slug = res['slug'] if 'slug' in res.keys() else "" + r_name = res['name'] if 'name' in res.keys() else "" + summary = res['summary'] if 'summary' in res.keys() else "" + + if not r_name: r_name = search_term + if not r_igdb_id: log.warning(f"{r_name} not found in IGDB") + return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id)} + @check_twitch_token - def get_matched_roms(self, file_name: str, p_igdb_id: int, p_slug: str) -> list: - matched_roms: list[dict] = [] - if p_igdb_id != '': - search_term: str = unidecode.unidecode(get_file_name_with_no_tags(file_name)) - matched_roms: list = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"search \"{search_term}\";fields name, id, slug, summary; where platforms=[{p_igdb_id}];").json() - for rom in matched_roms: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/covers/", headers=self.headers, - data=f"fields url; where game={rom['id']};").json()[0] - rom['url_cover'] = f"https:{res_details['url']}".replace('t_thumb', f't_cover_big') - except IndexError: - rom['url_cover'] = DEFAULT_URL_COVER_L + def get_matched_rom_by_id(self, igdb_id: str) -> list: + matched_rom: list[dict] = [] + res: list = requests.post(self.games_url, headers=self.headers, + data=f"fields name, id, slug, summary; where id={igdb_id};") + if res.status_code == 200: + matched_rom = res.json() + for rom in matched_rom: + rom['url_cover'] = self._search_cover(rom['id']).replace('t_thumb', f't_cover_big') rom['r_igdb_id'] = rom.pop('id') rom['r_slug'] = rom.pop('slug') rom['r_name'] = rom.pop('name') - else: - log.warning(f"{p_slug} is not supported!") - return matched_roms + return matched_rom - - def get_matched_roms_by_id(self, igdb_id: str) -> list: - res: list = requests.post("https://api.igdb.com/v4/games/", headers=self.headers, - data=f"fields name, id, slug, summary; where id={igdb_id};") - if res.status_code == 200: - matched_roms = res.json() + + @check_twitch_token + def get_matched_roms(self, file_name: str, p_igdb_id: int, p_slug: str) -> list: + matched_roms: list[dict] = [] + if p_igdb_id: + matched_roms: list = requests.post(self.games_url, headers=self.headers, + data=f"search \"{uc(get_search_term(file_name))}\"; \ + fields name, id, slug, summary; \ + where platforms=[{p_igdb_id}];").json() for rom in matched_roms: - try: - res_details: dict = requests.post("https://api.igdb.com/v4/covers/", headers=self.headers, - data=f"fields url; where game={rom['id']};").json()[0] - rom['url_cover'] = f"https:{res_details['url']}".replace('t_thumb', f't_cover_big') - except IndexError: - rom['url_cover'] = DEFAULT_URL_COVER_L + rom['url_cover'] = self._search_cover(rom['id']).replace('t_thumb', f't_cover_big') rom['r_igdb_id'] = rom.pop('id') rom['r_slug'] = rom.pop('slug') rom['r_name'] = rom.pop('name') else: - matched_roms: list = [] + log.warning(f"{p_slug} is not supported!") return matched_roms @@ -156,6 +126,7 @@ def get_matched_roms_by_id(self, igdb_id: str) -> list: class TwitchAuth(): def __init__(self) -> None: + self.base_url: str = 'https://id.twitch.tv/oauth2/token' self.token: str = "" self.token_checkout: int = int(time()) self.SECURE_SECONDS_OFFSET: int = 10 # seconds offset to avoid invalid token @@ -169,7 +140,7 @@ def _is_token_valid(self) -> str: def _update_twitch_token(self) -> str: - res = requests.post(url=f"https://id.twitch.tv/oauth2/token", + res = requests.post(url=self.base_url, params={ 'client_id': self.client_id, 'client_secret': self.client_secret, diff --git a/backend/src/utils/fastapi.py b/backend/src/utils/fastapi.py index b9d361e86..b975b8522 100644 --- a/backend/src/utils/fastapi.py +++ b/backend/src/utils/fastapi.py @@ -24,7 +24,7 @@ def scan_platform(fs_slug: str) -> Platform: platform_attrs['slug'] = fs_slug except (KeyError, TypeError, AttributeError): platform_attrs['slug'] = fs_slug - platform_attrs.update(igdbh.get_platform_details(platform_attrs['slug'])) + platform_attrs.update(igdbh.get_platform(platform_attrs['slug'])) platform_attrs['n_roms'] = len(fs.get_roms(platform_attrs['fs_slug'])) platform = Platform(**platform_attrs) return platform @@ -33,8 +33,9 @@ def scan_platform(fs_slug: str) -> Platform: def scan_rom(platform: Platform, rom_attrs: dict, r_igbd_id_search: str = '', overwrite: bool = False) -> Rom: p_slug: str = platform.fs_slug if platform.fs_slug else platform.slug roms_path: str = fs.get_roms_structure(p_slug) - rom_attrs.update(igdbh.get_rom_details(rom_attrs['file_name'], platform.igdb_id, r_igbd_id_search)) - rom_attrs.update(fs.get_cover_details(overwrite, platform.slug, rom_attrs['file_name'], rom_attrs['url_cover'])) + if r_igbd_id_search: rom_attrs.update(igdbh.get_rom_by_id(r_igbd_id_search)) + else: rom_attrs.update(igdbh.get_rom(rom_attrs['file_name'], platform.igdb_id)) + rom_attrs.update(fs.get_cover(overwrite, platform.slug, rom_attrs['file_name'], rom_attrs['url_cover'])) file_size, file_size_units = fs.get_rom_size(rom_attrs['multi'], rom_attrs['file_name'], rom_attrs['files'], roms_path) reg, rev, other_tags = parse_tags(rom_attrs['file_name']) rom_attrs.update({'file_path': roms_path, 'file_name': rom_attrs['file_name'], 'file_name_no_tags': get_file_name_with_no_tags(rom_attrs['file_name']), diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 61e3949ce..4119c9a87 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -57,7 +57,7 @@ def _get_cover_path(p_slug: str, file_name: str, size: str) -> str: return f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}_{size}.png" -def get_cover_details(overwrite: bool, p_slug: str, file_name: str, url_cover: str) -> tuple: +def get_cover(overwrite: bool, p_slug: str, file_name: str, url_cover: str) -> tuple: path_cover_s: str = DEFAULT_PATH_COVER_S path_cover_l: str = DEFAULT_PATH_COVER_L has_cover: int = 0 From 529d3c36777eb0422e30f34955814856198a5853 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 11:23:20 +0200 Subject: [PATCH 03/45] removed unused import --- backend/src/handler/igdb_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index 1b2dcda7d..8e02ebd6f 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -4,7 +4,7 @@ from time import time import requests -from config import CLIENT_ID, CLIENT_SECRET, DEFAULT_URL_COVER_L +from config import CLIENT_ID, CLIENT_SECRET from utils import get_file_name_with_no_tags as get_search_term from logger.logger import log From f13441e52fbcd819df2ce9b7d2a8d73074d625ed Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 11:41:51 +0200 Subject: [PATCH 04/45] search rom by id fixed --- backend/src/handler/igdb_handler.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index 8e02ebd6f..aee2d6d0b 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -32,13 +32,13 @@ def wrapper(*args): return wrapper - def _search_rom_by_category(self, search_term: str, p_igdb_id: str, category: int = None) -> dict: - category_query: str = f"& category={category}" if category else "" + def _search_rom(self, search_term: str, p_igdb_id: str, category: int = None) -> dict: + category_filter: str = f"& category={category}" if category else "" try: return requests.post(self.games_url, headers=self.headers, data=f"search \"{search_term}\"; \ fields id, slug, name, summary; \ - where platforms=[{p_igdb_id}] {category_query};").json()[0] + where platforms=[{p_igdb_id}] {category_filter};").json()[0] except IndexError: return {} @@ -65,7 +65,8 @@ def get_platform(self, slug: str) -> tuple: @check_twitch_token def get_rom_by_id(self, r_igdb_id: str) -> dict: res: dict = requests.post(self.games_url, headers=self.headers, - data=f"fields slug, name, summary; where id=[{r_igdb_id}];").json()[0] + data=f"fields slug, name, summary; where id={r_igdb_id};").json()[0] + log.debug(res) r_slug = res['slug'] if 'slug' in res.keys() else "" r_name = res['name'] if 'name' in res.keys() else "" summary = res['summary'] if 'summary' in res.keys() else "" @@ -75,9 +76,9 @@ def get_rom_by_id(self, r_igdb_id: str) -> dict: @check_twitch_token def get_rom(self, file_name: str, p_igdb_id: int) -> dict: search_term: str = uc(get_search_term(file_name)) - res = (self._search_rom_by_category(search_term, p_igdb_id, 0) or - self._search_rom_by_category(search_term, p_igdb_id, 10) or - self._search_rom_by_category(search_term, p_igdb_id)) + res = (self._search_rom(search_term, p_igdb_id, 0) or + self._search_rom(search_term, p_igdb_id, 10) or + self._search_rom(search_term, p_igdb_id)) r_igdb_id = res['id'] if 'id' in res.keys() else "" r_slug = res['slug'] if 'slug' in res.keys() else "" @@ -91,17 +92,9 @@ def get_rom(self, file_name: str, p_igdb_id: int) -> dict: @check_twitch_token def get_matched_rom_by_id(self, igdb_id: str) -> list: - matched_rom: list[dict] = [] - res: list = requests.post(self.games_url, headers=self.headers, - data=f"fields name, id, slug, summary; where id={igdb_id};") - if res.status_code == 200: - matched_rom = res.json() - for rom in matched_rom: - rom['url_cover'] = self._search_cover(rom['id']).replace('t_thumb', f't_cover_big') - rom['r_igdb_id'] = rom.pop('id') - rom['r_slug'] = rom.pop('slug') - rom['r_name'] = rom.pop('name') - return matched_rom + matched_rom: dict = self.get_rom_by_id(igdb_id) + matched_rom['url_cover'] = matched_rom['url_cover'].replace('t_thumb', f't_cover_big') + return [matched_rom] @check_twitch_token From cd110cdb7507b750532c09ef6c64a2fd4fcbcb5a Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 11:42:48 +0200 Subject: [PATCH 05/45] debug log removed --- backend/src/handler/igdb_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index aee2d6d0b..82be9f4cb 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -66,7 +66,6 @@ def get_platform(self, slug: str) -> tuple: def get_rom_by_id(self, r_igdb_id: str) -> dict: res: dict = requests.post(self.games_url, headers=self.headers, data=f"fields slug, name, summary; where id={r_igdb_id};").json()[0] - log.debug(res) r_slug = res['slug'] if 'slug' in res.keys() else "" r_name = res['name'] if 'name' in res.keys() else "" summary = res['summary'] if 'summary' in res.keys() else "" From 2715802c4a14fd3fe50e55c41895180bfea06d75 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 12:04:15 +0200 Subject: [PATCH 06/45] excluded platforms code improved --- backend/src/config/__init__.py | 3 +++ backend/src/endpoints/scan.py | 2 +- backend/src/handler/igdb_handler.py | 7 ++++--- backend/src/utils/fs.py | 32 +++++++++++------------------ 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/backend/src/config/__init__.py b/backend/src/config/__init__.py index 1a84b4164..de6e59fd6 100644 --- a/backend/src/config/__init__.py +++ b/backend/src/config/__init__.py @@ -32,6 +32,9 @@ config = None user_config: dict = {} if not config else config +EXCLUDED_PLATFORMS: list[str] = user_config['exclude']['platforms'] if user_config['exclude']['platforms'] else [] + + # DB DRIVERS SUPPORTED_DB_DRIVERS: list = ['sqlite', 'mariadb'] ROMM_DB_DRIVER: str = os.getenv('ROMM_DB_DRIVER', 'sqlite') diff --git a/backend/src/endpoints/scan.py b/backend/src/endpoints/scan.py index 9878224cd..ce6baf8e3 100644 --- a/backend/src/endpoints/scan.py +++ b/backend/src/endpoints/scan.py @@ -44,10 +44,10 @@ def scan(platforms: str, full_scan: bool=False) -> dict: log.warning(error) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) for rom in fs_roms: - if rom['multi']: [log.info(f"\t - {COLORS['orange_i']}{file}{COLORS['reset']}") for file in rom['files']] rom_id: int = dbh.rom_exists(rom['file_name'], scanned_platform.slug) if rom_id and not full_scan: continue log.info(f"Scanning {COLORS['orange']}{rom['file_name']}{COLORS['reset']}") + if rom['multi']: [log.info(f"\t - {COLORS['orange_i']}{file}{COLORS['reset']}") for file in rom['files']] scanned_rom: Rom = fastapi.scan_rom(scanned_platform, rom) if rom_id: scanned_rom.id = rom_id dbh.add_rom(scanned_rom) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index 82be9f4cb..472e13fd5 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -51,14 +51,15 @@ def _search_cover(self, rom_id: str) -> str: @check_twitch_token def get_platform(self, slug: str) -> tuple: + igdb_id: str = "" + name: str = slug try: res: dict = requests.post(self.platform_url, headers=self.headers, data=f"fields id, name; where slug=\"{slug}\";").json()[0] - igdb_id = res['id'] if 'id' in res.keys() else "" - name = res['name'] if 'name' in res.keys() else "" + igdb_id = res['id'] + name = res['name'] except IndexError: log.warning(f"{slug} not found in IGDB") - if not name: name = slug return {'igdb_id': igdb_id, 'name': name, 'slug': slug, 'logo_path': ''} diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 4119c9a87..72eb55d58 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -4,7 +4,9 @@ import requests -from config import user_config, LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, RESOURCES_BASE_PATH, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S +from config import user_config, EXCLUDED_PLATFORMS, \ + LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, \ + RESOURCES_BASE_PATH, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S from logger.logger import log from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException, RomNotFoundError, RomAlreadyExistsException @@ -58,19 +60,12 @@ def _get_cover_path(p_slug: str, file_name: str, size: str) -> str: def get_cover(overwrite: bool, p_slug: str, file_name: str, url_cover: str) -> tuple: - path_cover_s: str = DEFAULT_PATH_COVER_S - path_cover_l: str = DEFAULT_PATH_COVER_L - has_cover: int = 0 - if (overwrite or not _cover_exists(p_slug, file_name, 's')) and url_cover: - _store_cover(p_slug, file_name, url_cover, 's') - if _cover_exists(p_slug, file_name, 's'): - path_cover_s = _get_cover_path(p_slug, file_name, 's') - - if (overwrite or not _cover_exists(p_slug, file_name, 'l')) and url_cover: - _store_cover(p_slug, file_name, url_cover, 'l') - if _cover_exists(p_slug, file_name, 'l'): - path_cover_l = _get_cover_path(p_slug, file_name, 'l') - has_cover = 1 + # Cover small + if (overwrite or not _cover_exists(p_slug, file_name, 's')) and url_cover: _store_cover(p_slug, file_name, url_cover, 's') + path_cover_s = _get_cover_path(p_slug, file_name, 's') if _cover_exists(p_slug, file_name, 's') else DEFAULT_PATH_COVER_S + # Cover big + if (overwrite or not _cover_exists(p_slug, file_name, 'l')) and url_cover: _store_cover(p_slug, file_name, url_cover, 'l') + (path_cover_l, has_cover) = (_get_cover_path(p_slug, file_name, 'l'), 1) if _cover_exists(p_slug, file_name, 'l') else (DEFAULT_PATH_COVER_L, 0) return {'path_cover_s': path_cover_s, 'path_cover_l': path_cover_l, 'has_cover': has_cover} @@ -83,8 +78,8 @@ def store_default_resources() -> None: # ========= Platforms utils ========= -def _exclude_platforms(platforms) -> list['str']: - [platforms.remove(excluded) for excluded in user_config['exclude']['platforms'] if excluded in platforms] +def _exclude_platforms(platforms) -> None: + [platforms.remove(excluded) for excluded in EXCLUDED_PLATFORMS if excluded in platforms] def get_platforms() -> list[str]: @@ -95,12 +90,9 @@ def get_platforms() -> list[str]: """ try: platforms: list[str] = list(os.walk(HIGH_PRIO_STRUCTURE_PATH))[0][1] if os.path.exists(HIGH_PRIO_STRUCTURE_PATH) else list(os.walk(LIBRARY_BASE_PATH))[0][1] + _exclude_platforms(platforms) except IndexError: raise PlatformsNotFoundException - try: - _exclude_platforms(platforms) - except (KeyError, TypeError): - pass return platforms From ee8f77df7629a40a5891e84fa069cb2f0991e010 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 12:58:21 +0200 Subject: [PATCH 07/45] config management refactored --- backend/src/config/__init__.py | 12 -------- backend/src/config/config_loader.py | 45 +++++++++++++++++++++++++++-- backend/src/endpoints/scan.py | 1 - backend/src/handler/__init__.py | 3 +- backend/src/handler/db_handler.py | 3 +- backend/src/utils/fastapi.py | 8 ++--- backend/src/utils/fs.py | 44 +++++++++------------------- 7 files changed, 63 insertions(+), 53 deletions(-) diff --git a/backend/src/config/__init__.py b/backend/src/config/__init__.py index de6e59fd6..3fbf963af 100644 --- a/backend/src/config/__init__.py +++ b/backend/src/config/__init__.py @@ -1,6 +1,4 @@ import os -import yaml -from yaml.loader import SafeLoader # Uvicorn DEV_PORT: int = 5000 @@ -25,16 +23,6 @@ # STEAMGRIDDB STEAMGRIDDB_API_KEY: str = os.getenv('STEAMGRIDDB_API_KEY') -# USER CONFIG -try: - with open(ROMM_USER_CONFIG_PATH) as config: config = yaml.load(config, Loader=SafeLoader) -except FileNotFoundError: - config = None -user_config: dict = {} if not config else config - -EXCLUDED_PLATFORMS: list[str] = user_config['exclude']['platforms'] if user_config['exclude']['platforms'] else [] - - # DB DRIVERS SUPPORTED_DB_DRIVERS: list = ['sqlite', 'mariadb'] ROMM_DB_DRIVER: str = os.getenv('ROMM_DB_DRIVER', 'sqlite') diff --git a/backend/src/config/config_loader.py b/backend/src/config/config_loader.py index 5a991bc14..d055d0304 100644 --- a/backend/src/config/config_loader.py +++ b/backend/src/config/config_loader.py @@ -1,18 +1,25 @@ import os import sys +import yaml +from yaml.loader import SafeLoader from urllib.parse import quote_plus -from config import ROMM_DB_DRIVER, SUPPORTED_DB_DRIVERS, SQLITE_DB_BASE_PATH, user_config +from config import ROMM_DB_DRIVER, SUPPORTED_DB_DRIVERS, SQLITE_DB_BASE_PATH, ROMM_USER_CONFIG_PATH from logger.logger import log class ConfigLoader: def __init__(self): - pass + try: + with open(ROMM_USER_CONFIG_PATH) as config_file: self.config: dict = yaml.load(config_file, Loader=SafeLoader) + except FileNotFoundError: + self.config: dict = {} + self._parse_config() - def get_db_engine(self): + @staticmethod + def get_db_engine(): if ROMM_DB_DRIVER in SUPPORTED_DB_DRIVERS: if ROMM_DB_DRIVER == 'mariadb': @@ -35,3 +42,35 @@ def get_db_engine(self): else: log.critical(f"{ROMM_DB_DRIVER} database not supported") sys.exit(3) + + + def _parse_config(self) -> dict: + try: + self.config['EXCLUDED_PLATFORMS'] = self.config['exclude']['platforms'] if self.config['exclude']['platforms'] else [] + except KeyError: + self.config['EXCLUDED_PLATFORMS'] = [] + try: + self.config['EXCLUDED_SINGLE_EXT'] = self.config['exclude']['roms']['single_file']['extensions'] if self.config['exclude']['roms']['single_file']['extensions'] else [] + except KeyError: + self.config['EXCLUDED_SINGLE_EXT'] = [] + try: + self.config['EXCLUDED_SINGLE_FILES'] = self.config['exclude']['roms']['single_file']['names'] if self.config['exclude']['roms']['single_file']['names'] else [] + except KeyError: + self.config['EXCLUDED_SINGLE_FILES'] = [] + try: + self.config['EXCLUDED_MULTI_FILES'] = self.config['exclude']['roms']['multi_file']['names'] if self.config['exclude']['roms']['multi_file']['names'] else [] + except KeyError: + self.config['EXCLUDED_MULTI_FILES'] = [] + try: + self.config['EXCLUDED_MULTI_PARTS_EXT'] = self.config['exclude']['roms']['multi_file']['parts']['extensions'] if self.config['exclude']['roms']['multi_file']['parts']['extensions'] else [] + except KeyError: + self.config['EXCLUDED_MULTI_PARTS_EXT'] = [] + try: + self.config['EXCLUDED_MULTI_PARTS_FILES'] = self.config['exclude']['roms']['multi_file']['parts']['names'] if self.config['exclude']['roms']['multi_file']['parts']['names'] else [] + except KeyError: + self.config['EXCLUDED_MULTI_PARTS_FILES'] = [] + try: + self.config['PLATFORMS_BINDING'] = self.config['system']['platforms'] if self.config['system']['platforms'] else {} + except KeyError: + self.config['PLATFORMS_BINDING'] = {} + \ No newline at end of file diff --git a/backend/src/endpoints/scan.py b/backend/src/endpoints/scan.py index ce6baf8e3..992c3d408 100644 --- a/backend/src/endpoints/scan.py +++ b/backend/src/endpoints/scan.py @@ -3,7 +3,6 @@ import json from logger.logger import log, COLORS -from config import user_config from utils import fs, fastapi from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException from handler import dbh diff --git a/backend/src/handler/__init__.py b/backend/src/handler/__init__.py index 138b37a28..86c969382 100644 --- a/backend/src/handler/__init__.py +++ b/backend/src/handler/__init__.py @@ -1,7 +1,8 @@ from handler.db_handler import DBHandler from handler.igdb_handler import IGDBHandler from handler.sgdb_handler import SGDBHandler +from config.config_loader import ConfigLoader igdbh: IGDBHandler = IGDBHandler() sgdbh: SGDBHandler = SGDBHandler() -dbh: DBHandler = DBHandler() +dbh: DBHandler = DBHandler(ConfigLoader()) diff --git a/backend/src/handler/db_handler.py b/backend/src/handler/db_handler.py index 14d9011df..e53064109 100644 --- a/backend/src/handler/db_handler.py +++ b/backend/src/handler/db_handler.py @@ -13,8 +13,7 @@ class DBHandler: - def __init__(self) -> None: - cl = ConfigLoader() + def __init__(self, cl: ConfigLoader) -> None: self.engine = create_engine(cl.get_db_engine(), pool_pre_ping=True) self.session = sessionmaker(bind=self.engine, expire_on_commit=False) diff --git a/backend/src/utils/fastapi.py b/backend/src/utils/fastapi.py index b975b8522..b596db82b 100644 --- a/backend/src/utils/fastapi.py +++ b/backend/src/utils/fastapi.py @@ -1,7 +1,7 @@ from handler import igdbh from utils import fs, parse_tags, get_file_extension, get_file_name_with_no_tags -from config import user_config -from logger.logger import log +from config.config_loader import ConfigLoader +cl = ConfigLoader() from models.platform import Platform from models.rom import Rom @@ -18,8 +18,8 @@ def scan_platform(fs_slug: str) -> Platform: platform_attrs: dict = {} platform_attrs['fs_slug'] = fs_slug try: - if fs_slug in user_config['system']['platforms'].keys(): - platform_attrs['slug'] = user_config['system']['platforms'][fs_slug] + if fs_slug in cl.config['PLATFORMS_BINDING'].keys(): + platform_attrs['slug'] = cl.config['PLATFORMS_BINDING'][fs_slug] else: platform_attrs['slug'] = fs_slug except (KeyError, TypeError, AttributeError): diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 72eb55d58..0a5db2206 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -4,9 +4,10 @@ import requests -from config import user_config, EXCLUDED_PLATFORMS, \ - LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, \ +from config import LIBRARY_BASE_PATH, HIGH_PRIO_STRUCTURE_PATH, \ RESOURCES_BASE_PATH, DEFAULT_URL_COVER_L, DEFAULT_PATH_COVER_L, DEFAULT_URL_COVER_S, DEFAULT_PATH_COVER_S +from config.config_loader import ConfigLoader +cl = ConfigLoader() from logger.logger import log from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException, RomNotFoundError, RomAlreadyExistsException @@ -79,7 +80,8 @@ def store_default_resources() -> None: # ========= Platforms utils ========= def _exclude_platforms(platforms) -> None: - [platforms.remove(excluded) for excluded in EXCLUDED_PLATFORMS if excluded in platforms] + print(cl.config['EXCLUDED_PLATFORMS']) + [platforms.remove(excluded) for excluded in cl.config['EXCLUDED_PLATFORMS'] if excluded in platforms] def get_platforms() -> list[str]: @@ -110,30 +112,15 @@ def get_rom_files(rom: str, roms_path: str) -> list[str]: def _exclude_files(files, type) -> list[str]: if type == 'single': - try: - excluded_extensions = user_config['exclude']['roms'][f'{type}_file']['extensions'] - except (TypeError, KeyError): - excluded_extensions: list = [] - try: - excluded_names = user_config['exclude']['roms'][f'{type}_file']['names'] - except (TypeError, KeyError): - excluded_names: list = [] + excluded_extensions = cl.config['EXCLUDED_SINGLE_EXT'] + excluded_names = cl.config['EXCLUDED_SINGLE_FILES'] elif type == 'multi': - try: - excluded_extensions = user_config['exclude']['roms'][f'{type}_file']['parts']['extensions'] - except (TypeError, KeyError): - excluded_extensions: list = [] - try: - excluded_names = user_config['exclude']['roms'][f'{type}_file']['parts']['names'] - except (TypeError, KeyError): - excluded_names: list = [] + excluded_extensions = cl.config['EXCLUDED_MULTI_PARTS_EXT'] + excluded_names = cl.config['EXCLUDED_MULTI_PARTS_FILES'] filtered_files: list = [] for file in files: - try: - if file.split('.')[-1] in excluded_extensions or file in excluded_names: - filtered_files.append(file) - except TypeError: - pass + if file.split('.')[-1] in excluded_extensions or file in excluded_names: + filtered_files.append(file) files = [f for f in files if f not in filtered_files] return files @@ -141,16 +128,13 @@ def _exclude_files(files, type) -> list[str]: def _exclude_multi_roms(roms) -> list[str]: try: excluded_names: list = [] - excluded_names = user_config['exclude']['roms']['multi_file']['names'] + excluded_names = cl.config['EXCLUDED_MULTI_FILES'] except (TypeError, KeyError): pass filtered_files: list = [] for rom in roms: - try: - if rom in excluded_names: - filtered_files.append(rom) - except TypeError: - pass + if rom in excluded_names: + filtered_files.append(rom) roms = [f for f in roms if f not in filtered_files] return roms From be0926b96ce304399c2fdfc74e3a82b942744ca4 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 13:02:25 +0200 Subject: [PATCH 08/45] exclude multi part file fixed --- backend/src/utils/fs.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 0a5db2206..890c369e0 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -80,7 +80,6 @@ def store_default_resources() -> None: # ========= Platforms utils ========= def _exclude_platforms(platforms) -> None: - print(cl.config['EXCLUDED_PLATFORMS']) [platforms.remove(excluded) for excluded in cl.config['EXCLUDED_PLATFORMS'] if excluded in platforms] @@ -106,17 +105,13 @@ def get_roms_structure(p_slug: str) -> tuple: def get_rom_files(rom: str, roms_path: str) -> list[str]: rom_files: list = [] for path, _, files in os.walk(f"{roms_path}/{rom}"): - [rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", '')) for f in _exclude_files(files, 'multi')] + [rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", '')) for f in _exclude_files(files, 'multi_parts')] return rom_files def _exclude_files(files, type) -> list[str]: - if type == 'single': - excluded_extensions = cl.config['EXCLUDED_SINGLE_EXT'] - excluded_names = cl.config['EXCLUDED_SINGLE_FILES'] - elif type == 'multi': - excluded_extensions = cl.config['EXCLUDED_MULTI_PARTS_EXT'] - excluded_names = cl.config['EXCLUDED_MULTI_PARTS_FILES'] + excluded_extensions = cl.config[f'EXCLUDED_{type.upper()}_EXT'] + excluded_names = cl.config[f'EXCLUDED_{type.upper()}_FILES'] filtered_files: list = [] for file in files: if file.split('.')[-1] in excluded_extensions or file in excluded_names: @@ -126,11 +121,7 @@ def _exclude_files(files, type) -> list[str]: def _exclude_multi_roms(roms) -> list[str]: - try: - excluded_names: list = [] - excluded_names = cl.config['EXCLUDED_MULTI_FILES'] - except (TypeError, KeyError): - pass + excluded_names = cl.config['EXCLUDED_MULTI_FILES'] filtered_files: list = [] for rom in roms: if rom in excluded_names: From 04f64fd43cdd0895a5eeecf3a8d5751f99486895 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 16:41:59 +0200 Subject: [PATCH 09/45] metadata added: screenshots --- backend/src/alembic/versions/1.8.1_.py | 34 +++++++++ backend/src/endpoints/rom.py | 1 + backend/src/handler/igdb_handler.py | 17 ++++- backend/src/models/rom.py | 3 + backend/src/utils/fastapi.py | 1 + backend/src/utils/fs.py | 100 ++++++++++++++++--------- frontend/src/views/Details.vue | 17 +++-- 7 files changed, 129 insertions(+), 44 deletions(-) create mode 100644 backend/src/alembic/versions/1.8.1_.py diff --git a/backend/src/alembic/versions/1.8.1_.py b/backend/src/alembic/versions/1.8.1_.py new file mode 100644 index 000000000..84eef27d4 --- /dev/null +++ b/backend/src/alembic/versions/1.8.1_.py @@ -0,0 +1,34 @@ +"""update to 1.8.1 + +Revision ID: 1.8.1 +Revises: 1.8 +Create Date: 2023-04-17 12:03:19.163501 + +""" +import os +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + + +# revision identifiers, used by Alembic. +revision = '1.8.1' +down_revision = '1.8' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("roms") as batch_op: + batch_op.add_column(sa.Column('url_screenshots', sa.JSON(), nullable=True)) + batch_op.add_column(sa.Column('path_screenshots', sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("roms") as batch_op: + batch_op.drop_column('url_screenshots') + batch_op.drop_column('path_screenshots') + # ### end Alembic commands ### diff --git a/backend/src/endpoints/rom.py b/backend/src/endpoints/rom.py index 91f02d6ec..4432fcaa9 100644 --- a/backend/src/endpoints/rom.py +++ b/backend/src/endpoints/rom.py @@ -40,6 +40,7 @@ async def updateRom(req: Request, p_slug: str, id: int) -> dict: raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error) updated_rom['file_name_no_tags'] = get_file_name_with_no_tags(updated_rom['file_name']) updated_rom.update(fs.get_cover(True, p_slug, updated_rom['file_name'], updated_rom['url_cover'])) + updated_rom.update(fs.get_screenshots(p_slug, updated_rom['file_name'], updated_rom['url_screenshots'])) dbh.update_rom(id, updated_rom) return {'data': dbh.get_rom(id), 'msg': f"{updated_rom['file_name']} updated successfully!"} diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index 472e13fd5..e80a66d30 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -16,6 +16,7 @@ def __init__(self) -> None: self.platform_url: str = f'{base_url}/platforms/' self.games_url: str = f'{base_url}/games/' self.covers_url: str = f'{base_url}/covers/' + self.screenshots_url: str = f'{base_url}/screenshots/' self.twitch_auth: TwitchAuth = TwitchAuth() self.headers = { 'Client-ID': self.twitch_auth.client_id, @@ -37,7 +38,7 @@ def _search_rom(self, search_term: str, p_igdb_id: str, category: int = None) -> try: return requests.post(self.games_url, headers=self.headers, data=f"search \"{search_term}\"; \ - fields id, slug, name, summary; \ + fields id, slug, name, summary, screenshots; \ where platforms=[{p_igdb_id}] {category_filter};").json()[0] except IndexError: return {} @@ -47,6 +48,12 @@ def _search_cover(self, rom_id: str) -> str: res: dict = requests.post(self.covers_url, headers=self.headers, data=f"fields url; where game={rom_id};").json()[0] return f"https:{res['url']}" if 'url' in res.keys() else "" + + + def _search_screenshots(self, rom_id: str) -> list: + res: dict = requests.post(self.screenshots_url, headers=self.headers, + data=f"fields url; where game={rom_id}; limit 5;").json() + return [f"https:{r['url']}".replace('t_thumb', 't_original') for r in res if 'url' in r.keys()] @check_twitch_token @@ -70,7 +77,7 @@ def get_rom_by_id(self, r_igdb_id: str) -> dict: r_slug = res['slug'] if 'slug' in res.keys() else "" r_name = res['name'] if 'name' in res.keys() else "" summary = res['summary'] if 'summary' in res.keys() else "" - return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id)} + return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id), 'url_screenshots': self._search_screenshots(r_igdb_id)} @check_twitch_token @@ -87,13 +94,14 @@ def get_rom(self, file_name: str, p_igdb_id: int) -> dict: if not r_name: r_name = search_term if not r_igdb_id: log.warning(f"{r_name} not found in IGDB") - return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id)} + return {'r_igdb_id': r_igdb_id, 'r_slug': r_slug, 'r_name': r_name, 'summary': summary, 'url_cover': self._search_cover(r_igdb_id), 'url_screenshots': self._search_screenshots(r_igdb_id)} @check_twitch_token def get_matched_rom_by_id(self, igdb_id: str) -> list: matched_rom: dict = self.get_rom_by_id(igdb_id) matched_rom['url_cover'] = matched_rom['url_cover'].replace('t_thumb', f't_cover_big') + matched_rom['url_screenshots'] = self._search_screenshots(matched_rom['id']) return [matched_rom] @@ -103,10 +111,11 @@ def get_matched_roms(self, file_name: str, p_igdb_id: int, p_slug: str) -> list: if p_igdb_id: matched_roms: list = requests.post(self.games_url, headers=self.headers, data=f"search \"{uc(get_search_term(file_name))}\"; \ - fields name, id, slug, summary; \ + fields id, slug, name, summary; \ where platforms=[{p_igdb_id}];").json() for rom in matched_roms: rom['url_cover'] = self._search_cover(rom['id']).replace('t_thumb', f't_cover_big') + rom['url_screenshots'] = self._search_screenshots(rom['id']) rom['r_igdb_id'] = rom.pop('id') rom['r_slug'] = rom.pop('slug') rom['r_name'] = rom.pop('name') diff --git a/backend/src/models/rom.py b/backend/src/models/rom.py index 20cbcf685..dc70aa4f7 100644 --- a/backend/src/models/rom.py +++ b/backend/src/models/rom.py @@ -40,6 +40,9 @@ class Rom(BaseModel): multi = Column(Boolean, default=False) files = Column(JSON, default=[]) + url_screenshots = Column(JSON, default=[]) + path_screenshots = Column(JSON, default=[]) + @property def full_path(self) -> str: return f"{self.file_path}/{self.file_name}" diff --git a/backend/src/utils/fastapi.py b/backend/src/utils/fastapi.py index b596db82b..eac4fae6e 100644 --- a/backend/src/utils/fastapi.py +++ b/backend/src/utils/fastapi.py @@ -36,6 +36,7 @@ def scan_rom(platform: Platform, rom_attrs: dict, r_igbd_id_search: str = '', ov if r_igbd_id_search: rom_attrs.update(igdbh.get_rom_by_id(r_igbd_id_search)) else: rom_attrs.update(igdbh.get_rom(rom_attrs['file_name'], platform.igdb_id)) rom_attrs.update(fs.get_cover(overwrite, platform.slug, rom_attrs['file_name'], rom_attrs['url_cover'])) + rom_attrs.update(fs.get_screenshots(platform.slug, rom_attrs['file_name'], rom_attrs['url_screenshots'])) file_size, file_size_units = fs.get_rom_size(rom_attrs['multi'], rom_attrs['file_name'], rom_attrs['files'], roms_path) reg, rev, other_tags = parse_tags(rom_attrs['file_name']) rom_attrs.update({'file_path': roms_path, 'file_name': rom_attrs['file_name'], 'file_name_no_tags': get_file_name_with_no_tags(rom_attrs['file_name']), diff --git a/backend/src/utils/fs.py b/backend/src/utils/fs.py index 890c369e0..a0a3fda21 100644 --- a/backend/src/utils/fs.py +++ b/backend/src/utils/fs.py @@ -23,7 +23,7 @@ def _cover_exists(p_slug: str, file_name: str, size: str) -> bool: Returns True if cover exists in filesystem else False """ - logo_path: str = f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}_{size}.png" + logo_path: str = f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}/cover/{size}.png" return True if os.path.exists(logo_path) else False @@ -34,19 +34,15 @@ def _store_cover(p_slug: str, file_name: str, url_cover: str, size: str) -> None p_slug: short name of the platform file_name: name of rom file url_cover: url to get the cover - size: size of the cover -> big as 'l' | small as 's' + size: size of the cover -> big | small """ - cover_file: str = f"{file_name}_{size}.png" - cover_path: str = f"{RESOURCES_BASE_PATH}/{p_slug}/" - sizes: dict = {'l': 'big', 's': 'small'} - res = requests.get(url_cover.replace('t_thumb', f't_cover_{sizes[size]}'), stream=True) + cover_file: str = f"{size}.png" + cover_path: str = f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}/cover" + res = requests.get(url_cover.replace('t_thumb', f't_cover_{size}'), stream=True) if res.status_code == 200: Path(cover_path).mkdir(parents=True, exist_ok=True) with open(f"{cover_path}/{cover_file}", 'wb') as f: shutil.copyfileobj(res.raw, f) - log.info(f"{file_name} {sizes[size]} cover downloaded successfully!") - else: - log.error(f"{file_name} {sizes[size]} cover couldn't be downloaded") def _get_cover_path(p_slug: str, file_name: str, size: str) -> str: @@ -55,27 +51,63 @@ def _get_cover_path(p_slug: str, file_name: str, size: str) -> str: Args: p_slug: short name of the platform file_name: name of rom file - size: size of the cover -> big as 'l' | small as 's' + size: size of the cover -> big | small """ - return f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}_{size}.png" + return f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}/cover/{size}.png" def get_cover(overwrite: bool, p_slug: str, file_name: str, url_cover: str) -> tuple: # Cover small - if (overwrite or not _cover_exists(p_slug, file_name, 's')) and url_cover: _store_cover(p_slug, file_name, url_cover, 's') - path_cover_s = _get_cover_path(p_slug, file_name, 's') if _cover_exists(p_slug, file_name, 's') else DEFAULT_PATH_COVER_S + if (overwrite or not _cover_exists(p_slug, file_name, 'small')) and url_cover: _store_cover(p_slug, file_name, url_cover, 'small') + path_cover_s = _get_cover_path(p_slug, file_name, 'small') if _cover_exists(p_slug, file_name, 'small') else DEFAULT_PATH_COVER_S # Cover big - if (overwrite or not _cover_exists(p_slug, file_name, 'l')) and url_cover: _store_cover(p_slug, file_name, url_cover, 'l') - (path_cover_l, has_cover) = (_get_cover_path(p_slug, file_name, 'l'), 1) if _cover_exists(p_slug, file_name, 'l') else (DEFAULT_PATH_COVER_L, 0) + if (overwrite or not _cover_exists(p_slug, file_name, 'big')) and url_cover: _store_cover(p_slug, file_name, url_cover, 'big') + (path_cover_l, has_cover) = (_get_cover_path(p_slug, file_name, 'big'), 1) if _cover_exists(p_slug, file_name, 'big') else (DEFAULT_PATH_COVER_L, 0) return {'path_cover_s': path_cover_s, 'path_cover_l': path_cover_l, 'has_cover': has_cover} +def _store_screenshot(p_slug: str, file_name: str, url: str, idx: int) -> None: + """Store roms resources in filesystem + + Args: + p_slug: short name of the platform + file_name: name of rom file + url: url to get the screenshot + """ + screenshot_file: str = f"{idx}.jpg" + screenshot_path: str = f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}/screenshots" + res = requests.get(url, stream=True) + if res.status_code == 200: + Path(screenshot_path).mkdir(parents=True, exist_ok=True) + with open(f"{screenshot_path}/{screenshot_file}", 'wb') as f: + shutil.copyfileobj(res.raw, f) + + +def _get_screenshot_path(p_slug: str, file_name: str, idx: str) -> str: + """Returns rom cover filesystem path adapted to frontend folder structure + + Args: + p_slug: short name of the platform + file_name: name of rom file + idx: index number of screenshot + """ + return f"{RESOURCES_BASE_PATH}/{p_slug}/{file_name}/screenshots/{idx}.jpg" + + +def get_screenshots(p_slug: str, file_name: str, url_screenshots: list) -> tuple: + path_screenshots: list[str] = [] + for idx, url in enumerate(url_screenshots): + _store_screenshot(p_slug, file_name, url, idx) + path_screenshots.append(_get_screenshot_path(p_slug, file_name, idx)) + return {'path_screenshots': path_screenshots} + + def store_default_resources() -> None: """Store default cover resources in the filesystem""" - defaul_covers: dict = [{'url': DEFAULT_URL_COVER_L, 'size': 'l'}, {'url': DEFAULT_URL_COVER_S, 'size': 's'}] + defaul_covers: dict = [{'url': DEFAULT_URL_COVER_L, 'size': 'big'}, {'url': DEFAULT_URL_COVER_S, 'size': 'small'}] for cover in defaul_covers: - if not _cover_exists('default', 'cover', cover['size']): - _store_cover('default', 'cover', cover['url'], cover['size']) + if not _cover_exists('default', 'default', cover['size']): + _store_cover('default', 'default', cover['url'], cover['size']) # ========= Platforms utils ========= @@ -102,13 +134,6 @@ def get_roms_structure(p_slug: str) -> tuple: return f"{HIGH_PRIO_STRUCTURE_PATH}/{p_slug}" if os.path.exists(HIGH_PRIO_STRUCTURE_PATH) else f"{LIBRARY_BASE_PATH}/{p_slug}/roms" -def get_rom_files(rom: str, roms_path: str) -> list[str]: - rom_files: list = [] - for path, _, files in os.walk(f"{roms_path}/{rom}"): - [rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", '')) for f in _exclude_files(files, 'multi_parts')] - return rom_files - - def _exclude_files(files, type) -> list[str]: excluded_extensions = cl.config[f'EXCLUDED_{type.upper()}_EXT'] excluded_names = cl.config[f'EXCLUDED_{type.upper()}_FILES'] @@ -130,15 +155,11 @@ def _exclude_multi_roms(roms) -> list[str]: return roms -def get_rom_size(multi: bool, rom: str, files: list, roms_path:str) -> str: - files: list = [f"{roms_path}/{rom}"] if not multi else [f"{roms_path}/{rom}/{file}" for file in files] - total_size: int = 0 - for file in files: - total_size += os.stat(file).st_size - for unit in ['B', 'KB', 'MB', 'GB', 'TB', 'PB']: - if total_size < 1024.0 or unit == 'PB': break - total_size /= 1024.0 - return round(total_size, 2), unit +def get_rom_files(rom: str, roms_path: str) -> list[str]: + rom_files: list = [] + for path, _, files in os.walk(f"{roms_path}/{rom}"): + [rom_files.append(f"{Path(path, f)}".replace(f"{roms_path}/{rom}/", '')) for f in _exclude_files(files, 'multi_parts')] + return rom_files def get_roms(p_slug: str) -> list[dict] or int: @@ -163,6 +184,17 @@ def get_roms(p_slug: str) -> list[dict] or int: return fs_roms +def get_rom_size(multi: bool, rom: str, files: list, roms_path:str) -> str: + files: list = [f"{roms_path}/{rom}"] if not multi else [f"{roms_path}/{rom}/{file}" for file in files] + total_size: int = 0 + for file in files: + total_size += os.stat(file).st_size + for unit in ['B', 'KB', 'MB', 'GB', 'TB', 'PB']: + if total_size < 1024.0 or unit == 'PB': break + total_size /= 1024.0 + return round(total_size, 2), unit + + def _rom_exists(p_slug: str, file_name: str) -> bool: """Check if rom exists in filesystem diff --git a/frontend/src/views/Details.vue b/frontend/src/views/Details.vue index bb65d2a6f..ddeb3ef6a 100644 --- a/frontend/src/views/Details.vue +++ b/frontend/src/views/Details.vue @@ -52,6 +52,7 @@ async function updateRom(updatedData={...updatedRom.value}) { updatedRom.value.r_slug = updatedData.r_slug updatedRom.value.summary = updatedData.summary updatedRom.value.url_cover = updatedData.url_cover + updatedRom.value.url_screenshots = updatedData.url_screenshots updatedRom.value.r_name = updatedData.r_name if (renameAsIGDB.value) { updatedRom.value.file_name = updatedRom.value.file_name.replace(updatedRom.value.file_name_no_tags, updatedData.r_name) } await axios.patch('/api/platforms/'+rom.value.p_slug+'/roms/'+rom.value.id, { updatedRom: updatedRom.value }).then((response) => { @@ -167,8 +168,8 @@ onMounted(() => {
Info - Saves - Screenshots + Saves[comming soon] + Screenshots @@ -198,12 +199,16 @@ onMounted(() => {

{{ rom.summary }}

 - - - -

{{ rom.summary }}

 + + + + + + + + From 3ac6adcfb9a9bfbe6bae5902c9793f847784cf54 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 17:09:29 +0200 Subject: [PATCH 10/45] get screenshots searching game by id fixed --- backend/src/handler/igdb_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/handler/igdb_handler.py b/backend/src/handler/igdb_handler.py index e80a66d30..4d6d4f0b0 100644 --- a/backend/src/handler/igdb_handler.py +++ b/backend/src/handler/igdb_handler.py @@ -101,7 +101,7 @@ def get_rom(self, file_name: str, p_igdb_id: int) -> dict: def get_matched_rom_by_id(self, igdb_id: str) -> list: matched_rom: dict = self.get_rom_by_id(igdb_id) matched_rom['url_cover'] = matched_rom['url_cover'].replace('t_thumb', f't_cover_big') - matched_rom['url_screenshots'] = self._search_screenshots(matched_rom['id']) + matched_rom['url_screenshots'] = self._search_screenshots(igdb_id) return [matched_rom] From aa1d8d11e90f0bf40bbd5758f3db83d37689ab76 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Wed, 26 Apr 2023 17:09:39 +0200 Subject: [PATCH 11/45] details view ui tweaks --- frontend/src/views/Details.vue | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/Details.vue b/frontend/src/views/Details.vue index ddeb3ef6a..290328383 100644 --- a/frontend/src/views/Details.vue +++ b/frontend/src/views/Details.vue @@ -94,7 +94,7 @@ onMounted(() => { \ No newline at end of file + From 88ab3ccd6a688bbf2392b4357eed3d08d418f9d6 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Thu, 27 Apr 2023 15:06:16 +0200 Subject: [PATCH 22/45] details page ui tweaks --- .../src/components/GameGallery/ListItem/Header.vue | 8 ++++---- .../src/components/GameGallery/ListItem/Item.vue | 10 +++++----- frontend/src/views/Details.vue | 2 +- frontend/src/views/Gallery.vue | 14 +++++++------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/frontend/src/components/GameGallery/ListItem/Header.vue b/frontend/src/components/GameGallery/ListItem/Header.vue index ea547706a..b4578446c 100644 --- a/frontend/src/components/GameGallery/ListItem/Header.vue +++ b/frontend/src/components/GameGallery/ListItem/Header.vue @@ -7,16 +7,16 @@ Name File Platform - - - + + + - + diff --git a/frontend/src/components/GameGallery/ListItem/Item.vue b/frontend/src/components/GameGallery/ListItem/Item.vue index 7648652bd..9a8bfefdc 100644 --- a/frontend/src/components/GameGallery/ListItem/Item.vue +++ b/frontend/src/components/GameGallery/ListItem/Item.vue @@ -20,13 +20,13 @@ const emitter = inject('emitter') :to="`/platform/${$route.params.platform}/rom/${rom.id}`" :value="rom.id" :key="rom.id"> - + {{ rom.r_name }} {{ rom.file_name }} {{ rom.p_slug }} - - - + + + - + { - + Apply diff --git a/frontend/src/views/Gallery.vue b/frontend/src/views/Gallery.vue index aa690c179..1803c7eb7 100644 --- a/frontend/src/views/Gallery.vue +++ b/frontend/src/views/Gallery.vue @@ -62,7 +62,7 @@ onBeforeRouteUpdate(async (to, _) => { getRoms(to.params.platform) }) - + { getRoms(to.params.platform) }) - + @@ -87,13 +87,13 @@ onBeforeRouteUpdate(async (to, _) => { getRoms(to.params.platform) }) - -
Feels cold here... mdi-emoticon-sad
+ +
Feels cold here... mdi-emoticon-sad
 - - - + + + From ce491e56012c9919004751fc7df98f03a6505bf1 Mon Sep 17 00:00:00 2001 From: zurdi zurdo Date: Thu, 27 Apr 2023 15:15:52 +0200 Subject: [PATCH 23/45] details info content fixed for sm sizes --- frontend/src/views/Details.vue | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/Details.vue b/frontend/src/views/Details.vue index 6f1e4ec6d..bc286f3b9 100644 --- a/frontend/src/views/Details.vue +++ b/frontend/src/views/Details.vue @@ -29,7 +29,7 @@ const deleteFromFs = ref(false) const filesToDownload = ref([]) const downloader = storeDownloader() const tab = ref('info') -const { xs, sm, mdAndUp, mdAndDown } = useDisplay() +const { xs, sm, mdAndUp, mdAndDown, smAndDown } = useDisplay() // Event listeners bus const emitter = inject('emitter') @@ -95,7 +95,7 @@ onMounted(() => {