diff --git a/README.md b/README.md index 7124073..632bfca 100644 --- a/README.md +++ b/README.md @@ -1,411 +1,416 @@ -# Magisk Modules Repo Util - -This util is to build module repository for [MMRL](https://github.com/DerGoogler/MMRL) - -- `sync` is a python package -- `cli.py` is a cli tool - -## Getting Started - -### Install dependencies - -```shell -pip3 install -r util/requirements.txt -``` - -### New config.json - -You can write it to `your-repo/json/config.json` by yourself, or - -```shell -cli.py config --stdin << EOF -{ - "name": "Your Magisk Repo", - "base_url": "https://you.github.io/magisk-modules-repo/", - "max_num": 3, - "enable_log": true, - "log_dir": "log" -} -EOF -``` - -or - -```shell -cli.py config --write name="Your Magisk Repo" base_url="https://you.github.io/magisk-modules-repo/" max_num=3 enable_log=true log_dir="log" -``` - -### New track.json - -You can write it to `your-repo/modules/{id}/track.json` by yourself, or - -```shell -cli.py track --stdin << EOF -{ - "id": "zygisk_lsposed", - "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", - "license": "GPL-3.0" -} -EOF -``` - -or - -```shell -cli.py track --add id="zygisk_lsposed" update_to="https://lsposed.github.io/LSPosed/release/zygisk.json" license="GPL-3.0" -``` - -If you want to generate `track.json`s from repositories on github - -```shell -cli.py github --token -u -r -``` - -> [!TIP] -> Click [here](https://github.com/settings/personal-access-tokens/new) to create a new api token. - -### Sync - -```shell -cli.py sync -``` - -### Generate a sitemap - -```shell -cli.py sitemap --base-url "https://mmrl.dergoogler.com/?module=" -``` - -## How to update by GitHub Actions? - -- You can refer to [GMR](https://github.com/Googlers-Repo/gmr). - -## cli.py - -``` -cli.py --help -usage: cli.py [-h] [-v] [-V] command ... - -Magisk Modules Repo Util - -positional arguments: - command - config Modify config of repository. - track Module tracks utility. - github Generate tracks from GitHub. - sync Sync modules in repository. - index Generate modules.json from local. - check Content check and migrate. - sitemap Sitemap generator. - -options: - -h, --help Show this help message and exit. - -v, --version Show util version and exit. - -V, --version-code Show util version code and exit. -``` - -## config.json - -```json -{ - "name": "Googlers Magisk Repo", - "website": "https://mmrl.dergoogler.com", - "support": "https://github.com/Googlers-Repo/repo/issues", - "donate": "https://github.com/sponsors/DerGoogler", - "submission": null, - "base_url": "https://gr.dergoogler.com/repo/", - "max_num": 3, - "enable_log": true, - "log_dir": "log" -} -``` - -| Key | Attribute | Description | -| ---------- | --------- | ----------------------------------------------- | -| name | required | Name of your module repository | -| base_url | required | Need to end with `/` | -| website | optional | Name of your website | -| donate | optional | Name of your donation url | -| submission | optional | Link to your submission requests | -| support | optional | Link to your support chat | -| max_num | optional | Max num of versions for modules, default is `3` | -| enable_log | optional | default is `true` | -| log_dir | optional | default is `null` | - -## track.json - -```json -{ - "id": "str", - "enable": "bool", - "verified": "bool", - "update_to": "str", - "source": "str", - "readme": "str", - "max_num": "int", - "antifeatures": ["array"] -} -``` - -| Key | Attribute | Type | Description | -| ------------ | --------- | ----- | ------------------------------------------------- | -| id | required | Str | Id of Module (_in `module.prop`_) | -| enable | required | Bool | Whether to enable updates | -| update_to | required | Str | Follow examples below | -| source | optional | Str | Url of where the source code lives | -| homepage | optional | Str | URL | -| readme | optional | Str | URL with e.g. description, instructions | -| changelog | optional | Str | Markdown or Simple Text (**_no HTML_**) | -| support | optional | Str | URL to issue tracker/support forum | -| donate | optional | Str | URL to donation page | -| cover | optional | Str | URL to cover image (featureGraphic) | -| icon | optional | Str | URL to icon.png (squared, max 512x512 px) | -| screenshots | optional | Str[] | URLs to screenshots of the module | -| license | optional | Str | SPDX identifier (see below) | -| antifeatures | optional | Str[] | potentially unwanted "features" (see below) | -| category | optional | Str | category the module belongs to (deprecated) | -| categories | optional | Str[] | array of categories the module belongs to | -| require | optional | Str[] | array of `module_id`s this module depends on | -| verified | optional | Bool | if module has good quality and is well maintained | -| max_num | optional | Int | Overload `MAX_NUM` in `config.json` | -| versions | auto | Int | how many versions are present (do not touch!) | - -Examples for antifeatures and their meanings can e.g. be [found here](https://gitlab.com/IzzyOnDroid/repo/-/blob/master/lib/antifeatures.json). - -For SPDX identifiers, see the [SPDX license list](https://spdx.org/licenses/). - -## `common/repo.json` - -> [!IMPORTANT] -> This file can be placed in the modules root directory. If a repo owner has added your module to his repo he can override those fields with the `track.json` file - -```jsonc -{ - "support": "str", - "donate": "str", - "cover": "str", - "icon": "str", - "license": "str", - "homepage": "str", - "readme": "str", - "screenshots": ["array"], - "category": "str", - "categories": ["array"], - "require": ["array"], - "note": { - "title": "str" // optional - "color": "red,blue,yellow,green", // optional - "message": "str" // required - } -} -``` - -| Key | Attribute | Description | -| ------------ | --------- | ----------- | -| license | optional | SPDX ID | -| cover | optional | Url | -| icon | optional | Url | -| readme | optional | Str | -| screenshots | optional | Url[] | -| antifeatures | optional | Str[] | -| category | optional | Str | -| categories | optional | Str[] | -| homepage | optional | Url | -| support | optional | Url | -| donate | optional | Url | -| note | optional | Note | - -### Update from updateJson - -> For those modules that provide [updateJson](https://topjohnwu.github.io/Magisk/guides.html#moduleprop). - -```json -{ - "id": "zygisk_lsposed", - "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", - "license": "GPL-3.0" -} -``` - -### Update from local updateJson - -> _update_to_ requires a relative directory of _local_. - -```json -{ - "id": "zygisk_lsposed", - "update_to": "zygisk.json", - "license": "GPL-3.0" -} -``` - -### Update from url - -> For those have a same url to release new modules. - -```json -{ - "id": "zygisk_lsposed", - "update_to": "https://github.com/LSPosed/LSPosed/releases/download/v1.8.6/LSPosed-v1.8.6-6712-zygisk-release.zip", - "license": "GPL-3.0", - "changelog": "https://lsposed.github.io/LSPosed/release/changelog.md" -} -``` - -### Update from git - -> For those we can get module by packaging all files in the repository, such as [Magisk-Modules-Repo](https://github.com/Magisk-Modules-Repo) and [Magisk-Modules-Alt-Repo](https://github.com/Magisk-Modules-Alt-Repo). - -```json -{ - "id": "busybox-ndk", - "update_to": "https://github.com/Magisk-Modules-Repo/busybox-ndk.git" -} -``` - -### Update from local zip - -> _update_to_ and _changelog_ requires a relative directory of _local_. - -```json -{ - "id": "zygisk_lsposed", - "update_to": "LSPosed-v1.8.6-6712-zygisk-release.zip", - "license": "GPL-3.0", - "changelog": "changelog.md" -} -``` - -## For developer - -``` -your-repo -├── json -│   ├── config.json -│   └── modules.json -│ -├── local -│   ├── ... -│ └── ... -│ -├── log -│   ├── sync_2023-03-18.log -│   ├── ... -│   └── ... -│ -├── modules -│   ├── zygisk_lsposed -│   │   ├── track.json -│   │   ├── update.json -│   │   ├── v1.8.6_6712.md -│   │   ├── v1.8.6_6712.zip -│   │   ├── ... -│   │   └── ... -│ │ -│ ├── another_module -│   │   ├── ... -│   │   └── ... -│ └── . -│ -└── util -``` - -### update.json - -```json -{ - "id": "zygisk_lsposed", - "timestamp": 1673882223.0, - "versions": [ - { - "timestamp": 1673882223.0, - "version": "v1.8.6 (6712)", - "versionCode": 6712, - "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", - "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" - } - ] -} -``` - -### track.json - -```json -{ - "id": "zygisk_lsposed", - "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", - "license": "GPL-3.0", - "homepage": "https://lsposed.org/", - "source": "https://github.com/LSPosed/LSPosed.git", - "support": "https://github.com/LSPosed/LSPosed/issues", - "added": 1679025505.129431, - "last_update": 1673882223.0, - "versions": 1 -} -``` - -## modules.json - -### version 1 - -```json -{ - "name": "{name}", - "metadata": { - "version": 1, - "timestamp": 1692439764.10608 - }, - "modules": [ - { - "id": "zygisk_lsposed", - "name": "Zygisk - LSPosed", - "version": "v1.8.6 (6712)", - "versionCode": 6712, - "author": "LSPosed Developers", - "description": "Another enhanced implementation of Xposed Framework. Supports Android 8.1 ~ 13. Requires Magisk 24.0+ and Zygisk enabled.", - "track": { - "type": "ONLINE_JSON", - "added": 1679025505.129431, - "license": "GPL-3.0", - "homepage": "https://lsposed.org/", - "source": "https://github.com/LSPosed/LSPosed.git", - "support": "https://github.com/LSPosed/LSPosed/issues", - "donate": "" - }, - "versions": [ - { - "timestamp": 1673882223.0, - "version": "v1.8.6 (6712)", - "versionCode": 6712, - "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", - "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" - } - ] - } - ] -} -``` - -### version 0 - -```json -{ - "name": "{name}", - "timestamp": 1692439602.46997, - "modules": [ - { - "id": "zygisk_lsposed", - "name": "Zygisk - LSPosed", - "version": "v1.8.6 (6712)", - "versionCode": 6712, - "author": "LSPosed Developers", - "description": "Another enhanced implementation of Xposed Framework. Supports Android 8.1 ~ 13. Requires Magisk 24.0+ and Zygisk enabled.", - "license": "GPL-3.0", - "states": { - "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", - "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" - } - } - ] -} -``` +# Magisk Modules Repo Util + +This util is to build module repository for [MMRL](https://github.com/DerGoogler/MMRL) + +- `sync` is a python package +- `cli.py` is a cli tool + +## Getting Started + +### Install dependencies + +```shell +pip3 install -r util/requirements.txt +``` + +### New config.json + +You can write it to `your-repo/json/config.json` by yourself, or + +```shell +cli.py config --stdin << EOF +{ + "name": "Your Magisk Repo", + "base_url": "https://you.github.io/magisk-modules-repo/", + "max_num": 3, + "enable_log": true, + "log_dir": "log" +} +EOF +``` + +or + +```shell +cli.py config --write name="Your Magisk Repo" base_url="https://you.github.io/magisk-modules-repo/" max_num=3 enable_log=true log_dir="log" +``` + +### New track.json + +You can write it to `your-repo/modules/{id}/track.json` by yourself, or + +```shell +cli.py track --stdin << EOF +{ + "id": "zygisk_lsposed", + "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", + "license": "GPL-3.0" +} +EOF +``` + +or + +```shell +cli.py track --add id="zygisk_lsposed" update_to="https://lsposed.github.io/LSPosed/release/zygisk.json" license="GPL-3.0" +``` + +If you want to generate `track.json`s from repositories on github + +```shell +cli.py github --token -u -r +``` + +> [!TIP] +> Click [here](https://github.com/settings/personal-access-tokens/new) to create a new api token. + +### Sync + +```shell +cli.py sync +``` + +### Generate a sitemap + +```shell +cli.py sitemap --base-url "https://mmrl.dergoogler.com/?module=" +``` + +## How to update by GitHub Actions? + +- You can refer to [GMR](https://github.com/Googlers-Repo/gmr). + +## cli.py + +``` +cli.py --help +usage: cli.py [-h] [-v] [-V] command ... + +Magisk Modules Repo Util + +positional arguments: + command + config Modify config of repository. + track Module tracks utility. + github Generate tracks from GitHub. + sync Sync modules in repository. + index Generate modules.json from local. + check Content check and migrate. + sitemap Sitemap generator. + +options: + -h, --help Show this help message and exit. + -v, --version Show util version and exit. + -V, --version-code Show util version code and exit. +``` + +## config.json + +```json +{ + "name": "Googlers Magisk Repo", + "website": "https://mmrl.dergoogler.com", + "support": "https://github.com/Googlers-Repo/repo/issues", + "donate": "https://github.com/sponsors/DerGoogler", + "submission": null, + "base_url": "https://gr.dergoogler.com/repo/", + "max_num": 3, + "enable_log": true, + "log_dir": "log" +} +``` + +| Key | Attribute | Description | +| ---------- | --------- | ----------------------------------------------- | +| name | required | Name of your module repository | +| base_url | required | Need to end with `/` | +| website | optional | Name of your website | +| donate | optional | Name of your donation url | +| submission | optional | Link to your submission requests | +| support | optional | Link to your support chat | +| max_num | optional | Max num of versions for modules, default is `3` | +| enable_log | optional | default is `true` | +| log_dir | optional | default is `null` | + +## track.json + +```json +{ + "id": "str", + "enable": "bool", + "verified": "bool", + "update_to": "str", + "source": "str", + "readme": "str", + "max_num": "int", + "antifeatures": ["array"] +} +``` + +| Key | Attribute | Type | Description | +| ------------ | --------- | ----- | ------------------------------------------------- | +| id | required | Str | Id of Module (_in `module.prop`_) | +| enable | required | Bool | Whether to enable updates | +| update_to | required | Str | Follow examples below | +| source | optional | Str | Url of where the source code lives | +| homepage | optional | Str | URL | +| readme | optional | Str | URL with e.g. description, instructions | +| changelog | optional | Str | Markdown or Simple Text (**_no HTML_**) | +| support | optional | Str | URL to issue tracker/support forum | +| donate | optional | Str | URL to donation page | +| cover | optional | Str | URL to cover image (featureGraphic) | +| icon | optional | Str | URL to icon.png (squared, max 512x512 px) | +| screenshots | optional | Str[] | URLs to screenshots of the module | +| license | optional | Str | SPDX identifier (see below) | +| antifeatures | optional | Str[] | potentially unwanted "features" (see below) | +| category | optional | Str | category the module belongs to (deprecated) | +| categories | optional | Str[] | array of categories the module belongs to | +| require | optional | Str[] | array of `module_id`s this module depends on | +| verified | optional | Bool | if module has good quality and is well maintained | +| max_num | optional | Int | Overload `MAX_NUM` in `config.json` | +| versions | auto | Int | how many versions are present (do not touch!) | + +Examples for antifeatures and their meanings can e.g. be [found here](https://gitlab.com/IzzyOnDroid/repo/-/blob/master/lib/antifeatures.json). + +For SPDX identifiers, see the [SPDX license list](https://spdx.org/licenses/). + +## `common/repo.json` + +> [!IMPORTANT] +> This file can be placed in the modules root directory. If a repo owner has added your module to his repo he can override those fields with the `track.json` file + +```jsonc +{ + "support": "str", + "donate": "str", + "cover": "str", + "icon": "str", + "license": "str", + "homepage": "str", + "readme": "str", + "screenshots": ["array"], + "category": "str", + "categories": ["array"], + "require": ["array"], + "note": { + "title": "str", // optional + "color": "red,blue,yellow,green", // optional + "message": "str" // required + }, + "root": { + "kernelsu": ">= 1.0.0", + "magisk": ">= 24.0 + } +} +``` + +| Key | Attribute | Description | +| ------------ | --------- | ------------- | +| license | optional | SPDX ID | +| cover | optional | Url | +| icon | optional | Url | +| readme | optional | Str | +| screenshots | optional | Url[] | +| antifeatures | optional | Str[] | +| category | optional | Str | +| categories | optional | Str[] | +| homepage | optional | Url | +| support | optional | Url | +| donate | optional | Url | +| note | optional | Note | +| root | optional | RootSolutions | + +### Update from updateJson + +> For those modules that provide [updateJson](https://topjohnwu.github.io/Magisk/guides.html#moduleprop). + +```json +{ + "id": "zygisk_lsposed", + "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", + "license": "GPL-3.0" +} +``` + +### Update from local updateJson + +> _update_to_ requires a relative directory of _local_. + +```json +{ + "id": "zygisk_lsposed", + "update_to": "zygisk.json", + "license": "GPL-3.0" +} +``` + +### Update from url + +> For those have a same url to release new modules. + +```json +{ + "id": "zygisk_lsposed", + "update_to": "https://github.com/LSPosed/LSPosed/releases/download/v1.8.6/LSPosed-v1.8.6-6712-zygisk-release.zip", + "license": "GPL-3.0", + "changelog": "https://lsposed.github.io/LSPosed/release/changelog.md" +} +``` + +### Update from git + +> For those we can get module by packaging all files in the repository, such as [Magisk-Modules-Repo](https://github.com/Magisk-Modules-Repo) and [Magisk-Modules-Alt-Repo](https://github.com/Magisk-Modules-Alt-Repo). + +```json +{ + "id": "busybox-ndk", + "update_to": "https://github.com/Magisk-Modules-Repo/busybox-ndk.git" +} +``` + +### Update from local zip + +> _update_to_ and _changelog_ requires a relative directory of _local_. + +```json +{ + "id": "zygisk_lsposed", + "update_to": "LSPosed-v1.8.6-6712-zygisk-release.zip", + "license": "GPL-3.0", + "changelog": "changelog.md" +} +``` + +## For developer + +``` +your-repo +├── json +│   ├── config.json +│   └── modules.json +│ +├── local +│   ├── ... +│ └── ... +│ +├── log +│   ├── sync_2023-03-18.log +│   ├── ... +│   └── ... +│ +├── modules +│   ├── zygisk_lsposed +│   │   ├── track.json +│   │   ├── update.json +│   │   ├── v1.8.6_6712.md +│   │   ├── v1.8.6_6712.zip +│   │   ├── ... +│   │   └── ... +│ │ +│ ├── another_module +│   │   ├── ... +│   │   └── ... +│ └── . +│ +└── util +``` + +### update.json + +```json +{ + "id": "zygisk_lsposed", + "timestamp": 1673882223.0, + "versions": [ + { + "timestamp": 1673882223.0, + "version": "v1.8.6 (6712)", + "versionCode": 6712, + "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", + "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" + } + ] +} +``` + +### track.json + +```json +{ + "id": "zygisk_lsposed", + "update_to": "https://lsposed.github.io/LSPosed/release/zygisk.json", + "license": "GPL-3.0", + "homepage": "https://lsposed.org/", + "source": "https://github.com/LSPosed/LSPosed.git", + "support": "https://github.com/LSPosed/LSPosed/issues", + "added": 1679025505.129431, + "last_update": 1673882223.0, + "versions": 1 +} +``` + +## modules.json + +### version 1 + +```json +{ + "name": "{name}", + "metadata": { + "version": 1, + "timestamp": 1692439764.10608 + }, + "modules": [ + { + "id": "zygisk_lsposed", + "name": "Zygisk - LSPosed", + "version": "v1.8.6 (6712)", + "versionCode": 6712, + "author": "LSPosed Developers", + "description": "Another enhanced implementation of Xposed Framework. Supports Android 8.1 ~ 13. Requires Magisk 24.0+ and Zygisk enabled.", + "track": { + "type": "ONLINE_JSON", + "added": 1679025505.129431, + "license": "GPL-3.0", + "homepage": "https://lsposed.org/", + "source": "https://github.com/LSPosed/LSPosed.git", + "support": "https://github.com/LSPosed/LSPosed/issues", + "donate": "" + }, + "versions": [ + { + "timestamp": 1673882223.0, + "version": "v1.8.6 (6712)", + "versionCode": 6712, + "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", + "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" + } + ] + } + ] +} +``` + +### version 0 + +```json +{ + "name": "{name}", + "timestamp": 1692439602.46997, + "modules": [ + { + "id": "zygisk_lsposed", + "name": "Zygisk - LSPosed", + "version": "v1.8.6 (6712)", + "versionCode": 6712, + "author": "LSPosed Developers", + "description": "Another enhanced implementation of Xposed Framework. Supports Android 8.1 ~ 13. Requires Magisk 24.0+ and Zygisk enabled.", + "license": "GPL-3.0", + "states": { + "zipUrl": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.zip", + "changelog": "{base_url}modules/zygisk_lsposed/v1.8.6_(6712)_6712.md" + } + } + ] +} +``` diff --git a/sync/__version__.py b/sync/__version__.py index e7f9350..a84d244 100644 --- a/sync/__version__.py +++ b/sync/__version__.py @@ -1,12 +1,12 @@ -def get_version() -> str: - return "2.7.6" - - -def get_version_code() -> int: - return 276 - - -__all__ = [ - "get_version", - "get_version_code" -] +def get_version() -> str: + return "2.7.7" + + +def get_version_code() -> int: + return 277 + + +__all__ = [ + "get_version", + "get_version_code" +] diff --git a/sync/core/Index.py b/sync/core/Index.py index 5701e6d..94118f3 100644 --- a/sync/core/Index.py +++ b/sync/core/Index.py @@ -1,172 +1,172 @@ -from datetime import datetime - -from git import Repo -from tabulate import tabulate - -from .Config import Config -from ..error import Result -from ..model import ( - AttrDict, - ModulesJson, - UpdateJson, - LocalModule, - OnlineModule -) -from ..track import LocalTracks -from ..utils import Log - - -class Index: - versions = [0, 1] - latest_version = versions[-1] - - def __init__(self, root_folder, config): - self._log = Log("Index", enable_log=config.enable_log, log_dir=config.log_dir) - self._root_folder = root_folder - - self._modules_folder = Config.get_modules_folder(root_folder) - self._tracks = LocalTracks(self._modules_folder, config) - self._config = config - - self._json_folder = Config.get_json_folder(root_folder) - - # noinspection PyTypeChecker - self.modules_json = None - - def _add_modules_json_0(self, track, update_json, online_module): - if self.modules_json is None: - self.modules_json = ModulesJson( - name=self._config.name, - timestamp=datetime.now().timestamp(), - modules=list() - ) - - latest_item = update_json.versions[-1] - - online_module.license = track.license or "" - online_module.states = AttrDict( - zipUrl=latest_item.zipUrl, - changelog=latest_item.changelog - ) - - self.modules_json.modules.append(online_module) - - def _add_modules_json_1(self, track, update_json, online_module): - if self.modules_json is None: - self.modules_json = ModulesJson( - name=self._config.name, - website=self._config.website, - support=self._config.support, - donate=self._config.donate, - submission=self._config.submission, - metadata=AttrDict( - version=1, - timestamp=datetime.now().timestamp() - ), - modules=list() - ) - - online_module.track = track.json() - online_module.versions = update_json.versions - - self.modules_json.modules.append(online_module) - - def _add_modules_json(self, track, update_json, online_module, version): - if version not in self.versions: - raise RuntimeError(f"unsupported version: {version}") - - func = getattr(self, f"_add_modules_json_{version}") - func( - track=track, - update_json=update_json, - online_module=online_module, - ) - - def get_online_module(self, track, zip_file): - @Result.catching() - def get_online_module(): - local_module = LocalModule.load(zip_file, track, self._config) - return OnlineModule.from_dict(local_module) - - result = get_online_module() - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"get_online_module: [{track.id}] -> {msg}") - return None - - else: - return result.value - - def __call__(self, version, to_file): - for track in self._tracks.get_tracks(): - module_folder = self._modules_folder.joinpath(track.id) - update_json_file = module_folder.joinpath(UpdateJson.filename()) - if not update_json_file.exists(): - continue - - update_json = UpdateJson.load(update_json_file) - latest_item = update_json.versions[-1] - - zip_file = module_folder.joinpath(latest_item.zipfile_name) - if not zip_file.exists(): - continue - - online_module = self.get_online_module(track, zip_file) - if online_module is None: - continue - - self._add_modules_json( - track=track, - update_json=update_json, - online_module=online_module, - version=version - ) - - self.modules_json.modules.sort(key=lambda v: v.id) - if to_file: - json_file = self._json_folder.joinpath(ModulesJson.filename()) - self.modules_json.write(json_file) - - return self.modules_json - - def push_by_git(self, branch): - json_file = self._json_folder.joinpath(ModulesJson.filename()) - timestamp = ModulesJson.load(json_file).get_timestamp() - msg = f"Update by CLI ({datetime.fromtimestamp(timestamp)})" - - repo = Repo(self._root_folder) - repo.git.add(all=True) - repo.index.commit(msg) - repo.remote().push(branch) - - def get_versions_table(self): - headers = ["id", "name", "latest version"] - table = [] - - for track in self._tracks.get_tracks(): - module_folder = self._modules_folder.joinpath(track.id) - update_json_file = module_folder.joinpath(UpdateJson.filename()) - - if not update_json_file.exists(): - table.append( - [track.id, "-", "-"] - ) - continue - - update_json = UpdateJson.load(update_json_file) - latest = update_json.versions[-1] - zip_file = module_folder.joinpath(latest.zipfile_name) - online_module = self.get_online_module(track, zip_file) - - if online_module is not None: - name = online_module.name.replace("|", "-") - table.append( - [online_module.id, name, online_module.version_display] - ) - else: - table.append( - [track.id, "-", "-"] - ) - - markdown_text = tabulate(table, headers, tablefmt="github") - return markdown_text +from datetime import datetime + +from git import Repo +from tabulate import tabulate + +from .Config import Config +from ..error import Result +from ..model import ( + AttrDict, + ModulesJson, + UpdateJson, + LocalModule, + OnlineModule +) +from ..track import LocalTracks +from ..utils import Log + + +class Index: + versions = [0, 1] + latest_version = versions[-1] + + def __init__(self, root_folder, config): + self._log = Log("Index", enable_log=config.enable_log, log_dir=config.log_dir) + self._root_folder = root_folder + + self._modules_folder = Config.get_modules_folder(root_folder) + self._tracks = LocalTracks(self._modules_folder, config) + self._config = config + + self._json_folder = Config.get_json_folder(root_folder) + + # noinspection PyTypeChecker + self.modules_json = None + + def _add_modules_json_0(self, track, update_json, online_module): + if self.modules_json is None: + self.modules_json = ModulesJson( + name=self._config.name, + timestamp=datetime.now().timestamp(), + modules=list() + ) + + latest_item = update_json.versions[-1] + + online_module.license = track.license or "" + online_module.states = AttrDict( + zipUrl=latest_item.zipUrl, + changelog=latest_item.changelog + ) + + self.modules_json.modules.append(online_module) + + def _add_modules_json_1(self, track, update_json, online_module): + if self.modules_json is None: + self.modules_json = ModulesJson( + name=self._config.name, + website=self._config.website, + support=self._config.support, + donate=self._config.donate, + submission=self._config.submission, + metadata=AttrDict( + version=1, + timestamp=datetime.now().timestamp() + ), + modules=list() + ) + + online_module.track = track.json() + online_module.versions = update_json.versions + + self.modules_json.modules.append(online_module) + + def _add_modules_json(self, track, update_json, online_module, version): + if version not in self.versions: + raise RuntimeError(f"unsupported version: {version}") + + func = getattr(self, f"_add_modules_json_{version}") + func( + track=track, + update_json=update_json, + online_module=online_module, + ) + + def get_online_module(self, track, zip_file): + @Result.catching() + def get_online_module(): + local_module = LocalModule.load(zip_file, track, self._config) + return OnlineModule.from_dict(local_module) + + result = get_online_module() + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"get_online_module: [{track.id}] -> {msg}") + return None + + else: + return result.value + + def __call__(self, version, to_file): + for track in self._tracks.get_tracks(): + module_folder = self._modules_folder.joinpath(track.id) + update_json_file = module_folder.joinpath(UpdateJson.filename()) + if not update_json_file.exists(): + continue + + update_json = UpdateJson.load(update_json_file) + latest_item = update_json.versions[-1] + + zip_file = module_folder.joinpath(latest_item.zipfile_name) + if not zip_file.exists(): + continue + + online_module = self.get_online_module(track, zip_file) + if online_module is None: + continue + + self._add_modules_json( + track=track, + update_json=update_json, + online_module=online_module, + version=version + ) + + self.modules_json.modules.sort(key=lambda v: v.id) + if to_file: + json_file = self._json_folder.joinpath(ModulesJson.filename()) + self.modules_json.write(json_file) + + return self.modules_json + + def push_by_git(self, branch): + json_file = self._json_folder.joinpath(ModulesJson.filename()) + timestamp = ModulesJson.load(json_file).get_timestamp() + msg = f"Update by CLI ({datetime.fromtimestamp(timestamp)})" + + repo = Repo(self._root_folder) + repo.git.add(all=True) + repo.index.commit(msg) + repo.remote().push(branch) + + def get_versions_table(self): + headers = ["id", "name", "latest version"] + table = [] + + for track in self._tracks.get_tracks(): + module_folder = self._modules_folder.joinpath(track.id) + update_json_file = module_folder.joinpath(UpdateJson.filename()) + + if not update_json_file.exists(): + table.append( + [track.id, "-", "-"] + ) + continue + + update_json = UpdateJson.load(update_json_file) + latest = update_json.versions[-1] + zip_file = module_folder.joinpath(latest.zipfile_name) + online_module = self.get_online_module(track, zip_file) + + if online_module is not None: + name = online_module.name.replace("|", "-") + table.append( + [online_module.id, name, online_module.version_display] + ) + else: + table.append( + [track.id, "-", "-"] + ) + + markdown_text = tabulate(table, headers, tablefmt="github") + return markdown_text diff --git a/sync/core/Pull.py b/sync/core/Pull.py index 8a0f4a4..ef8c6e9 100644 --- a/sync/core/Pull.py +++ b/sync/core/Pull.py @@ -1,265 +1,265 @@ -import shutil - -from .Config import Config -from ..error import Result -from ..model import ( - LocalModule, - AttrDict, - MagiskUpdateJson, - OnlineModule, - TrackType, - UpdateJson -) -from ..utils import ( - Log, - HttpUtils, - GitUtils, - StrUtils -) - - -class Pull: - _max_size = 50 - - def __init__(self, root_folder, config): - self._log = Log("Pull", enable_log=config.enable_log, log_dir=config.log_dir) - - self._local_folder = Config.get_local_folder(root_folder) - self._modules_folder = Config.get_modules_folder(root_folder) - self._config = config - - @staticmethod - def _copy_file(old, new, delete_old): - shutil.copy(old, new) - if delete_old: - old.unlink() - - @staticmethod - @Result.catching() - def _download(url, out): - return HttpUtils.download(url, out) - - def _check_changelog(self, module_id, file): - text = file.read_text() - if StrUtils.is_html(text): - self._log.w(f"_check_changelog: [{module_id}] -> unsupported type (html text)") - return False - else: - return True - - def _check_version_code(self, module_id, version_code): - module_folder = self._modules_folder.joinpath(module_id) - json_file = module_folder.joinpath(UpdateJson.filename()) - - if not json_file.exists(): - return True - - update_json = UpdateJson.load(json_file) - if version_code is None: - self._log.e(f"_check_version_code: [{module_id}] has no version code set") - return False - if len(update_json.versions) != 0 and version_code > update_json.versions[-1].versionCode: - return True - - self._log.i(f"_check_version_code: [{module_id}] -> already the latest version") - return False - - def _get_file_url(self, module_id, file): - module_folder = self._modules_folder.joinpath(module_id) - url = f"{self._config.base_url}{self._modules_folder.name}/{module_id}/{file.name}" - - if not (file.parent == module_folder and file.exists()): - raise FileNotFoundError(f"{file} is not in {module_folder}") - else: - return url - - def _get_changelog_common(self, module_id, changelog): - if changelog is None: - return None - elif isinstance(changelog, str) and changelog == "": - return None - - if StrUtils.is_url(changelog): - if StrUtils.is_blob_url(changelog): - msg = f"'{changelog}' is not unsupported type, please use 'https://raw.githubusercontent.com'" - self._log.w(f"_get_changelog_common: [{module_id}] -> {msg}") - return None - - changelog_file = self._modules_folder.joinpath(module_id, f"{module_id}.md") - result = self._download(changelog, changelog_file) - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"_get_changelog_common: [{module_id}] -> {msg}") - changelog_file = None - - else: - changelog_file = self._modules_folder.joinpath(module_id, f"{module_id}.md") - changelog_file.write_text(changelog) - - if changelog_file is not None: - if not self._check_changelog(module_id, changelog_file): - changelog_file.unlink() - changelog_file = None - - return changelog_file - - def _from_zip_common(self, track, zip_file, changelog_file, *, delete_tmp): - module_folder = self._modules_folder.joinpath(track.id) - - def remove_file(): - if delete_tmp: - zip_file.unlink() - if delete_tmp and changelog_file is not None: - changelog_file.unlink() - - zip_file_size = zip_file.stat().st_size / (1024 ** 2) - if zip_file_size > self._max_size: - new_module_folder = self._local_folder.joinpath(track.id) - msg = f"zip file is oversize ({self._max_size} MB), move this module to {new_module_folder}" - self._log.w(f"_from_zip_common: [{track.id}] -> {msg}") - shutil.rmtree(new_module_folder, ignore_errors=True) - shutil.move(module_folder, new_module_folder) - - return None - - @Result.catching() - def get_online_module(): - local_module = LocalModule.load(zip_file, track, self._config) - return OnlineModule.from_dict(local_module) - - result = get_online_module() - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"_from_zip_common: [{track.id}] -> {msg}") - remove_file() - return None - else: - online_module: OnlineModule = result.value - - target_zip_file = module_folder.joinpath(online_module.zipfile_name) - if self._check_version_code(track.id, online_module.versionCode): - self._copy_file(zip_file, target_zip_file, delete_tmp) - else: - remove_file() - return None - - target_changelog_file = module_folder.joinpath(online_module.changelog_filename) - changelog_url = "" - if changelog_file is not None: - self._copy_file(changelog_file, target_changelog_file, delete_tmp) - changelog_url = self._get_file_url(track.id, target_changelog_file) - - # For OnlineModule.to_VersionItem - online_module.latest = AttrDict( - zipUrl=self._get_file_url(track.id, target_zip_file), - size=target_zip_file.stat().st_size, - changelog=changelog_url - ) - - return online_module - - def from_json(self, track, *, local): - if local: - update_to = self._local_folder.joinpath(track.update_to) - else: - update_to = track.update_to - - @Result.catching() - def load_json(): - return MagiskUpdateJson.load(update_to) - - result = load_json() - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"from_json: [{track.id}] -> {msg}") - return None, 0.0 - else: - update_json: MagiskUpdateJson = result.value - - if not self._check_version_code(track.id, update_json.versionCode): - return None, 0.0 - - zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") - - result = self._download(update_json.zipUrl, zip_file) - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"from_json: [{track.id}] -> {msg}") - return None, 0.0 - else: - last_modified = result.value - - changelog = self._get_changelog_common(track.id, update_json.changelog) - online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) - return online_module, last_modified - - def from_url(self, track): - zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") - - result = self._download(track.update_to, zip_file) - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"from_url: [{track.id}] -> {msg}") - return None, 0.0 - else: - last_modified = result.value - - changelog = self._get_changelog_common(track.id, track.changelog) - online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) - return online_module, last_modified - - def from_git(self, track): - zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") - - @Result.catching() - def git_clone(): - return GitUtils.clone_and_zip(track.update_to, zip_file) - - result = git_clone() - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"from_git: [{track.id}] -> {msg}") - return None, 0.0 - else: - last_committed = result.value - - changelog = self._get_changelog_common(track.id, track.changelog) - online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) - return online_module, last_committed - - def from_zip(self, track): - zip_file = self._local_folder.joinpath(track.update_to) - changelog = self._local_folder.joinpath(track.changelog) - last_modified = zip_file.stat().st_mtime - - if not zip_file.exists(): - msg = f"{track.update_to} is not in {self._local_folder}" - self._log.i(f"from_zip: [{track.id}] -> {msg}") - return None, 0.0 - - if not changelog.exists(): - changelog = None - - online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=False) - return online_module, last_modified - - def from_track(self, track): - self._log.d(f"from_track: [{track.id}] -> type: {track.type.name}") - - if track.type == TrackType.ONLINE_JSON: - return self.from_json(track, local=False) - elif track.type == TrackType.ONLINE_ZIP: - return self.from_url(track) - elif track.type == TrackType.GIT: - return self.from_git(track) - elif track.type == TrackType.LOCAL_JSON: - return self.from_json(track, local=True) - elif track.type == TrackType.LOCAL_ZIP: - return self.from_zip(track) - - self._log.e(f"from_track: [{track.id}] -> unsupported type [{track.update_to}]") - return None, 0.0 - - @classmethod - def set_max_size(cls, value): - cls._max_size = value +import shutil + +from .Config import Config +from ..error import Result +from ..model import ( + LocalModule, + AttrDict, + MagiskUpdateJson, + OnlineModule, + TrackType, + UpdateJson +) +from ..utils import ( + Log, + HttpUtils, + GitUtils, + StrUtils +) + + +class Pull: + _max_size = 50 + + def __init__(self, root_folder, config): + self._log = Log("Pull", enable_log=config.enable_log, log_dir=config.log_dir) + + self._local_folder = Config.get_local_folder(root_folder) + self._modules_folder = Config.get_modules_folder(root_folder) + self._config = config + + @staticmethod + def _copy_file(old, new, delete_old): + shutil.copy(old, new) + if delete_old: + old.unlink() + + @staticmethod + @Result.catching() + def _download(url, out): + return HttpUtils.download(url, out) + + def _check_changelog(self, module_id, file): + text = file.read_text() + if StrUtils.is_html(text): + self._log.w(f"_check_changelog: [{module_id}] -> unsupported type (html text)") + return False + else: + return True + + def _check_version_code(self, module_id, version_code): + module_folder = self._modules_folder.joinpath(module_id) + json_file = module_folder.joinpath(UpdateJson.filename()) + + if not json_file.exists(): + return True + + update_json = UpdateJson.load(json_file) + if version_code is None: + self._log.e(f"_check_version_code: [{module_id}] has no version code set") + return False + if len(update_json.versions) != 0 and version_code > update_json.versions[-1].versionCode: + return True + + self._log.i(f"_check_version_code: [{module_id}] -> already the latest version") + return False + + def _get_file_url(self, module_id, file): + module_folder = self._modules_folder.joinpath(module_id) + url = f"{self._config.base_url}{self._modules_folder.name}/{module_id}/{file.name}" + + if not (file.parent == module_folder and file.exists()): + raise FileNotFoundError(f"{file} is not in {module_folder}") + else: + return url + + def _get_changelog_common(self, module_id, changelog): + if changelog is None: + return None + elif isinstance(changelog, str) and changelog == "": + return None + + if StrUtils.is_url(changelog): + if StrUtils.is_blob_url(changelog): + msg = f"'{changelog}' is not unsupported type, please use 'https://raw.githubusercontent.com'" + self._log.w(f"_get_changelog_common: [{module_id}] -> {msg}") + return None + + changelog_file = self._modules_folder.joinpath(module_id, f"{module_id}.md") + result = self._download(changelog, changelog_file) + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"_get_changelog_common: [{module_id}] -> {msg}") + changelog_file = None + + else: + changelog_file = self._modules_folder.joinpath(module_id, f"{module_id}.md") + changelog_file.write_text(changelog) + + if changelog_file is not None: + if not self._check_changelog(module_id, changelog_file): + changelog_file.unlink() + changelog_file = None + + return changelog_file + + def _from_zip_common(self, track, zip_file, changelog_file, *, delete_tmp): + module_folder = self._modules_folder.joinpath(track.id) + + def remove_file(): + if delete_tmp: + zip_file.unlink() + if delete_tmp and changelog_file is not None: + changelog_file.unlink() + + zip_file_size = zip_file.stat().st_size / (1024 ** 2) + if zip_file_size > self._max_size: + new_module_folder = self._local_folder.joinpath(track.id) + msg = f"zip file is oversize ({self._max_size} MB), move this module to {new_module_folder}" + self._log.w(f"_from_zip_common: [{track.id}] -> {msg}") + shutil.rmtree(new_module_folder, ignore_errors=True) + shutil.move(module_folder, new_module_folder) + + return None + + @Result.catching() + def get_online_module(): + local_module = LocalModule.load(zip_file, track, self._config) + return OnlineModule.from_dict(local_module) + + result = get_online_module() + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"_from_zip_common: [{track.id}] -> {msg}") + remove_file() + return None + else: + online_module: OnlineModule = result.value + + target_zip_file = module_folder.joinpath(online_module.zipfile_name) + if self._check_version_code(track.id, online_module.versionCode): + self._copy_file(zip_file, target_zip_file, delete_tmp) + else: + remove_file() + return None + + target_changelog_file = module_folder.joinpath(online_module.changelog_filename) + changelog_url = "" + if changelog_file is not None: + self._copy_file(changelog_file, target_changelog_file, delete_tmp) + changelog_url = self._get_file_url(track.id, target_changelog_file) + + # For OnlineModule.to_VersionItem + online_module.latest = AttrDict( + zipUrl=self._get_file_url(track.id, target_zip_file), + size=target_zip_file.stat().st_size, + changelog=changelog_url + ) + + return online_module + + def from_json(self, track, *, local): + if local: + update_to = self._local_folder.joinpath(track.update_to) + else: + update_to = track.update_to + + @Result.catching() + def load_json(): + return MagiskUpdateJson.load(update_to) + + result = load_json() + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"from_json: [{track.id}] -> {msg}") + return None, 0.0 + else: + update_json: MagiskUpdateJson = result.value + + if not self._check_version_code(track.id, update_json.versionCode): + return None, 0.0 + + zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") + + result = self._download(update_json.zipUrl, zip_file) + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"from_json: [{track.id}] -> {msg}") + return None, 0.0 + else: + last_modified = result.value + + changelog = self._get_changelog_common(track.id, update_json.changelog) + online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) + return online_module, last_modified + + def from_url(self, track): + zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") + + result = self._download(track.update_to, zip_file) + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"from_url: [{track.id}] -> {msg}") + return None, 0.0 + else: + last_modified = result.value + + changelog = self._get_changelog_common(track.id, track.changelog) + online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) + return online_module, last_modified + + def from_git(self, track): + zip_file = self._modules_folder.joinpath(track.id, f"{track.id}.zip") + + @Result.catching() + def git_clone(): + return GitUtils.clone_and_zip(track.update_to, zip_file) + + result = git_clone() + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"from_git: [{track.id}] -> {msg}") + return None, 0.0 + else: + last_committed = result.value + + changelog = self._get_changelog_common(track.id, track.changelog) + online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=True) + return online_module, last_committed + + def from_zip(self, track): + zip_file = self._local_folder.joinpath(track.update_to) + changelog = self._local_folder.joinpath(track.changelog) + last_modified = zip_file.stat().st_mtime + + if not zip_file.exists(): + msg = f"{track.update_to} is not in {self._local_folder}" + self._log.i(f"from_zip: [{track.id}] -> {msg}") + return None, 0.0 + + if not changelog.exists(): + changelog = None + + online_module = self._from_zip_common(track, zip_file, changelog, delete_tmp=False) + return online_module, last_modified + + def from_track(self, track): + self._log.d(f"from_track: [{track.id}] -> type: {track.type.name}") + + if track.type == TrackType.ONLINE_JSON: + return self.from_json(track, local=False) + elif track.type == TrackType.ONLINE_ZIP: + return self.from_url(track) + elif track.type == TrackType.GIT: + return self.from_git(track) + elif track.type == TrackType.LOCAL_JSON: + return self.from_json(track, local=True) + elif track.type == TrackType.LOCAL_ZIP: + return self.from_zip(track) + + self._log.e(f"from_track: [{track.id}] -> unsupported type [{track.update_to}]") + return None, 0.0 + + @classmethod + def set_max_size(cls, value): + cls._max_size = value diff --git a/sync/model/JsonIO.py b/sync/model/JsonIO.py index 5f217b3..3a02dc1 100644 --- a/sync/model/JsonIO.py +++ b/sync/model/JsonIO.py @@ -1,29 +1,29 @@ -import json -import re - -class JsonIO: - def write(self, file): - assert isinstance(self, dict) - - file.parent.mkdir(parents=True, exist_ok=True) - - with open(file, "w") as f: - json.dump(self, f, indent=2) - - @classmethod - def filter(cls, text): - return re.sub(r",(?=\s*?[}\]])", "", text) - - @classmethod - def filterArray(cls, filter, toFilter): - return [i for i in filter if i in toFilter] - - @classmethod - def load(cls, file): - with open(file, encoding="utf-8", mode="r") as f: - text = cls.filter(f.read()) - obj = json.loads(text) - - assert isinstance(obj, dict) - - return obj +import json +import re + +class JsonIO: + def write(self, file): + assert isinstance(self, dict) + + file.parent.mkdir(parents=True, exist_ok=True) + + with open(file, "w") as f: + json.dump(self, f, indent=2) + + @classmethod + def filter(cls, text): + return re.sub(r",(?=\s*?[}\]])", "", text) + + @classmethod + def filterArray(cls, filter, toFilter): + return [i for i in filter if i in toFilter] + + @classmethod + def load(cls, file): + with open(file, encoding="utf-8", mode="r") as f: + text = cls.filter(f.read()) + obj = json.loads(text) + + assert isinstance(obj, dict) + + return obj diff --git a/sync/model/JsonIO.pyi b/sync/model/JsonIO.pyi index 184a1d8..126ff42 100644 --- a/sync/model/JsonIO.pyi +++ b/sync/model/JsonIO.pyi @@ -1,12 +1,12 @@ -from pathlib import Path -from typing import Dict, List - - -class JsonIO: - def write(self: Dict, file: Path): ... - @classmethod - def filter(cls, text: str) -> str: ... - @classmethod - def filterArray(cls, filter: List[str], toFilter: List[str]) -> List[str]: ... - @classmethod - def load(cls, file: Path) -> Dict: ... +from pathlib import Path +from typing import Dict, List + + +class JsonIO: + def write(self: Dict, file: Path): ... + @classmethod + def filter(cls, text: str) -> str: ... + @classmethod + def filterArray(cls, filter: List[str], toFilter: List[str]) -> List[str]: ... + @classmethod + def load(cls, file: Path) -> Dict: ... diff --git a/sync/model/LocalModule.py b/sync/model/LocalModule.py index a61024a..411f5b6 100644 --- a/sync/model/LocalModule.py +++ b/sync/model/LocalModule.py @@ -1,148 +1,149 @@ -import json - -from zipfile import ZipFile -from pathlib import Path - -from .AttrDict import AttrDict -from ..error import MagiskModuleError - -from .JsonIO import JsonIO - -from .ModuleNote import ModuleNote -from .ModuleFeatures import ModuleFeatures - - -class LocalModule(AttrDict): - id: str - name: str - version: str - versionCode: int - author: str - description: str - - added: float - timestamp: float - size: float - - # FoxMMM supported props - maxApi: int - minApi: int - - # MMRL supported props - category: str - categories: list[str] - icon: str - homepage: str - donate: str - support: str - cover: str - screenshots: list[str] - license: str - screenshots: list[str] - readme: str - require: list[str] - verified: bool - note: ModuleNote - features: ModuleFeatures - - @classmethod - def load(cls, file, track, config): - cls._zipfile = ZipFile(file, "r") - fields = cls.expected_fields() - - try: - if ("#MAGISK" not in cls.file_read("META-INF/com/google/android/updater-script")): - raise - if (not cls.file_exist("META-INF/com/google/android/update-binary")): - raise - except BaseException: - msg = f"{file.name} is not a magisk module" - raise MagiskModuleError(msg) - - try: - props = cls.file_read( "module.prop") - except BaseException as err: - raise MagiskModuleError(err.args) - - obj = AttrDict() - for item in props.splitlines(): - prop = item.split("=", maxsplit=1) - if len(prop) != 2: - continue - - key, value = prop - if key == "" or key.startswith("#") or key not in fields: - continue - - _type = fields[key] - obj[key] = _type(value) - - local_module = LocalModule() - for key in fields.keys(): - if config.allowedCategories and key == "categories" and track.get("categories"): - local_module[key] = JsonIO.filterArray(config.allowedCategories, track.get(key)) - else: - value = track.get(key) if track.get(key) is not None else obj.get(key) - if value is not None and value is not False: # Filter out None and False values - local_module[key] = value - - try: - raw_json = json.loads(cls.file_read("common/repo.json")) - - for item in raw_json.items(): - key, value = item - - _type = fields[key] - obj[key] = _type(value) - - for key in fields.keys(): - value = obj.get(key) - if value is not None and value is not False: # Filter out None and False values - local_module[key] = value - - except BaseException: - pass - - local_module.verified = track.verified or False - local_module.added = track.added or 0 - local_module.timestamp = track.last_update - local_module.size = Path(file).stat().st_size - - features = { - "service": cls.file_exist(f"service.sh") or cls.file_exist(f"common/service.sh"), - "post_fs_data": cls.file_exist(f"post-fs-data.sh") or cls.file_exist(f"common/post-fs-data.sh"), - # system.prop - "resetprop": cls.file_exist(f"system.prop") or cls.file_exist(f"common/system.prop"), - "sepolicy": cls.file_exist(f"sepolicy.rule"), - - "zygisk": cls.file_exist(f"zygisk/"), - - # KernelSU - "webroot": cls.file_exist(f"webroot/index.html"), - "post_mount": cls.file_exist(f"post-mount.sh") or cls.file_exist(f"common/post-mount.sh"), - "boot_completed": cls.file_exist(f"boot-completed.sh") or cls.file_exist(f"common/boot-completed.sh"), - - # MMRL - "modconf": cls.file_exist(f"system/usr/share/mmrl/config/{local_module.id}/index.jsx"), - - "apks": len([name for name in cls._zipfile.namelist() if name.endswith('.apk')]) != 0 - } - - local_module.features = {k: v for k, v in features.items() if v is not None and v is not False} - - return local_module - - @classmethod - def file_exist(cls, name: str): - return name in cls._zipfile.namelist() - - @classmethod - def file_read(cls, name: str): - return cls._zipfile.read(name).decode("utf-8") - - @classmethod - def expected_fields(cls, __type=True): - if __type: - return cls.__annotations__ - - return {k: v.__name__ for k, v in cls.__annotations__.items() if v is not None and v is not False} +import json + +from zipfile import ZipFile +from pathlib import Path + +from .AttrDict import AttrDict +from ..error import MagiskModuleError + +from .JsonIO import JsonIO + +from .ModuleNote import ModuleNote +from .ModuleFeatures import ModuleFeatures +from .RootSolutions import RootSolutions + +class LocalModule(AttrDict): + id: str + name: str + version: str + versionCode: int + author: str + description: str + + added: float + timestamp: float + size: float + + # FoxMMM supported props + maxApi: int + minApi: int + + # MMRL supported props + category: str + categories: list[str] + icon: str + homepage: str + donate: str + support: str + cover: str + screenshots: list[str] + license: str + screenshots: list[str] + readme: str + require: list[str] + verified: bool + note: ModuleNote + features: ModuleFeatures + root: RootSolutions + + @classmethod + def load(cls, file, track, config): + cls._zipfile = ZipFile(file, "r") + fields = cls.expected_fields() + + try: + if ("#MAGISK" not in cls.file_read("META-INF/com/google/android/updater-script")): + raise + if (not cls.file_exist("META-INF/com/google/android/update-binary")): + raise + except BaseException: + msg = f"{file.name} is not a magisk module" + raise MagiskModuleError(msg) + + try: + props = cls.file_read( "module.prop") + except BaseException as err: + raise MagiskModuleError(err.args) + + obj = AttrDict() + for item in props.splitlines(): + prop = item.split("=", maxsplit=1) + if len(prop) != 2: + continue + + key, value = prop + if key == "" or key.startswith("#") or key not in fields: + continue + + _type = fields[key] + obj[key] = _type(value) + + local_module = LocalModule() + for key in fields.keys(): + if config.allowedCategories and key == "categories" and track.get("categories"): + local_module[key] = JsonIO.filterArray(config.allowedCategories, track.get(key)) + else: + value = track.get(key) if track.get(key) is not None else obj.get(key) + if value is not None and value is not False: # Filter out None and False values + local_module[key] = value + + try: + raw_json = json.loads(cls.file_read("common/repo.json")) + + for item in raw_json.items(): + key, value = item + + _type = fields[key] + obj[key] = _type(value) + + for key in fields.keys(): + value = obj.get(key) + if value is not None and value is not False: # Filter out None and False values + local_module[key] = value + + except BaseException: + pass + + local_module.verified = track.verified or False + local_module.added = track.added or 0 + local_module.timestamp = track.last_update + local_module.size = Path(file).stat().st_size + + features = { + "service": cls.file_exist(f"service.sh") or cls.file_exist(f"common/service.sh"), + "post_fs_data": cls.file_exist(f"post-fs-data.sh") or cls.file_exist(f"common/post-fs-data.sh"), + # system.prop + "resetprop": cls.file_exist(f"system.prop") or cls.file_exist(f"common/system.prop"), + "sepolicy": cls.file_exist(f"sepolicy.rule"), + + "zygisk": cls.file_exist(f"zygisk/"), + + # KernelSU + "webroot": cls.file_exist(f"webroot/index.html"), + "post_mount": cls.file_exist(f"post-mount.sh") or cls.file_exist(f"common/post-mount.sh"), + "boot_completed": cls.file_exist(f"boot-completed.sh") or cls.file_exist(f"common/boot-completed.sh"), + + # MMRL + "modconf": cls.file_exist(f"system/usr/share/mmrl/config/{local_module.id}/index.jsx"), + + "apks": len([name for name in cls._zipfile.namelist() if name.endswith('.apk')]) != 0 + } + + local_module.features = {k: v for k, v in features.items() if v is not None and v is not False} + + return local_module + + @classmethod + def file_exist(cls, name: str): + return name in cls._zipfile.namelist() + + @classmethod + def file_read(cls, name: str): + return cls._zipfile.read(name).decode("utf-8") + + @classmethod + def expected_fields(cls, __type=True): + if __type: + return cls.__annotations__ + + return {k: v.__name__ for k, v in cls.__annotations__.items() if v is not None and v is not False} diff --git a/sync/model/LocalModule.pyi b/sync/model/LocalModule.pyi index c678d41..7d3d21b 100644 --- a/sync/model/LocalModule.pyi +++ b/sync/model/LocalModule.pyi @@ -1,48 +1,48 @@ -from pathlib import Path -from typing import Dict, Type - -from .AttrDict import AttrDict -from .TrackJson import TrackJson - -from ..core.Config import Config - -from .ModuleNote import ModuleNote -from .ModuleFeatures import ModuleFeatures - -class LocalModule(AttrDict): - id: str - name: str - version: str - versionCode: int - author: str - description: str - - added: float - timestamp: float - size: float - - # FoxMMM supported props - maxApi: int - minApi: int - - # MMRL supported props - category: str - categories: list[str] - icon: str - homepage: str - donate: str - support: str - cover: str - screenshots: list[str] - license: str - screenshots: list[str] - readme: str - require: list[str] - verified: bool - note: ModuleNote - features: ModuleFeatures - - @classmethod - def load(cls, file: Path, track: TrackJson, config: Config) -> LocalModule: ... - @classmethod - def expected_fields(cls, __type: bool = ...) -> Dict[str, Type]: ... +from pathlib import Path +from typing import Dict, Type + +from .AttrDict import AttrDict +from .TrackJson import TrackJson + +from ..core.Config import Config + +from .ModuleNote import ModuleNote +from .ModuleFeatures import ModuleFeatures + +class LocalModule(AttrDict): + id: str + name: str + version: str + versionCode: int + author: str + description: str + + added: float + timestamp: float + size: float + + # FoxMMM supported props + maxApi: int + minApi: int + + # MMRL supported props + category: str + categories: list[str] + icon: str + homepage: str + donate: str + support: str + cover: str + screenshots: list[str] + license: str + screenshots: list[str] + readme: str + require: list[str] + verified: bool + note: ModuleNote + features: ModuleFeatures + + @classmethod + def load(cls, file: Path, track: TrackJson, config: Config) -> LocalModule: ... + @classmethod + def expected_fields(cls, __type: bool = ...) -> Dict[str, Type]: ... diff --git a/sync/model/ModuleFeatures.py b/sync/model/ModuleFeatures.py index ad06870..f1f3bf4 100644 --- a/sync/model/ModuleFeatures.py +++ b/sync/model/ModuleFeatures.py @@ -1,25 +1,25 @@ -from .AttrDict import AttrDict - -class ModuleFeatures(AttrDict): - service: bool - post_fs_data: bool - # system.prop - resetprop: bool - sepolicy: bool - zygisk: bool - apks: bool - - # KernelSU - webroot: bool - post_mount: bool - boot_completed: bool - - # MMRL - modconf: bool - - @classmethod - def expected_fields(cls, __type=True): - if __type: - return cls.__annotations__ - +from .AttrDict import AttrDict + +class ModuleFeatures(AttrDict): + service: bool + post_fs_data: bool + # system.prop + resetprop: bool + sepolicy: bool + zygisk: bool + apks: bool + + # KernelSU + webroot: bool + post_mount: bool + boot_completed: bool + + # MMRL + modconf: bool + + @classmethod + def expected_fields(cls, __type=True): + if __type: + return cls.__annotations__ + return {k: v.__name__ for k, v in cls.__annotations__.items()} \ No newline at end of file diff --git a/sync/model/ModuleFeatures.pyi b/sync/model/ModuleFeatures.pyi index 04aac82..3890701 100644 --- a/sync/model/ModuleFeatures.pyi +++ b/sync/model/ModuleFeatures.pyi @@ -1,24 +1,24 @@ -from typing import Dict, Type - -from .AttrDict import AttrDict - - -class ModuleFeatures(AttrDict): - service: bool - post_fs_data: bool - # system.prop - resetprop: bool - sepolicy: bool - zygisk: bool - apks: bool - - # KernelSU - webroot: bool - post_mount: bool - boot_completed: bool - - # MMRL - modconf: bool - - @classmethod - def expected_fields(cls, __type: bool = ...) -> Dict[str, Type]: ... +from typing import Dict, Type + +from .AttrDict import AttrDict + + +class ModuleFeatures(AttrDict): + service: bool + post_fs_data: bool + # system.prop + resetprop: bool + sepolicy: bool + zygisk: bool + apks: bool + + # KernelSU + webroot: bool + post_mount: bool + boot_completed: bool + + # MMRL + modconf: bool + + @classmethod + def expected_fields(cls, __type: bool = ...) -> Dict[str, Type]: ... diff --git a/sync/model/RootSolutions.py b/sync/model/RootSolutions.py new file mode 100644 index 0000000..c7491cb --- /dev/null +++ b/sync/model/RootSolutions.py @@ -0,0 +1,13 @@ +from .AttrDict import AttrDict + +class RootSolutions(AttrDict): + magisk: str + kernelsu: str + apatch: str + + @classmethod + def expected_fields(cls, __type=True): + if __type: + return cls.__annotations__ + + return {k: v.__name__ for k, v in cls.__annotations__.items()} \ No newline at end of file diff --git a/sync/model/RootSolutions.pyi b/sync/model/RootSolutions.pyi new file mode 100644 index 0000000..6b310d0 --- /dev/null +++ b/sync/model/RootSolutions.pyi @@ -0,0 +1,10 @@ +from .AttrDict import AttrDict +from typing import Dict, Type + +class RootSolutions(AttrDict): + magisk: str + kernelsu: str + apatch: str + + @classmethod + def expected_fields(cls, __type=True) -> Dict[str, Type]: ... \ No newline at end of file diff --git a/sync/model/TrackJson.py b/sync/model/TrackJson.py index 0646613..7f95b98 100644 --- a/sync/model/TrackJson.py +++ b/sync/model/TrackJson.py @@ -1,112 +1,112 @@ -from enum import Enum - -from .AttrDict import AttrDict -from .JsonIO import JsonIO - -from .ModuleNote import ModuleNote - -class TrackJson(AttrDict, JsonIO): - id: str - enable: bool - verified: bool - update_to: str - changelog: str - license: str - homepage: str - source: str - support: str - donate: str - max_num: int - # author: str - # contributors: list[str] - cover: str - icon:str - screenshots: list[str] - require: list[str] - category: str - categories: list[str] - readme: str - require: list[str] - antifeatures: list[str] - note: ModuleNote - - # noinspection PyAttributeOutsideInit - @property - def type(self): - if self._type is not None: - return self._type - - if self.update_to.startswith("http"): - if self.update_to.endswith(".json"): - self._type = TrackType.ONLINE_JSON - elif self.update_to.endswith(".zip"): - self._type = TrackType.ONLINE_ZIP - elif self.update_to.endswith(".git"): - self._type = TrackType.GIT - - elif self.update_to.startswith("git@"): - if self.update_to.endswith(".git"): - self._type = TrackType.GIT - - else: - if self.update_to.endswith(".json"): - self._type = TrackType.LOCAL_JSON - elif self.update_to.endswith(".zip"): - self._type = TrackType.LOCAL_ZIP - - if self._type is None: - self._type = TrackType.UNKNOWN - - return self._type - - def json(self): - return AttrDict( - type=self.type.name, - added=self.added, - source=self.source or None - ) - - def write(self, file): - new = AttrDict() - keys = list(self.expected_fields().keys()) - - # fields without manually - keys.extend(["added", "last_update", "versions"]) - - for key in keys: - value = self.get(key, "") - if value is None: - continue - - if isinstance(value, str): - if value == "" or value.isspace(): - continue - - new[key] = value - - JsonIO.write(new, file) - - @classmethod - def load(cls, file): - obj = JsonIO.load(file) - return TrackJson(obj) - - @classmethod - def filename(cls): - return "track.json" - - @classmethod - def expected_fields(cls, __type=True): - if __type: - return cls.__annotations__ - - return {k: v.__name__ for k, v in cls.__annotations__.items() if v is not None and v is not False} - - -class TrackType(Enum): - UNKNOWN = 0 - ONLINE_JSON = 1 - ONLINE_ZIP = 2 - GIT = 3 - LOCAL_JSON = 4 - LOCAL_ZIP = 5 +from enum import Enum + +from .AttrDict import AttrDict +from .JsonIO import JsonIO + +from .ModuleNote import ModuleNote + +class TrackJson(AttrDict, JsonIO): + id: str + enable: bool + verified: bool + update_to: str + changelog: str + license: str + homepage: str + source: str + support: str + donate: str + max_num: int + # author: str + # contributors: list[str] + cover: str + icon:str + screenshots: list[str] + require: list[str] + category: str + categories: list[str] + readme: str + require: list[str] + antifeatures: list[str] + note: ModuleNote + + # noinspection PyAttributeOutsideInit + @property + def type(self): + if self._type is not None: + return self._type + + if self.update_to.startswith("http"): + if self.update_to.endswith(".json"): + self._type = TrackType.ONLINE_JSON + elif self.update_to.endswith(".zip"): + self._type = TrackType.ONLINE_ZIP + elif self.update_to.endswith(".git"): + self._type = TrackType.GIT + + elif self.update_to.startswith("git@"): + if self.update_to.endswith(".git"): + self._type = TrackType.GIT + + else: + if self.update_to.endswith(".json"): + self._type = TrackType.LOCAL_JSON + elif self.update_to.endswith(".zip"): + self._type = TrackType.LOCAL_ZIP + + if self._type is None: + self._type = TrackType.UNKNOWN + + return self._type + + def json(self): + return AttrDict( + type=self.type.name, + added=self.added, + source=self.source or None + ) + + def write(self, file): + new = AttrDict() + keys = list(self.expected_fields().keys()) + + # fields without manually + keys.extend(["added", "last_update", "versions"]) + + for key in keys: + value = self.get(key, "") + if value is None: + continue + + if isinstance(value, str): + if value == "" or value.isspace(): + continue + + new[key] = value + + JsonIO.write(new, file) + + @classmethod + def load(cls, file): + obj = JsonIO.load(file) + return TrackJson(obj) + + @classmethod + def filename(cls): + return "track.json" + + @classmethod + def expected_fields(cls, __type=True): + if __type: + return cls.__annotations__ + + return {k: v.__name__ for k, v in cls.__annotations__.items() if v is not None and v is not False} + + +class TrackType(Enum): + UNKNOWN = 0 + ONLINE_JSON = 1 + ONLINE_ZIP = 2 + GIT = 3 + LOCAL_JSON = 4 + LOCAL_ZIP = 5