Merge pull request #47 from sethmachine/stormlib-wrapper

introduce wrapper around stormlib api

.github/workflows/pre-commit.yaml

@@ -4,7 +4,10 @@ on: [pull_request]
 
 jobs:
   pre-commit:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-14]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -18,7 +21,10 @@ jobs:
       - name: Run pre-commit
         run: pre-commit run --all-files
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-14]
     needs: pre-commit
     steps:
       - uses: actions/checkout@v2

src/richchk/model/mpq/stormlib/stormlib_archive_mode.py

@@ -0,0 +1,15 @@
+from enum import Enum
+
+from richchk.model.mpq.stormlib.stormlib_flag import StormLibFlag
+
+
+class StormLibArchiveMode(Enum):
+    STORMLIB_READ_ONLY = (StormLibFlag.STREAM_FLAG_READ_ONLY.value,)
+    STORMLIB_WRITE_ONLY = (StormLibFlag.STREAM_FLAG_WRITE_SHARE.value,)
+
+    def __init__(self, value: int):
+        self._value = value
+
+    @property
+    def value(self) -> int:
+        return self._value

src/richchk/model/mpq/stormlib/stormlib_flag.py

@@ -0,0 +1,174 @@
+from enum import Enum
+
+
+class StormLibFlag(Enum):
+    STORMLIB_VERSION = (0x0916,)
+    ID_MPQ = (0x1A51504D,)
+    ID_MPQ_USERDATA = (0x1B51504D,)
+    ID_MPK = (0x1A4B504D,)
+    ERROR_AVI_FILE = (10000,)
+    ERROR_UNKNOWN_FILE_KEY = (10001,)
+    ERROR_CHECKSUM_ERROR = (10002,)
+    ERROR_INTERNAL_FILE = (10003,)
+    ERROR_BASE_FILE_MISSING = (10004,)
+    ERROR_MARKED_FOR_DELETE = (10005,)
+    ERROR_FILE_INCOMPLETE = (10006,)
+    ERROR_UNKNOWN_FILE_NAMES = (10007,)
+    ERROR_CANT_FIND_PATCH_PREFIX = (10008,)
+    HASH_TABLE_SIZE_MIN = (0x00000004,)
+    HASH_TABLE_SIZE_DEFAULT = (0x00001000,)
+    HASH_TABLE_SIZE_MAX = (0x00080000,)
+    HASH_ENTRY_DELETED = (0xFFFFFFFE,)
+    HASH_ENTRY_FREE = (0xFFFFFFFF,)
+    HET_ENTRY_DELETED = (0x80,)
+    HET_ENTRY_FREE = (0x00,)
+    HASH_STATE_SIZE = (0x60,)
+    SFILE_OPEN_HARD_DISK_FILE = (2,)
+    SFILE_OPEN_CDROM_FILE = (3,)
+    SFILE_OPEN_FROM_MPQ = (0x00000000,)
+    SFILE_OPEN_CHECK_EXISTS = (0xFFFFFFFC,)
+    SFILE_OPEN_BASE_FILE = (0xFFFFFFFD,)
+    SFILE_OPEN_ANY_LOCALE = (0xFFFFFFFE,)
+    SFILE_OPEN_LOCAL_FILE = (0xFFFFFFFF,)
+    MPQ_FLAG_READ_ONLY = (0x00000001,)
+    MPQ_FLAG_CHANGED = (0x00000002,)
+    MPQ_FLAG_MALFORMED = (0x00000004,)
+    MPQ_FLAG_HASH_TABLE_CUT = (0x00000008,)
+    MPQ_FLAG_BLOCK_TABLE_CUT = (0x00000010,)
+    MPQ_FLAG_CHECK_SECTOR_CRC = (0x00000020,)
+    MPQ_FLAG_SAVING_TABLES = (0x00000040,)
+    MPQ_FLAG_PATCH = (0x00000080,)
+    MPQ_FLAG_WAR3_MAP = (0x00000100,)
+    MPQ_FLAG_LISTFILE_NONE = (0x00000200,)
+    MPQ_FLAG_LISTFILE_NEW = (0x00000400,)
+    MPQ_FLAG_ATTRIBUTES_NONE = (0x00000800,)
+    MPQ_FLAG_ATTRIBUTES_NEW = (0x00001000,)
+    MPQ_FLAG_SIGNATURE_NONE = (0x00002000,)
+    MPQ_FLAG_SIGNATURE_NEW = (0x00004000,)
+    MPQ_SUBTYPE_MPQ = (0x00000000,)
+    MPQ_SUBTYPE_SQP = (0x00000001,)
+    MPQ_SUBTYPE_MPK = (0x00000002,)
+    SFILE_INVALID_SIZE = (0xFFFFFFFF,)
+    SFILE_INVALID_POS = (0xFFFFFFFF,)
+    SFILE_INVALID_ATTRIBUTES = (0xFFFFFFFF,)
+    MPQ_FILE_IMPLODE = (0x00000100,)
+    MPQ_FILE_COMPRESS = (0x00000200,)
+    MPQ_FILE_ENCRYPTED = (0x00010000,)
+    MPQ_FILE_FIX_KEY = (0x00020000,)
+    MPQ_FILE_PATCH_FILE = (0x00100000,)
+    MPQ_FILE_SINGLE_UNIT = (0x01000000,)
+    MPQ_FILE_DELETE_MARKER = (0x02000000,)
+    MPQ_FILE_SECTOR_CRC = (0x04000000,)
+    MPQ_FILE_SIGNATURE = (0x10000000,)
+    MPQ_FILE_EXISTS = (0x80000000,)
+    MPQ_FILE_REPLACEEXISTING = (0x80000000,)
+    MPQ_FILE_COMPRESS_MASK = (0x0000FF00,)
+    MPQ_FILE_DEFAULT_INTERNAL = (0xFFFFFFFF,)
+    BLOCK_INDEX_MASK = (0x0FFFFFFF,)
+    MPQ_COMPRESSION_HUFFMANN = (0x01,)
+    MPQ_COMPRESSION_ZLIB = (0x02,)
+    MPQ_COMPRESSION_PKWARE = (0x08,)
+    MPQ_COMPRESSION_BZIP2 = (0x10,)
+    MPQ_COMPRESSION_SPARSE = (0x20,)
+    MPQ_COMPRESSION_ADPCM_MONO = (0x40,)
+    MPQ_COMPRESSION_ADPCM_STEREO = (0x80,)
+    MPQ_COMPRESSION_LZMA = (0x12,)
+    MPQ_COMPRESSION_NEXT_SAME = (0xFFFFFFFF,)
+    MPQ_WAVE_QUALITY_HIGH = (0,)
+    MPQ_WAVE_QUALITY_MEDIUM = (1,)
+    MPQ_WAVE_QUALITY_LOW = (2,)
+    HET_TABLE_SIGNATURE = (0x1A544548,)
+    BET_TABLE_SIGNATURE = (0x1A544542,)
+    MPQ_KEY_HASH_TABLE = (0xC3AF3770,)
+    MPQ_KEY_BLOCK_TABLE = (0xEC83B3A3,)
+    MPQ_FORMAT_VERSION_1 = (0,)
+    MPQ_FORMAT_VERSION_2 = (1,)
+    MPQ_FORMAT_VERSION_3 = (2,)
+    MPQ_FORMAT_VERSION_4 = (3,)
+    MPQ_ATTRIBUTE_CRC32 = (0x00000001,)
+    MPQ_ATTRIBUTE_FILETIME = (0x00000002,)
+    MPQ_ATTRIBUTE_MD5 = (0x00000004,)
+    MPQ_ATTRIBUTE_PATCH_BIT = (0x00000008,)
+    MPQ_ATTRIBUTE_ALL = (0x0000000F,)
+    MPQ_ATTRIBUTES_V1 = (100,)
+    BASE_PROVIDER_FILE = (0x00000000,)
+    BASE_PROVIDER_MAP = (0x00000001,)
+    BASE_PROVIDER_HTTP = (0x00000002,)
+    BASE_PROVIDER_MASK = (0x0000000F,)
+    STREAM_PROVIDER_FLAT = (0x00000000,)
+    STREAM_PROVIDER_PARTIAL = (0x00000010,)
+    STREAM_PROVIDER_MPQE = (0x00000020,)
+    STREAM_PROVIDER_BLOCK4 = (0x00000030,)
+    STREAM_PROVIDER_MASK = (0x000000F0,)
+    STREAM_FLAG_READ_ONLY = (0x00000100,)
+    STREAM_FLAG_WRITE_SHARE = (0x00000200,)
+    STREAM_FLAG_USE_BITMAP = (0x00000400,)
+    STREAM_OPTIONS_MASK = (0x0000FF00,)
+    STREAM_PROVIDERS_MASK = (0x000000FF,)
+    STREAM_FLAGS_MASK = (0x0000FFFF,)
+    MPQ_OPEN_NO_LISTFILE = (0x00010000,)
+    MPQ_OPEN_NO_ATTRIBUTES = (0x00020000,)
+    MPQ_OPEN_NO_HEADER_SEARCH = (0x00040000,)
+    MPQ_OPEN_FORCE_MPQ_V1 = (0x00080000,)
+    MPQ_OPEN_CHECK_SECTOR_CRC = (0x00100000,)
+    MPQ_OPEN_PATCH = (0x00200000,)
+    MPQ_OPEN_READ_ONLY = (0x00000100,)  # Stream is read only
+    MPQ_CREATE_LISTFILE = (0x00100000,)
+    MPQ_CREATE_ATTRIBUTES = (0x00200000,)
+    MPQ_CREATE_SIGNATURE = (0x00400000,)
+    MPQ_CREATE_ARCHIVE_V1 = (0x00000000,)
+    MPQ_CREATE_ARCHIVE_V2 = (0x01000000,)
+    MPQ_CREATE_ARCHIVE_V3 = (0x02000000,)
+    MPQ_CREATE_ARCHIVE_V4 = (0x03000000,)
+    MPQ_CREATE_ARCHIVE_VMASK = (0x0F000000,)
+    FLAGS_TO_FORMAT_SHIFT = (24,)
+    SFILE_VERIFY_SECTOR_CRC = (0x00000001,)
+    SFILE_VERIFY_FILE_CRC = (0x00000002,)
+    SFILE_VERIFY_FILE_MD5 = (0x00000004,)
+    SFILE_VERIFY_RAW_MD5 = (0x00000008,)
+    SFILE_VERIFY_ALL = (0x0000000F,)
+    VERIFY_OPEN_ERROR = (0x0001,)
+    VERIFY_READ_ERROR = (0x0002,)
+    VERIFY_FILE_HAS_SECTOR_CRC = (0x0004,)
+    VERIFY_FILE_SECTOR_CRC_ERROR = (0x0008,)
+    VERIFY_FILE_HAS_CHECKSUM = (0x0010,)
+    VERIFY_FILE_CHECKSUM_ERROR = (0x0020,)
+    VERIFY_FILE_HAS_MD5 = (0x0040,)
+    VERIFY_FILE_MD5_ERROR = (0x0080,)
+    VERIFY_FILE_HAS_RAW_MD5 = (0x0100,)
+    VERIFY_FILE_RAW_MD5_ERROR = (0x0200,)
+    SFILE_VERIFY_MPQ_HEADER = (0x0001,)
+    SFILE_VERIFY_HET_TABLE = (0x0002,)
+    SFILE_VERIFY_BET_TABLE = (0x0003,)
+    SFILE_VERIFY_HASH_TABLE = (0x0004,)
+    SFILE_VERIFY_BLOCK_TABLE = (0x0005,)
+    SFILE_VERIFY_HIBLOCK_TABLE = (0x0006,)
+    SFILE_VERIFY_FILE = (0x0007,)
+    SIGNATURE_TYPE_NONE = (0x0000,)
+    SIGNATURE_TYPE_WEAK = (0x0001,)
+    SIGNATURE_TYPE_STRONG = (0x0002,)
+    ERROR_NO_SIGNATURE = (0,)
+    ERROR_VERIFY_FAILED = (1,)
+    ERROR_WEAK_SIGNATURE_OK = (2,)
+    ERROR_WEAK_SIGNATURE_ERROR = (3,)
+    ERROR_STRONG_SIGNATURE_OK = (4,)
+    ERROR_STRONG_SIGNATURE_ERROR = (5,)
+    MD5_DIGEST_SIZE = (0x10,)
+    SHA1_DIGEST_SIZE = (0x14,)
+    LANG_NEUTRAL = (0x00,)
+    CCB_CHECKING_FILES = (1,)
+    CCB_CHECKING_HASH_TABLE = (2,)
+    CCB_COPYING_NON_MPQ_DATA = (3,)
+    CCB_COMPACTING_FILES = (4,)
+    CCB_CLOSING_ARCHIVE = (5,)
+    MPQ_HEADER_SIZE_V1 = (0x20,)
+    MPQ_HEADER_SIZE_V2 = (0x2C,)
+    MPQ_HEADER_SIZE_V3 = (0x44,)
+    MPQ_HEADER_SIZE_V4 = (0xD0,)
+
+    def __init__(self, value: int):
+        self._value = value
+
+    @property
+    def value(self) -> int:
+        return self._value

src/richchk/model/mpq/stormlib/stormlib_operation_result.py

@@ -0,0 +1,20 @@
+"""The result of using a StormLib function each time.
+
+A zero result code indicates an error.
+"""
+import ctypes
+import dataclasses
+
+
+@dataclasses.dataclass(frozen=True)
+class StormLibOperationResult:
+    _handle: ctypes.c_void_p
+    _result: int
+
+    @property
+    def handle(self) -> ctypes.c_void_p:
+        return self._handle
+
+    @property
+    def result(self) -> int:
+        return self._result

src/richchk/mpq/stormlib/stormlib_wrapper.py

@@ -0,0 +1,66 @@
+"""Wraps StormLib DLL.
+
+Avoid using this directly unless you know what you are doing.
+
+For future compatibility, users of this wrapper should provide a path to their own
+StormLib library compiled for their operating system and CPU architecture.
+"""
+import ctypes
+import os
+
+from ...model.mpq.stormlib.stormlib_archive_mode import StormLibArchiveMode
+from ...model.mpq.stormlib.stormlib_operation_result import StormLibOperationResult
+from ...model.mpq.stormlib.stormlib_reference import StormLibReference
+from ...util import logger
+
+
+class StormLibWrapper:
+    def __init__(self, stormlib_reference: StormLibReference):
+        self._log = logger.get_logger(StormLibWrapper.__name__)
+        self._stormlib = stormlib_reference
+
+    def open_archive(
+        self, mpq_file_path: str, archive_mode: StormLibArchiveMode
+    ) -> StormLibOperationResult:
+        """Opens the MPQ archive, returning a pointer to its handle.
+
+        The handle should be referenced in all future operations and the MPQ archive
+        properly closed once done.
+        """
+        assert os.path.exists(mpq_file_path)
+        operation = "SFileOpenArchive"
+        handle = ctypes.c_void_p()
+        func = getattr(self._stormlib.stormlib_dll, "SFileOpenArchive")
+        result: int = func(
+            mpq_file_path.encode("ascii"), 0, archive_mode.value, ctypes.byref(handle)
+        )
+        self._throw_if_archive_operation_fails(operation, result)
+        return StormLibOperationResult(_handle=handle, _result=result)
+
+    def close_archive(
+        self, stormlib_operation_result: StormLibOperationResult
+    ) -> StormLibOperationResult:
+        """Closes an opened MPQ archive.
+
+        :param stormlib_operation_result: the handle and result of a previous operation
+            that opened the archive.
+        :return:
+        """
+        operation = "SFileCloseArchive"
+        func = getattr(self._stormlib.stormlib_dll, operation)
+        result: int = func(stormlib_operation_result.handle)
+        self._throw_if_archive_operation_fails(operation, result)
+        return StormLibOperationResult(
+            _handle=stormlib_operation_result.handle, _result=result
+        )
+
+    def _throw_if_archive_operation_fails(
+        self, operation_name: str, result: int
+    ) -> None:
+        if result == 0:
+            msg = (
+                f"StormLib archive operation: <{operation_name}> failed due to a {result} result value.  "
+                f"StormLib reference: {self._stormlib}"
+            )
+            self._log.error(msg)
+            raise ValueError(msg)

test/chk_resources.py

@@ -66,3 +66,7 @@ def _extract_chk_section_name_from_file_path(file_path: str) -> str:
 MACOS_STORMLIB_M1 = Path(
     Path.joinpath(_RESOURCES_DIR_PATH, "stormlib/macos/libstorm.9.22.0.dylib")
 ).absolute()
+
+EXAMPLE_STARCRAFT_SCX_MAP = Path(
+    Path.joinpath(_RESOURCES_DIR_PATH, "stormlib/example-stacraft-map.scx")
+).absolute()

test/helpers/stormlib_helper.py

@@ -0,0 +1,7 @@
+import platform
+
+
+def run_test_if_mac_m1() -> bool:
+    return (
+        platform.system().lower() == "darwin" and platform.machine().lower() == "arm64"
+    )

test/mpq/stormlib/stormlib_wrapper_test.py

@@ -0,0 +1,75 @@
+import ctypes
+import shutil
+import tempfile
+
+import pytest
+
+from richchk.model.mpq.stormlib.stormlib_archive_mode import StormLibArchiveMode
+from richchk.model.mpq.stormlib.stormlib_file_path import StormLibFilePath
+from richchk.model.mpq.stormlib.stormlib_operation_result import StormLibOperationResult
+from richchk.mpq.stormlib.stormlib_loader import StormLibLoader
+from richchk.mpq.stormlib.stormlib_wrapper import StormLibWrapper
+
+from ...chk_resources import EXAMPLE_STARCRAFT_SCX_MAP, MACOS_STORMLIB_M1
+from ...helpers.stormlib_helper import run_test_if_mac_m1
+
+
+@pytest.fixture(scope="function")
+def stormlib_wrapper():
+    if run_test_if_mac_m1():
+        return StormLibWrapper(
+            StormLibLoader.load_stormlib(
+                path_to_stormlib=StormLibFilePath(
+                    _path_to_stormlib_dll=MACOS_STORMLIB_M1
+                )
+            )
+        )
+
+
+def _read_file_as_bytes(infile: str) -> bytes:
+    with open(infile, "rb") as f:
+        return f.read()
+
+
+def test_it_opens_and_closes_scx_map_unchanged_in_read_mode(stormlib_wrapper):
+    if stormlib_wrapper:
+        with tempfile.NamedTemporaryFile() as temp_scx_file:
+            shutil.copyfile(EXAMPLE_STARCRAFT_SCX_MAP, temp_scx_file.name)
+            map_bytes_before_open = _read_file_as_bytes(temp_scx_file.name)
+            open_result = stormlib_wrapper.open_archive(
+                temp_scx_file.name,
+                archive_mode=StormLibArchiveMode.STORMLIB_READ_ONLY,
+            )
+            stormlib_wrapper.close_archive(open_result)
+            assert map_bytes_before_open == _read_file_as_bytes(temp_scx_file.name)
+
+
+def test_it_opens_and_closes_scx_map_unchanged_in_write_mode(stormlib_wrapper):
+    if stormlib_wrapper:
+        with tempfile.NamedTemporaryFile() as temp_scx_file:
+            shutil.copyfile(EXAMPLE_STARCRAFT_SCX_MAP, temp_scx_file.name)
+            map_bytes_before_open = _read_file_as_bytes(temp_scx_file.name)
+            open_result = stormlib_wrapper.open_archive(
+                temp_scx_file.name,
+                archive_mode=StormLibArchiveMode.STORMLIB_WRITE_ONLY,
+            )
+            stormlib_wrapper.close_archive(open_result)
+            assert map_bytes_before_open == _read_file_as_bytes(temp_scx_file.name)
+
+
+def test_it_throws_if_input_file_is_not_mpq(stormlib_wrapper):
+    if stormlib_wrapper:
+        with tempfile.NamedTemporaryFile() as temp_scx_file:
+            with pytest.raises(ValueError):
+                stormlib_wrapper.open_archive(
+                    temp_scx_file.name,
+                    archive_mode=StormLibArchiveMode.STORMLIB_READ_ONLY,
+                )
+
+
+def test_it_throws_if_closing_an_archive_never_opened(stormlib_wrapper):
+    if stormlib_wrapper:
+        with pytest.raises(ValueError):
+            stormlib_wrapper.close_archive(
+                StormLibOperationResult(ctypes.c_void_p(), _result=1)
+            )