From c49200d457976fd74e17c676c1c3cd48252bff1e Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Tue, 19 Oct 2021 20:32:56 +0200 Subject: [PATCH 1/6] Added verattr and implemented verattr tag codegen. --- codegen.py | 25 ++++++++++++-- codegen/Union.py | 11 ++++-- codegen/Versions.py | 34 ++++++++++--------- codegen/expression.py | 2 +- generated/formats/bnk/bnk.xml | 2 ++ generated/formats/bnk/compound/BKHDSection.py | 1 - .../formats/fgm/compound/FgmInfoHeader.py | 12 +++---- generated/formats/fgm/fgm.xml | 21 ++++++++++-- .../formats/manis/compound/InfoHeader.py | 2 -- generated/formats/manis/manis.xml | 5 +-- .../compound/MaterialcollectionInfoHeader.py | 2 -- generated/formats/matcol/matcol.xml | 6 +++- .../formats/ms2/compound/Mdl2InfoHeader.py | 3 -- .../formats/ms2/compound/Ms2InfoHeader.py | 3 -- .../formats/ms2/compound/Ms2SizedStrData.py | 1 - generated/formats/ms2/ms2.xml | 7 +++- .../formats/ovl/compound/GenericHeader.py | 3 -- generated/formats/ovl/compound/MimeEntry.py | 1 - generated/formats/ovl/ovl.xml | 7 +++- generated/formats/tex/tex.xml | 5 ++- .../formats/voxelskirt/compound/Header.py | 3 -- generated/formats/voxelskirt/voxelskirt.xml | 6 +++- source/formats/bnk/bnk.xml | 2 ++ source/formats/fgm/fgm.xml | 21 ++++++++++-- source/formats/manis/manis.xml | 5 +-- source/formats/matcol/matcol.xml | 6 +++- source/formats/ms2/ms2.xml | 7 +++- source/formats/ovl/ovl.xml | 7 +++- source/formats/tex/tex.xml | 5 ++- source/formats/voxelskirt/voxelskirt.xml | 6 +++- 30 files changed, 152 insertions(+), 69 deletions(-) diff --git a/codegen.py b/codegen.py index 824b11b81..965877d72 100644 --- a/codegen.py +++ b/codegen.py @@ -37,6 +37,8 @@ def __init__(self, format_name): self.tokens = [] self.versions = [([], ("versions", "until", "since")), ] + # maps version attribute name to [access, type] + self.verattrs = {} # maps each type to its generated py file's relative path self.path_dict = {} # enum name -> storage name @@ -74,17 +76,24 @@ def generate_module_paths(self, root): self.path_dict["UintEnum"] = "base_enum" self.path_dict["Uint64Enum"] = "base_enum" + def register_tokens(self, root): + """Register tokens before anything else""" + for child in root: + if child.tag == "token": + self.read_token(child) + def load_xml(self, xml_file): """Loads an XML (can be filepath or open file) and does all parsing Goes over all children of the root node and calls the appropriate function depending on type of the child""" tree = ET.parse(xml_file) root = tree.getroot() self.generate_module_paths(root) + self.register_tokens(root) versions = Versions(self) for child in root: self.replace_tokens(child) - if child.tag not in ('version', 'module'): + if child.tag not in ('version', 'verattr', 'module'): self.apply_conventions(child) try: if child.tag in self.struct_types: @@ -99,8 +108,8 @@ def load_xml(self, xml_file): Module(self, child) elif child.tag == "version": versions.read(child) - elif child.tag == "token": - self.read_token(child) + elif child.tag == "verattr": + self.read_verattr(child) except Exception as err: logging.error(err) traceback.print_exc() @@ -114,6 +123,16 @@ def read_token(self, token): for sub_token in token], token.attrib["attrs"].split(" "))) + def read_verattr(self, verattr): + """Reads an xml and stores it in the verattrs dict""" + name = verattr.attrib['name'] + assert name not in self.verattrs, f"verattr {name} already defined!" + access = '.'.join(convention.name_attribute(comp) for comp in verattr.attrib["access"].split('.')) + attr_type = verattr.attrib.get("type") + if attr_type: + attr_type = convention.name_class(attr_type) + self.verattrs[name] = [access, attr_type] + @staticmethod def apply_convention(struct, func, params): for k in params: diff --git a/codegen/Union.py b/codegen/Union.py index 462359c71..32f41432f 100644 --- a/codegen/Union.py +++ b/codegen/Union.py @@ -274,7 +274,12 @@ def write_io(self, f, method_type, condition=""): else: f.write( f"{indent}{self.compound.parser.method_for_type(field_type, mode=method_type, attr=f'self.{field_name}', arg=arg, template=template)}") - # store version related stuff on self.context - if "version" in field_name: - f.write(f"{indent}{CONTEXT}.{field_name} = self.{field_name}") + if method_type == 'read': + # store version related stuff on self.context on read + for k, (access, dtype) in self.compound.parser.verattrs.items(): + attr_path = access.split('.') + if field_name == attr_path[0]: + if dtype is None or len(attr_path) > 1 or field_type == dtype: + f.write(f"{indent}{CONTEXT}.{field_name} = self.{field_name}") + break return condition diff --git a/codegen/Versions.py b/codegen/Versions.py index 8d9777bc6..9fc9d4259 100644 --- a/codegen/Versions.py +++ b/codegen/Versions.py @@ -2,6 +2,8 @@ from codegen.expression import Version +base_ver_attrs = ("id", "supported", "custom", "ext") + class Versions: """Creates and writes a version block""" @@ -25,34 +27,34 @@ def write(self, out_file): stream.write(f"def is_{self.format_id(version.attrib['id'])}(context):") conds_list = [] for k, v in version.attrib.items(): - if k != "id": - name = k.lower() + if k not in base_ver_attrs: + if k in self.parent.verattrs: + name = self.parent.verattrs[k][0] + else: + name = k.lower() val = v.strip() - if name == 'num': - val = str(Version(val)) if " " in val: - conds_list.append(f"context.{name} in ({val.replace(' ', ', ')})") + conds_list.append(f"context.{name} in ({', '.join([str(Version(nr)) for nr in val.split(' ')])})") else: - conds_list.append(f"context.{name} == {val}") + conds_list.append(f"context.{name} == {str(Version(val))}") stream.write("\n\tif " + " and ".join(conds_list) + ":") stream.write("\n\t\treturn True") stream.write("\n\n\n") stream.write(f"def set_{self.format_id(version.attrib['id'])}(context):") for k, v in version.attrib.items(): - if k != "id": - name = k.lower() + if k not in base_ver_attrs: + suffix = "" + if k in self.parent.verattrs: + name, attr_type = self.parent.verattrs[k] + if attr_type and self.parent.tag_dict[attr_type.lower()] == 'bitfield': + suffix = "._value" + else: + name = k.lower() val = v.strip() if " " in val: val = val.split(" ")[0] - # todo - this should instead be detected by field type - if name == "user_version": - suffix = "._value" - else: - suffix = "" - if name == "num": - val = str(Version(val)) - stream.write(f"\n\tcontext.{name}{suffix} = {val}") + stream.write(f"\n\tcontext.{name}{suffix} = {str(Version(val))}") stream.write("\n\n\n") # go through all the games, record them and map defaults to versions diff --git a/codegen/expression.py b/codegen/expression.py index 9d6a8fafb..ac74327a2 100644 --- a/codegen/expression.py +++ b/codegen/expression.py @@ -17,7 +17,7 @@ def __init__(self, expr_str: str): byte_number_strs = expr_str.split(".") self.value = sum(int(n) << shift for n, shift in zip(byte_number_strs, self.shifts)) else: - self.value = int(expr_str) + self.value = int(expr_str, 0) # print(self) def version_number(version_str): diff --git a/generated/formats/bnk/bnk.xml b/generated/formats/bnk/bnk.xml index 91ecd6630..703a33172 100644 --- a/generated/formats/bnk/bnk.xml +++ b/generated/formats/bnk/bnk.xml @@ -2,6 +2,8 @@ + + All Operators except for unary not (!), parentheses, and member of (\) NOTE: These can be ignored entirely by string substitution and dealt with directly. diff --git a/generated/formats/bnk/compound/BKHDSection.py b/generated/formats/bnk/compound/BKHDSection.py index bd2c05654..08bc49397 100644 --- a/generated/formats/bnk/compound/BKHDSection.py +++ b/generated/formats/bnk/compound/BKHDSection.py @@ -58,7 +58,6 @@ def write(self, stream): self.io_start = stream.tell() stream.write_uint(self.length) stream.write_uint(self.version) - self.context.version = self.version stream.write_uint(self.id_a) stream.write_uint(self.id_b) stream.write_uint(self.constant_a) diff --git a/generated/formats/fgm/compound/FgmInfoHeader.py b/generated/formats/fgm/compound/FgmInfoHeader.py index 2fded2db3..eeae3d44c 100644 --- a/generated/formats/fgm/compound/FgmInfoHeader.py +++ b/generated/formats/fgm/compound/FgmInfoHeader.py @@ -2,6 +2,7 @@ import typing from generated.array import Array from generated.context import ContextReference +from generated.formats.fgm.bitfield.VersionInfo import VersionInfo from generated.formats.fgm.compound.AttributeInfo import AttributeInfo from generated.formats.fgm.compound.FourFragFgm import FourFragFgm from generated.formats.fgm.compound.TextureInfo import TextureInfo @@ -40,7 +41,7 @@ def __init__(self, context, arg=None, template=None): # always = 1 self.seventh_byte = 1 - self.user_version = 0 + self.user_version = VersionInfo() # fragment count self.num_frags = 0 @@ -74,7 +75,7 @@ def set_defaults(self): self.version = 0 self.bitswap = 0 self.seventh_byte = 1 - self.user_version = 0 + self.user_version = VersionInfo() self.num_frags = 0 self.num_textures = 0 self.tex_info_size = 0 @@ -100,7 +101,7 @@ def read(self, stream): self.context.version = self.version self.bitswap = stream.read_byte() self.seventh_byte = stream.read_byte() - self.user_version = stream.read_uint() + self.user_version = stream.read_type(VersionInfo) self.context.user_version = self.user_version self.num_frags = stream.read_uint() self.num_textures = stream.read_uint() @@ -124,13 +125,10 @@ def write(self, stream): self.io_start = stream.tell() stream.write_bytes(self.magic) stream.write_byte(self.version_flag) - self.context.version_flag = self.version_flag stream.write_byte(self.version) - self.context.version = self.version stream.write_byte(self.bitswap) stream.write_byte(self.seventh_byte) - stream.write_uint(self.user_version) - self.context.user_version = self.user_version + stream.write_type(self.user_version) stream.write_uint(self.num_frags) stream.write_uint(self.num_textures) stream.write_uint(self.tex_info_size) diff --git a/generated/formats/fgm/fgm.xml b/generated/formats/fgm/fgm.xml index 7d3beab20..71b574f78 100644 --- a/generated/formats/fgm/fgm.xml +++ b/generated/formats/fgm/fgm.xml @@ -2,6 +2,10 @@ + + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +23,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. @@ -88,6 +92,17 @@ A standard 32-bit floating point number. + + Determines the format of the OVL file. + n.b. pos counts from the end! + + + + + + + + 4 bytes @@ -129,7 +144,7 @@ stores index into shader Stores rgba color - + part of fgm fragment, repeated per attribute byte offset to name in fgm buffer @@ -147,7 +162,7 @@ 0x12 = PC, 0x13 = JWE, PZ endianness?, usually zero always = 1 - + fragment count texture ref count byte count to check for quirks diff --git a/generated/formats/manis/compound/InfoHeader.py b/generated/formats/manis/compound/InfoHeader.py index 78f31afca..557e397d6 100644 --- a/generated/formats/manis/compound/InfoHeader.py +++ b/generated/formats/manis/compound/InfoHeader.py @@ -71,9 +71,7 @@ def write(self, stream): self.io_start = stream.tell() stream.write_bytes(self.magic) stream.write_uint(self.version) - self.context.version = self.version stream.write_uint(self.user_version) - self.context.user_version = self.user_version stream.write_uint(self.mani_count) stream.write_zstrings(self.names) stream.write_type(self.header) diff --git a/generated/formats/manis/manis.xml b/generated/formats/manis/manis.xml index 62e0e3de7..3eaa9544e 100644 --- a/generated/formats/manis/manis.xml +++ b/generated/formats/manis/manis.xml @@ -2,7 +2,8 @@ - + + Commonly used version expressions. @@ -12,7 +13,7 @@ PC - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/generated/formats/matcol/compound/MaterialcollectionInfoHeader.py b/generated/formats/matcol/compound/MaterialcollectionInfoHeader.py index e7b12d848..86cfe7a3d 100644 --- a/generated/formats/matcol/compound/MaterialcollectionInfoHeader.py +++ b/generated/formats/matcol/compound/MaterialcollectionInfoHeader.py @@ -84,9 +84,7 @@ def write(self, stream): self.io_start = stream.tell() stream.write_bytes(self.magic) stream.write_uint(self.version) - self.context.version = self.version stream.write_uint(self.user_version) - self.context.user_version = self.user_version stream.write_ubyte(self.has_texture_list) stream.write_type(self.root_0) stream.write_type(self.root_1) diff --git a/generated/formats/matcol/matcol.xml b/generated/formats/matcol/matcol.xml index 1d19a561b..3bc82d0d4 100644 --- a/generated/formats/matcol/matcol.xml +++ b/generated/formats/matcol/matcol.xml @@ -2,15 +2,19 @@ + + + Commonly used version expressions. PZ JWE - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. + diff --git a/generated/formats/ms2/compound/Mdl2InfoHeader.py b/generated/formats/ms2/compound/Mdl2InfoHeader.py index c12af904e..9f24678ae 100644 --- a/generated/formats/ms2/compound/Mdl2InfoHeader.py +++ b/generated/formats/ms2/compound/Mdl2InfoHeader.py @@ -119,13 +119,10 @@ def write(self, stream): self.io_start = stream.tell() stream.write_type(self.magic) stream.write_byte(self.version_flag) - self.context.version_flag = self.version_flag stream.write_byte(self.version) - self.context.version = self.version stream.write_byte(self.bitswap) stream.write_byte(self.seventh_byte) stream.write_uint(self.user_version) - self.context.user_version = self.user_version stream.write_uint(self.index) stream.write_uint(self.bone_info_index) stream.write_string(self.ms_2_name) diff --git a/generated/formats/ms2/compound/Ms2InfoHeader.py b/generated/formats/ms2/compound/Ms2InfoHeader.py index bcde6de8a..49c9d7764 100644 --- a/generated/formats/ms2/compound/Ms2InfoHeader.py +++ b/generated/formats/ms2/compound/Ms2InfoHeader.py @@ -82,13 +82,10 @@ def write(self, stream): self.io_start = stream.tell() stream.write_type(self.magic) stream.write_byte(self.version_flag) - self.context.version_flag = self.version_flag stream.write_byte(self.version) - self.context.version = self.version stream.write_byte(self.bitswap) stream.write_byte(self.seventh_byte) stream.write_uint(self.user_version) - self.context.user_version = self.user_version stream.write_uint(self.bone_names_size) stream.write_uint(self.bone_info_size) stream.write_type(self.general_info) diff --git a/generated/formats/ms2/compound/Ms2SizedStrData.py b/generated/formats/ms2/compound/Ms2SizedStrData.py index 5db4daffe..78da5aa07 100644 --- a/generated/formats/ms2/compound/Ms2SizedStrData.py +++ b/generated/formats/ms2/compound/Ms2SizedStrData.py @@ -62,7 +62,6 @@ def read(self, stream): def write(self, stream): self.io_start = stream.tell() stream.write_uint(self.ms_2_version) - self.context.ms_2_version = self.ms_2_version stream.write_ushort(self.vertex_buffer_count) stream.write_ushort(self.mdl_2_count) stream.write_ushort(self.name_count) diff --git a/generated/formats/ms2/ms2.xml b/generated/formats/ms2/ms2.xml index f36e48df4..9990ba328 100644 --- a/generated/formats/ms2/ms2.xml +++ b/generated/formats/ms2/ms2.xml @@ -2,6 +2,11 @@ + + + + + Old @@ -15,7 +20,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/generated/formats/ovl/compound/GenericHeader.py b/generated/formats/ovl/compound/GenericHeader.py index 4dd988ae1..0d0ab9b63 100644 --- a/generated/formats/ovl/compound/GenericHeader.py +++ b/generated/formats/ovl/compound/GenericHeader.py @@ -64,13 +64,10 @@ def write(self, stream): self.io_start = stream.tell() stream.write_type(self.fres) stream.write_byte(self.version_flag) - self.context.version_flag = self.version_flag stream.write_byte(self.version) - self.context.version = self.version stream.write_byte(self.bitswap) stream.write_byte(self.seventh_byte) stream.write_type(self.user_version) - self.context.user_version = self.user_version self.io_size = stream.tell() - self.io_start diff --git a/generated/formats/ovl/compound/MimeEntry.py b/generated/formats/ovl/compound/MimeEntry.py index 5ee55a909..f303e47f7 100644 --- a/generated/formats/ovl/compound/MimeEntry.py +++ b/generated/formats/ovl/compound/MimeEntry.py @@ -81,7 +81,6 @@ def write(self, stream): stream.write_uint(self.unknown) stream.write_uint(self.mime_hash) stream.write_uint(self.mime_version) - self.context.mime_version = self.mime_version stream.write_uint(self.file_index_offset) stream.write_uint(self.file_count) if self.context.version >= 20: diff --git a/generated/formats/ovl/ovl.xml b/generated/formats/ovl/ovl.xml index d932b75c8..7b6b1bd35 100644 --- a/generated/formats/ovl/ovl.xml +++ b/generated/formats/ovl/ovl.xml @@ -2,6 +2,11 @@ + + + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +24,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/generated/formats/tex/tex.xml b/generated/formats/tex/tex.xml index ac510424e..1c55cb524 100644 --- a/generated/formats/tex/tex.xml +++ b/generated/formats/tex/tex.xml @@ -2,6 +2,9 @@ + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +22,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/generated/formats/voxelskirt/compound/Header.py b/generated/formats/voxelskirt/compound/Header.py index 1c2df2de4..d7f6096c3 100644 --- a/generated/formats/voxelskirt/compound/Header.py +++ b/generated/formats/voxelskirt/compound/Header.py @@ -70,13 +70,10 @@ def write(self, stream): self.io_start = stream.tell() stream.write_type(self.magic) stream.write_byte(self.version_flag) - self.context.version_flag = self.version_flag stream.write_byte(self.version) - self.context.version = self.version stream.write_byte(self.bitswap) stream.write_byte(self.seventh_byte) stream.write_type(self.user_version) - self.context.user_version = self.user_version stream.write_type(self.info) self.io_size = stream.tell() - self.io_start diff --git a/generated/formats/voxelskirt/voxelskirt.xml b/generated/formats/voxelskirt/voxelskirt.xml index 3facceb63..7f0806238 100644 --- a/generated/formats/voxelskirt/voxelskirt.xml +++ b/generated/formats/voxelskirt/voxelskirt.xml @@ -2,6 +2,10 @@ + + + + Zoo Tycoon Ultimate Animal Collection Planet Coaster Planet Zoo @@ -15,7 +19,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/source/formats/bnk/bnk.xml b/source/formats/bnk/bnk.xml index 91ecd6630..703a33172 100644 --- a/source/formats/bnk/bnk.xml +++ b/source/formats/bnk/bnk.xml @@ -2,6 +2,8 @@ + + All Operators except for unary not (!), parentheses, and member of (\) NOTE: These can be ignored entirely by string substitution and dealt with directly. diff --git a/source/formats/fgm/fgm.xml b/source/formats/fgm/fgm.xml index 7d3beab20..71b574f78 100644 --- a/source/formats/fgm/fgm.xml +++ b/source/formats/fgm/fgm.xml @@ -2,6 +2,10 @@ + + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +23,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. @@ -88,6 +92,17 @@ A standard 32-bit floating point number. + + Determines the format of the OVL file. + n.b. pos counts from the end! + + + + + + + + 4 bytes @@ -129,7 +144,7 @@ stores index into shader Stores rgba color - + part of fgm fragment, repeated per attribute byte offset to name in fgm buffer @@ -147,7 +162,7 @@ 0x12 = PC, 0x13 = JWE, PZ endianness?, usually zero always = 1 - + fragment count texture ref count byte count to check for quirks diff --git a/source/formats/manis/manis.xml b/source/formats/manis/manis.xml index 62e0e3de7..3eaa9544e 100644 --- a/source/formats/manis/manis.xml +++ b/source/formats/manis/manis.xml @@ -2,7 +2,8 @@ - + + Commonly used version expressions. @@ -12,7 +13,7 @@ PC - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/source/formats/matcol/matcol.xml b/source/formats/matcol/matcol.xml index 1d19a561b..3bc82d0d4 100644 --- a/source/formats/matcol/matcol.xml +++ b/source/formats/matcol/matcol.xml @@ -2,15 +2,19 @@ + + + Commonly used version expressions. PZ JWE - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. + diff --git a/source/formats/ms2/ms2.xml b/source/formats/ms2/ms2.xml index f36e48df4..9990ba328 100644 --- a/source/formats/ms2/ms2.xml +++ b/source/formats/ms2/ms2.xml @@ -2,6 +2,11 @@ + + + + + Old @@ -15,7 +20,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/source/formats/ovl/ovl.xml b/source/formats/ovl/ovl.xml index d932b75c8..7b6b1bd35 100644 --- a/source/formats/ovl/ovl.xml +++ b/source/formats/ovl/ovl.xml @@ -2,6 +2,11 @@ + + + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +24,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/source/formats/tex/tex.xml b/source/formats/tex/tex.xml index ac510424e..1c55cb524 100644 --- a/source/formats/tex/tex.xml +++ b/source/formats/tex/tex.xml @@ -2,6 +2,9 @@ + + + Disneyland Adventure Zoo Tycoon Ultimate Animal Collection Planet Coaster @@ -19,7 +22,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. diff --git a/source/formats/voxelskirt/voxelskirt.xml b/source/formats/voxelskirt/voxelskirt.xml index 3facceb63..7f0806238 100644 --- a/source/formats/voxelskirt/voxelskirt.xml +++ b/source/formats/voxelskirt/voxelskirt.xml @@ -2,6 +2,10 @@ + + + + Zoo Tycoon Ultimate Animal Collection Planet Coaster Planet Zoo @@ -15,7 +19,7 @@ JWE, 25108 is JWE on switch - + Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. From 74f558acb8c9b7027908f3c9044947f6c435708e Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Sun, 24 Oct 2021 17:38:01 +0200 Subject: [PATCH 2/6] Removed unused calculation of field_debug_str in write_defaults and added comments as per PR review. --- codegen/Union.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codegen/Union.py b/codegen/Union.py index 32f41432f..8f5b5aa1c 100644 --- a/codegen/Union.py +++ b/codegen/Union.py @@ -235,7 +235,6 @@ def write_init(self, f): def write_defaults(self, f, condition=""): base_indent = "\n\t\t" for field in self.members: - field_debug_str = clean_comment_str(field.text, indent="\t\t") arg, template, arr1, arr2, conditionals, field_name, field_type, pad_mode = get_params(field) indent, condition = condition_indent(base_indent, conditionals, condition) @@ -275,11 +274,13 @@ def write_io(self, f, method_type, condition=""): f.write( f"{indent}{self.compound.parser.method_for_type(field_type, mode=method_type, attr=f'self.{field_name}', arg=arg, template=template)}") if method_type == 'read': - # store version related stuff on self.context on read + # store version related fields on self.context on read for k, (access, dtype) in self.compound.parser.verattrs.items(): + # check all version-related global variables registered with the verattr tag attr_path = access.split('.') if field_name == attr_path[0]: if dtype is None or len(attr_path) > 1 or field_type == dtype: + # the verattr type isn't known, we can't check it or it matches f.write(f"{indent}{CONTEXT}.{field_name} = self.{field_name}") break return condition From 9449a1bbbeefe9db143fc88fdfd608ca3404f0de Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Sun, 24 Oct 2021 18:02:22 +0200 Subject: [PATCH 3/6] Add comment to int(expr, 0) to explain what the 0 does as per PR review. --- codegen/expression.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codegen/expression.py b/codegen/expression.py index ac74327a2..d2cf372a6 100644 --- a/codegen/expression.py +++ b/codegen/expression.py @@ -17,6 +17,7 @@ def __init__(self, expr_str: str): byte_number_strs = expr_str.split(".") self.value = sum(int(n) << shift for n, shift in zip(byte_number_strs, self.shifts)) else: + # use int(x, 0) to evaluate x as an int literal, allowing for non-decimal (e.g. hex) values to be read self.value = int(expr_str, 0) # print(self) From 28374ec1e45f16a2d195e16d8c748853b8f7f93e Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:07:11 +0100 Subject: [PATCH 4/6] Fixed issue where module generation would fail if the path didn't exist already. --- codegen/Module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codegen/Module.py b/codegen/Module.py index 8eedfad48..1140b5b62 100644 --- a/codegen/Module.py +++ b/codegen/Module.py @@ -16,7 +16,11 @@ def read(self, element): self.custom = bool(eval(element.attrib.get("custom","true").replace("true","True").replace("false","False"),{})) def write(self, rel_path): - with open(os.path.join(os.getcwd(), "generated", rel_path, "__init__.py"), "w", encoding=self.parser.encoding) as file: + abs_path = os.path.join(os.getcwd(), "generated", rel_path, "__init__.py") + out_dir = os.path.dirname(abs_path) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + with open(abs_path, "w", encoding=self.parser.encoding) as file: file.write(self.comment_str) file.write(f'\n\n__priority__ = {repr(self.priority)}') file.write(f'\n__depends__ = {repr(self.depends)}') From 315c3647005a3a56cd6b78823e80d53fe5aeb30b Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Thu, 4 Nov 2021 12:35:24 +0100 Subject: [PATCH 5/6] Add evaluation of XInclude tag to codegen and use it to separate out basic ovl-specific functionality. --- codegen.py | 67 ++++++++-- generated/formats/bani/bani.xml | 62 +-------- generated/formats/bnk/bnk.xml | 58 +------- .../formats/fgm/compound/FgmInfoHeader.py | 55 +------- generated/formats/fgm/fgm.xml | 110 +--------------- generated/formats/fgm/versions.py | 8 +- generated/formats/manis/manis.xml | 76 ++--------- generated/formats/matcol/matcol.xml | 66 +--------- generated/formats/ms2/compound/FixedString.py | 32 ----- generated/formats/ms2/compound/JointData.py | 2 +- .../formats/ms2/compound/Mdl2InfoHeader.py | 55 +------- .../formats/ms2/compound/Ms2InfoHeader.py | 55 +------- .../formats/ms2/compound/ZStringBuffer.py | 73 ----------- generated/formats/ms2/ms2.xml | 114 +--------------- generated/formats/ms2/versions.py | 84 ++++++++++-- generated/formats/ovl/bitfield/VersionInfo.py | 25 ---- generated/formats/ovl/compound/FixedString.py | 32 ----- .../formats/ovl/compound/GenericHeader.py | 91 ------------- generated/formats/ovl/compound/Header.py | 6 +- generated/formats/ovl/compound/PadAlign.py | 14 +- .../formats/ovl/compound/ZStringBuffer.py | 73 ----------- generated/formats/ovl/ovl.xml | 124 +----------------- generated/formats/tex/bitfield/VersionInfo.py | 25 ---- generated/formats/tex/tex.xml | 99 +------------- generated/formats/tex/versions.py | 8 +- .../voxelskirt/bitfield/VersionInfo.py | 25 ---- .../voxelskirt/compound/FixedString.py | 32 ----- .../formats/voxelskirt/compound/Header.py | 58 +------- .../voxelskirt/compound/ZStringBuffer.py | 73 ----------- generated/formats/voxelskirt/versions.py | 36 ++++- generated/formats/voxelskirt/voxelskirt.xml | 115 +--------------- source/formats/bani/bani.xml | 62 +-------- .../formats/base}/__init__.py | 0 source/formats/base/base.xml | 61 +++++++++ source/formats/bnk/bnk.xml | 58 +------- source/formats/fgm/fgm.xml | 110 +--------------- source/formats/manis/manis.xml | 76 ++--------- source/formats/matcol/matcol.xml | 66 +--------- source/formats/ms2/compound/FixedString.py | 22 ---- source/formats/ms2/ms2.xml | 114 +--------------- source/formats/ovl/compound/FixedString.py | 22 ---- source/formats/ovl/compound/PadAlign.py | 6 + source/formats/ovl/compound/ZStringBuffer.py | 65 --------- source/formats/ovl/ovl.xml | 124 +----------------- .../formats/ovl_base}/__init__.py | 0 .../formats/ovl_base}/compound/FixedString.py | 0 .../compound/ZStringBuffer.py | 2 +- .../formats/ovl_base/compound}/__init__.py | 0 source/formats/ovl_base/ovl_base.xml | 81 ++++++++++++ source/formats/tex/compound/FixedString.py | 22 ---- source/formats/tex/tex.xml | 99 +------------- .../voxelskirt/compound/FixedString.py | 22 ---- .../voxelskirt/compound/ZStringBuffer.py | 65 --------- .../formats/voxelskirt/compound/__init__.py | 0 source/formats/voxelskirt/voxelskirt.xml | 115 +--------------- 55 files changed, 411 insertions(+), 2534 deletions(-) delete mode 100644 generated/formats/ms2/compound/FixedString.py delete mode 100644 generated/formats/ms2/compound/ZStringBuffer.py delete mode 100644 generated/formats/ovl/bitfield/VersionInfo.py delete mode 100644 generated/formats/ovl/compound/FixedString.py delete mode 100644 generated/formats/ovl/compound/GenericHeader.py delete mode 100644 generated/formats/ovl/compound/ZStringBuffer.py delete mode 100644 generated/formats/tex/bitfield/VersionInfo.py delete mode 100644 generated/formats/voxelskirt/bitfield/VersionInfo.py delete mode 100644 generated/formats/voxelskirt/compound/FixedString.py delete mode 100644 generated/formats/voxelskirt/compound/ZStringBuffer.py rename {generated/formats/ovl/bitfield => source/formats/base}/__init__.py (100%) create mode 100644 source/formats/base/base.xml delete mode 100644 source/formats/ms2/compound/FixedString.py delete mode 100644 source/formats/ovl/compound/FixedString.py delete mode 100644 source/formats/ovl/compound/ZStringBuffer.py rename {generated/formats/tex/bitfield => source/formats/ovl_base}/__init__.py (100%) rename {generated/formats/tex => source/formats/ovl_base}/compound/FixedString.py (100%) rename source/formats/{ms2 => ovl_base}/compound/ZStringBuffer.py (100%) rename {generated/formats/voxelskirt/bitfield => source/formats/ovl_base/compound}/__init__.py (100%) create mode 100644 source/formats/ovl_base/ovl_base.xml delete mode 100644 source/formats/tex/compound/FixedString.py delete mode 100644 source/formats/voxelskirt/compound/FixedString.py delete mode 100644 source/formats/voxelskirt/compound/ZStringBuffer.py delete mode 100644 source/formats/voxelskirt/compound/__init__.py diff --git a/codegen.py b/codegen.py index 965877d72..240d26d08 100644 --- a/codegen.py +++ b/codegen.py @@ -31,11 +31,10 @@ def __init__(self, format_name): self.encoding='utf-8' # elements for versions - self.version_string = None + self.versions = None # ordered (!) list of tuples ({tokens}, (target_attribs)) for each self.tokens = [] - self.versions = [([], ("versions", "until", "since")), ] # maps version attribute name to [access, type] self.verattrs = {} @@ -50,7 +49,7 @@ def generate_module_paths(self, root): """preprocessing - generate module paths for imports relative to the output dir""" for child in root: # only check stuff that has a name - ignore version tags - if child.tag not in ("version", "token"): + if child.tag.split('}')[-1] not in ("version", "token", "include"): base_segments = os.path.join("formats", self.format_name) if child.tag == "module": # for modules, set the path to base/module_name @@ -82,14 +81,24 @@ def register_tokens(self, root): if child.tag == "token": self.read_token(child) - def load_xml(self, xml_file): + def load_xml(self, xml_file, parsed_xmls=None): """Loads an XML (can be filepath or open file) and does all parsing Goes over all children of the root node and calls the appropriate function depending on type of the child""" + try: + # try for case where xml_file is a passed file object + xml_path = xml_file.name + except AttributeError: + # if attribute error, assume it was a file path + xml_path = xml_file tree = ET.parse(xml_file) root = tree.getroot() self.generate_module_paths(root) self.register_tokens(root) - versions = Versions(self) + self.versions = Versions(self) + + # dictionary of xml file: XmlParser + if parsed_xmls is None: + parsed_xmls = {} for child in root: self.replace_tokens(child) @@ -107,14 +116,17 @@ def load_xml(self, xml_file): elif child.tag == "module": Module(self, child) elif child.tag == "version": - versions.read(child) + self.versions.read(child) elif child.tag == "verattr": self.read_verattr(child) + elif child.tag.split('}')[-1] == "include": + self.read_xinclude(child, xml_path, parsed_xmls) except Exception as err: logging.error(err) traceback.print_exc() out_file = os.path.join(os.getcwd(), "generated", "formats", self.format_name, "versions.py") - versions.write(out_file) + self.versions.write(out_file) + parsed_xmls[xml_path] = self # the following constructs do not create classes def read_token(self, token): @@ -133,6 +145,21 @@ def read_verattr(self, verattr): attr_type = convention.name_class(attr_type) self.verattrs[name] = [access, attr_type] + def read_xinclude(self, xinclude, xml_path, parsed_xmls): + """Reads an xi:include element, and parses the linked xml if it doesn't exist yet in parsed xmls""" + # convert the linked relative path to an absolute one + new_path = os.path.realpath(os.path.join(os.path.dirname(xml_path), xinclude.attrib['href'])) + # check if the xml file was already parsed + if new_path not in parsed_xmls: + # if not, parse it now + format_name = os.path.splitext(os.path.basename(new_path))[0] + new_parser = XmlParser(format_name) + new_parser.load_xml(new_path, parsed_xmls) + else: + new_parser = parsed_xmls[new_path] + # append all pertinent information (file paths etc) to self for access + self.copy_xml_dicts(new_parser) + @staticmethod def apply_convention(struct, func, params): for k in params: @@ -196,7 +223,7 @@ def map_type(self, in_type): def replace_tokens(self, xml_struct): """Update xml_struct's (and all of its children's) attrib dict with content of tokens+versions list.""" # replace versions after tokens because tokens include versions - for tokens, target_attribs in self.tokens + self.versions: + for tokens, target_attribs in self.tokens: for target_attrib in target_attribs: if target_attrib in xml_struct.attrib: expr_str = xml_struct.attrib[target_attrib] @@ -214,6 +241,22 @@ def replace_tokens(self, xml_struct): for xml_child in xml_struct: self.replace_tokens(xml_child) + @staticmethod + def copy_dict_info(own_dict, other_dict): + """Add information from other dict if we didn't have it yet""" + for key in other_dict.keys(): + if key not in own_dict: + own_dict[key] = other_dict[key] + + def copy_xml_dicts(self, other_parser): + """Copy information necessary for linking and generation from another parser as if we'd read the file""" + [self.versions.read(version) for version in other_parser.versions.versions] + self.tokens.extend(other_parser.tokens) + self.copy_dict_info(self.verattrs, other_parser.verattrs) + self.copy_dict_info(self.path_dict, other_parser.path_dict) + self.copy_dict_info(self.storage_dict, other_parser.storage_dict) + self.copy_dict_info(self.tag_dict, other_parser.tag_dict) + def copy_src_to_generated(): """copies the files from the source folder to the generated folder""" @@ -239,14 +282,18 @@ def generate_classes(): cwd = os.getcwd() root_dir = os.path.join(cwd, "source\\formats") copy_src_to_generated() + parsed_xmls = {} for format_name in os.listdir(root_dir): dir_path = os.path.join(root_dir, format_name) if os.path.isdir(dir_path): xml_path = os.path.join(dir_path, format_name+".xml") if os.path.isfile(xml_path): - logging.info(f"Reading {format_name} format") + if os.path.realpath(xml_path) in parsed_xmls: + logging.info(f"Already read {format_name}, skipping") + else: + logging.info(f"Reading {format_name} format") xmlp = XmlParser(format_name) - xmlp.load_xml(xml_path) + xmlp.load_xml(xml_path, parsed_xmls) create_inits() diff --git a/generated/formats/bani/bani.xml b/generated/formats/bani/bani.xml index 9db8d25d9..b4865b66d 100644 --- a/generated/formats/bani/bani.xml +++ b/generated/formats/bani/bani.xml @@ -2,65 +2,7 @@ - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + A string of given length. @@ -101,7 +43,7 @@ Member 3,3 (bottom left) - + A 4x4 transformation matrix. The (1,1) element. The (2,1) element. diff --git a/generated/formats/bnk/bnk.xml b/generated/formats/bnk/bnk.xml index 703a33172..be3a63e83 100644 --- a/generated/formats/bnk/bnk.xml +++ b/generated/formats/bnk/bnk.xml @@ -2,30 +2,9 @@ - + - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - + @@ -33,42 +12,9 @@ Null terminated string. - - An unsigned 8-bit integer. - - - A signed 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - A signed 64-bit integer. - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - second Section of a soundback aux diff --git a/generated/formats/fgm/compound/FgmInfoHeader.py b/generated/formats/fgm/compound/FgmInfoHeader.py index eeae3d44c..4ab1b7c60 100644 --- a/generated/formats/fgm/compound/FgmInfoHeader.py +++ b/generated/formats/fgm/compound/FgmInfoHeader.py @@ -1,15 +1,14 @@ import numpy import typing from generated.array import Array -from generated.context import ContextReference -from generated.formats.fgm.bitfield.VersionInfo import VersionInfo from generated.formats.fgm.compound.AttributeInfo import AttributeInfo from generated.formats.fgm.compound.FourFragFgm import FourFragFgm from generated.formats.fgm.compound.TextureInfo import TextureInfo from generated.formats.fgm.compound.TwoFragFgmExtra import TwoFragFgmExtra +from generated.formats.ovl_base.compound.GenericHeader import GenericHeader -class FgmInfoHeader: +class FgmInfoHeader(GenericHeader): """ Custom header struct @@ -17,32 +16,14 @@ class FgmInfoHeader: This reads a whole custom FGM file """ - context = ContextReference() - def __init__(self, context, arg=None, template=None): self.name = '' - self._context = context + super().__init__(context, arg, template) self.arg = arg self.template = template self.io_size = 0 self.io_start = 0 - # 'FGM ' - self.magic = numpy.zeros((4), dtype='byte') - - # if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - self.version_flag = 0 - - # 0x12 = PC, 0x13 = JWE, PZ - self.version = 0 - - # endianness?, usually zero - self.bitswap = 0 - - # always = 1 - self.seventh_byte = 1 - self.user_version = VersionInfo() - # fragment count self.num_frags = 0 @@ -70,12 +51,6 @@ def __init__(self, context, arg=None, template=None): self.set_defaults() def set_defaults(self): - self.magic = numpy.zeros((4), dtype='byte') - self.version_flag = 0 - self.version = 0 - self.bitswap = 0 - self.seventh_byte = 1 - self.user_version = VersionInfo() self.num_frags = 0 self.num_textures = 0 self.tex_info_size = 0 @@ -94,15 +69,7 @@ def set_defaults(self): def read(self, stream): self.io_start = stream.tell() - self.magic = stream.read_bytes((4)) - self.version_flag = stream.read_byte() - self.context.version_flag = self.version_flag - self.version = stream.read_byte() - self.context.version = self.version - self.bitswap = stream.read_byte() - self.seventh_byte = stream.read_byte() - self.user_version = stream.read_type(VersionInfo) - self.context.user_version = self.user_version + super().read(stream) self.num_frags = stream.read_uint() self.num_textures = stream.read_uint() self.tex_info_size = stream.read_uint() @@ -123,12 +90,7 @@ def read(self, stream): def write(self, stream): self.io_start = stream.tell() - stream.write_bytes(self.magic) - stream.write_byte(self.version_flag) - stream.write_byte(self.version) - stream.write_byte(self.bitswap) - stream.write_byte(self.seventh_byte) - stream.write_type(self.user_version) + super().write(stream) stream.write_uint(self.num_frags) stream.write_uint(self.num_textures) stream.write_uint(self.tex_info_size) @@ -152,12 +114,7 @@ def get_info_str(self): def get_fields_str(self): s = '' - s += f'\n * magic = {self.magic.__repr__()}' - s += f'\n * version_flag = {self.version_flag.__repr__()}' - s += f'\n * version = {self.version.__repr__()}' - s += f'\n * bitswap = {self.bitswap.__repr__()}' - s += f'\n * seventh_byte = {self.seventh_byte.__repr__()}' - s += f'\n * user_version = {self.user_version.__repr__()}' + s += super().get_fields_str() s += f'\n * num_frags = {self.num_frags.__repr__()}' s += f'\n * num_textures = {self.num_textures.__repr__()}' s += f'\n * tex_info_size = {self.tex_info_size.__repr__()}' diff --git a/generated/formats/fgm/fgm.xml b/generated/formats/fgm/fgm.xml index 71b574f78..c37e53c1f 100644 --- a/generated/formats/fgm/fgm.xml +++ b/generated/formats/fgm/fgm.xml @@ -2,106 +2,7 @@ - - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Planet Zoo 1.6 - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - Zero terminated bytes. - - - - A signed 16-bit integer. - - - - A standard 32-bit floating point number. - - - - Determines the format of the OVL file. - n.b. pos counts from the end! - - - - - - - + 4 bytes @@ -110,7 +11,6 @@ - Sized str entry of 16 bytes @@ -153,16 +53,10 @@ - + Custom header struct This reads a whole custom FGM file - 'FGM ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - fragment count texture ref count byte count to check for quirks diff --git a/generated/formats/fgm/versions.py b/generated/formats/fgm/versions.py index 8db126fc6..6b08a7244 100644 --- a/generated/formats/fgm/versions.py +++ b/generated/formats/fgm/versions.py @@ -29,7 +29,7 @@ def set_pc(context): def is_pz(context): - if context.version in (19, 20) and context.user_version in (8340, 8724): + if context.version == 19 and context.user_version in (8340, 8724): return True @@ -58,7 +58,7 @@ def set_jwe(context): context.user_version._value = 24724 -games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO', 'Planet Zoo'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) +games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6+'), ('PLANET_ZOO_PRE_1_6', 'Planet Zoo pre-1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) def get_game(context): @@ -69,7 +69,7 @@ def get_game(context): if is_pc(context): return [games.PLANET_COASTER] if is_pz(context): - return [games.PLANET_ZOO] + return [games.PLANET_ZOO_PRE_1_6] if is_pz16(context): return [games.PLANET_ZOO_1_6] if is_jwe(context): @@ -86,7 +86,7 @@ def set_game(context, game): return set_ztuac(context) if game in {games.PLANET_COASTER}: return set_pc(context) - if game in {games.PLANET_ZOO}: + if game in {games.PLANET_ZOO_PRE_1_6}: return set_pz(context) if game in {games.PLANET_ZOO_1_6}: return set_pz16(context) diff --git a/generated/formats/manis/manis.xml b/generated/formats/manis/manis.xml index 3eaa9544e..6d31ee927 100644 --- a/generated/formats/manis/manis.xml +++ b/generated/formats/manis/manis.xml @@ -2,9 +2,6 @@ - - - Commonly used version expressions. PZ @@ -20,65 +17,10 @@ - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - + - - A standard 32-bit floating point number. - + + A string of given length. @@ -86,13 +28,13 @@ The string itself. - - Grabs 00 bytes only - + + Grabs 00 bytes only + - - Grabs 00 bytes only - + + Grabs 00 bytes only + A string of given length. diff --git a/generated/formats/matcol/matcol.xml b/generated/formats/matcol/matcol.xml index 3bc82d0d4..35344d8ac 100644 --- a/generated/formats/matcol/matcol.xml +++ b/generated/formats/matcol/matcol.xml @@ -2,9 +2,6 @@ - - - Commonly used version expressions. PZ @@ -14,69 +11,14 @@ Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - + - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - A signed 32-bit integer. - + - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + + Null terminated string. diff --git a/generated/formats/ms2/compound/FixedString.py b/generated/formats/ms2/compound/FixedString.py deleted file mode 100644 index b35efc941..000000000 --- a/generated/formats/ms2/compound/FixedString.py +++ /dev/null @@ -1,32 +0,0 @@ -from generated.context import ContextReference - - -class FixedString: - - """ - Holds a string of a fixed size, given as an argument. - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - # arg is byte count - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - - diff --git a/generated/formats/ms2/compound/JointData.py b/generated/formats/ms2/compound/JointData.py index a014b408f..34d48f099 100644 --- a/generated/formats/ms2/compound/JointData.py +++ b/generated/formats/ms2/compound/JointData.py @@ -11,7 +11,7 @@ from generated.formats.ms2.compound.ListShort import ListShort from generated.formats.ms2.compound.PcFFCounter import PcFFCounter from generated.formats.ms2.compound.SmartPadding import SmartPadding -from generated.formats.ms2.compound.ZStringBuffer import ZStringBuffer +from generated.formats.ovl_base.compound.ZStringBuffer import ZStringBuffer class JointData: diff --git a/generated/formats/ms2/compound/Mdl2InfoHeader.py b/generated/formats/ms2/compound/Mdl2InfoHeader.py index 9f24678ae..baa37d46b 100644 --- a/generated/formats/ms2/compound/Mdl2InfoHeader.py +++ b/generated/formats/ms2/compound/Mdl2InfoHeader.py @@ -1,16 +1,15 @@ import numpy import typing from generated.array import Array -from generated.context import ContextReference from generated.formats.ms2.compound.CoreModelInfo import CoreModelInfo -from generated.formats.ms2.compound.FixedString import FixedString from generated.formats.ms2.compound.LodInfo import LodInfo from generated.formats.ms2.compound.MaterialName import MaterialName from generated.formats.ms2.compound.MeshLink import MeshLink from generated.formats.ms2.compound.ModelData import ModelData +from generated.formats.ovl_base.compound.GenericHeader import GenericHeader -class Mdl2InfoHeader: +class Mdl2InfoHeader(GenericHeader): """ Custom header struct @@ -18,32 +17,14 @@ class Mdl2InfoHeader: This reads a whole custom mdl2 file """ - context = ContextReference() - def __init__(self, context, arg=None, template=None): self.name = '' - self._context = context + super().__init__(context, arg, template) self.arg = arg self.template = template self.io_size = 0 self.io_start = 0 - # 'MS2 ' - self.magic = FixedString(self.context, 4, None) - - # if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - self.version_flag = 0 - - # 0x12 = PC, 0x13 = JWE, PZ - self.version = 0 - - # endianness?, usually zero - self.bitswap = 0 - - # always = 1 - self.seventh_byte = 1 - self.user_version = 0 - # index of this model inside the ms2 self.index = 0 @@ -70,12 +51,6 @@ def __init__(self, context, arg=None, template=None): self.set_defaults() def set_defaults(self): - self.magic = FixedString(self.context, 4, None) - self.version_flag = 0 - self.version = 0 - self.bitswap = 0 - self.seventh_byte = 1 - self.user_version = 0 self.index = 0 self.bone_info_index = 0 self.ms_2_name = 0 @@ -92,15 +67,7 @@ def set_defaults(self): def read(self, stream): self.io_start = stream.tell() - self.magic = stream.read_type(FixedString, (self.context, 4, None)) - self.version_flag = stream.read_byte() - self.context.version_flag = self.version_flag - self.version = stream.read_byte() - self.context.version = self.version - self.bitswap = stream.read_byte() - self.seventh_byte = stream.read_byte() - self.user_version = stream.read_uint() - self.context.user_version = self.user_version + super().read(stream) self.index = stream.read_uint() self.bone_info_index = stream.read_uint() self.ms_2_name = stream.read_string() @@ -117,12 +84,7 @@ def read(self, stream): def write(self, stream): self.io_start = stream.tell() - stream.write_type(self.magic) - stream.write_byte(self.version_flag) - stream.write_byte(self.version) - stream.write_byte(self.bitswap) - stream.write_byte(self.seventh_byte) - stream.write_uint(self.user_version) + super().write(stream) stream.write_uint(self.index) stream.write_uint(self.bone_info_index) stream.write_string(self.ms_2_name) @@ -142,12 +104,7 @@ def get_info_str(self): def get_fields_str(self): s = '' - s += f'\n * magic = {self.magic.__repr__()}' - s += f'\n * version_flag = {self.version_flag.__repr__()}' - s += f'\n * version = {self.version.__repr__()}' - s += f'\n * bitswap = {self.bitswap.__repr__()}' - s += f'\n * seventh_byte = {self.seventh_byte.__repr__()}' - s += f'\n * user_version = {self.user_version.__repr__()}' + s += super().get_fields_str() s += f'\n * index = {self.index.__repr__()}' s += f'\n * bone_info_index = {self.bone_info_index.__repr__()}' s += f'\n * ms_2_name = {self.ms_2_name.__repr__()}' diff --git a/generated/formats/ms2/compound/Ms2InfoHeader.py b/generated/formats/ms2/compound/Ms2InfoHeader.py index 49c9d7764..16c4b2f12 100644 --- a/generated/formats/ms2/compound/Ms2InfoHeader.py +++ b/generated/formats/ms2/compound/Ms2InfoHeader.py @@ -1,42 +1,23 @@ -from generated.context import ContextReference -from generated.formats.ms2.compound.FixedString import FixedString from generated.formats.ms2.compound.Ms2Buffer0 import Ms2Buffer0 from generated.formats.ms2.compound.Ms2BufferInfo import Ms2BufferInfo from generated.formats.ms2.compound.Ms2SizedStrData import Ms2SizedStrData +from generated.formats.ovl_base.compound.GenericHeader import GenericHeader -class Ms2InfoHeader: +class Ms2InfoHeader(GenericHeader): """ Custom header struct includes fragments but none of the 3 data buffers """ - context = ContextReference() - def __init__(self, context, arg=None, template=None): self.name = '' - self._context = context + super().__init__(context, arg, template) self.arg = arg self.template = template self.io_size = 0 self.io_start = 0 - - # 'MS2 ' - self.magic = FixedString(self.context, 4, None) - - # if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - self.version_flag = 0 - - # 0x12 = PC, 0x13 = JWE, PZ - self.version = 0 - - # endianness?, usually zero - self.bitswap = 0 - - # always = 1 - self.seventh_byte = 1 - self.user_version = 0 self.bone_names_size = 0 self.bone_info_size = 0 self.general_info = Ms2SizedStrData(self.context, None, None) @@ -45,12 +26,6 @@ def __init__(self, context, arg=None, template=None): self.set_defaults() def set_defaults(self): - self.magic = FixedString(self.context, 4, None) - self.version_flag = 0 - self.version = 0 - self.bitswap = 0 - self.seventh_byte = 1 - self.user_version = 0 self.bone_names_size = 0 self.bone_info_size = 0 self.general_info = Ms2SizedStrData(self.context, None, None) @@ -60,15 +35,7 @@ def set_defaults(self): def read(self, stream): self.io_start = stream.tell() - self.magic = stream.read_type(FixedString, (self.context, 4, None)) - self.version_flag = stream.read_byte() - self.context.version_flag = self.version_flag - self.version = stream.read_byte() - self.context.version = self.version - self.bitswap = stream.read_byte() - self.seventh_byte = stream.read_byte() - self.user_version = stream.read_uint() - self.context.user_version = self.user_version + super().read(stream) self.bone_names_size = stream.read_uint() self.bone_info_size = stream.read_uint() self.general_info = stream.read_type(Ms2SizedStrData, (self.context, None, None)) @@ -80,12 +47,7 @@ def read(self, stream): def write(self, stream): self.io_start = stream.tell() - stream.write_type(self.magic) - stream.write_byte(self.version_flag) - stream.write_byte(self.version) - stream.write_byte(self.bitswap) - stream.write_byte(self.seventh_byte) - stream.write_uint(self.user_version) + super().write(stream) stream.write_uint(self.bone_names_size) stream.write_uint(self.bone_info_size) stream.write_type(self.general_info) @@ -100,12 +62,7 @@ def get_info_str(self): def get_fields_str(self): s = '' - s += f'\n * magic = {self.magic.__repr__()}' - s += f'\n * version_flag = {self.version_flag.__repr__()}' - s += f'\n * version = {self.version.__repr__()}' - s += f'\n * bitswap = {self.bitswap.__repr__()}' - s += f'\n * seventh_byte = {self.seventh_byte.__repr__()}' - s += f'\n * user_version = {self.user_version.__repr__()}' + s += super().get_fields_str() s += f'\n * bone_names_size = {self.bone_names_size.__repr__()}' s += f'\n * bone_info_size = {self.bone_info_size.__repr__()}' s += f'\n * general_info = {self.general_info.__repr__()}' diff --git a/generated/formats/ms2/compound/ZStringBuffer.py b/generated/formats/ms2/compound/ZStringBuffer.py deleted file mode 100644 index a71885b33..000000000 --- a/generated/formats/ms2/compound/ZStringBuffer.py +++ /dev/null @@ -1,73 +0,0 @@ - -import logging -from generated.io import BinaryStream -from modules.formats.shared import get_padding - -ZERO = b"\x00" - - -from generated.context import ContextReference - - -class ZStringBuffer: - - """ - Holds a buffer of zero-terminated strings - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - self.strings = [] - - def read(self, stream): - self.data = stream.read(self.arg) - self.strings = self.data.split(ZERO) - - def write(self, stream): - stream.write(self.data) - - def get_str_at(self, pos): - end = self.data.find(ZERO, pos) - return self.data[pos:end].decode() - - def update_with(self, list_of_arrays): - """Updates this name buffer with a list of (array, attrib) whose elements have - offset: bytes relative to the start of the name block - [attrib]: string""" - logging.debug("Updating name buffer") - self.strings = [] - offset_dic = {} - with BinaryStream() as stream: - # for name in self.names: - for array, attrib in list_of_arrays: - for item in sorted(array, key=lambda i: getattr(i, attrib)): - name = getattr(item, attrib) - if name in offset_dic: - # known string, just get offset - address = offset_dic[name] - else: - # new string, store offset and write zstring - address = stream.tell() - self.strings.append(name) - offset_dic[name] = address - stream.write_zstring(name) - # store offset on item - item.offset = address - # get the actual result buffer - buffer_bytes = stream.getvalue() - - self.data = buffer_bytes + get_padding(len(buffer_bytes), alignment=8) - - def __repr__(self): - return str(self.strings) - diff --git a/generated/formats/ms2/ms2.xml b/generated/formats/ms2/ms2.xml index 9990ba328..8241e41dc 100644 --- a/generated/formats/ms2/ms2.xml +++ b/generated/formats/ms2/ms2.xml @@ -2,95 +2,13 @@ - - - - - - Old - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC PC, ZTUAC - PZ - PZ - JWE, 25108 is JWE on switch - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - + - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + @@ -126,22 +44,10 @@ - - Holds a string of a fixed size, given as an argument. - - - - Null terminated string. - - Grabs 00 bytes only - - Holds a buffer of zero-terminated strings - - A string of given length. @@ -314,16 +220,10 @@ seems to be zeros - + Custom header struct This reads a whole custom mdl2 file - 'MS2 ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - index of this model inside the ms2 used to find bone info name of ms2 @@ -347,15 +247,9 @@ index into models array - + Custom header struct includes fragments but none of the 3 data buffers - 'MS2 ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - diff --git a/generated/formats/ms2/versions.py b/generated/formats/ms2/versions.py index f7a5fafa8..6b08a7244 100644 --- a/generated/formats/ms2/versions.py +++ b/generated/formats/ms2/versions.py @@ -1,28 +1,96 @@ from enum import Enum -def is_old(context): - if context.version in (17, 18): +def is_dla(context): + if context.version == 15: return True -def set_old(context): +def set_dla(context): + context.version = 15 + + +def is_ztuac(context): + if context.version == 17: + return True + + +def set_ztuac(context): context.version = 17 -games = Enum('Games',[('OLD', 'Old'), ('UNKNOWN_GAME', 'Unknown Game')]) +def is_pc(context): + if context.version == 18: + return True + + +def set_pc(context): + context.version = 18 + + +def is_pz(context): + if context.version == 19 and context.user_version in (8340, 8724): + return True + + +def set_pz(context): + context.version = 19 + context.user_version._value = 8340 + + +def is_pz16(context): + if context.version == 20 and context.user_version in (8340, 8724): + return True + + +def set_pz16(context): + context.version = 20 + context.user_version._value = 8340 + + +def is_jwe(context): + if context.version == 19 and context.user_version in (24724, 25108): + return True + + +def set_jwe(context): + context.version = 19 + context.user_version._value = 24724 + + +games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6+'), ('PLANET_ZOO_PRE_1_6', 'Planet Zoo pre-1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) def get_game(context): - if is_old(context): - return [games.OLD] + if is_dla(context): + return [games.DISNEYLAND_ADVENTURE] + if is_ztuac(context): + return [games.ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION] + if is_pc(context): + return [games.PLANET_COASTER] + if is_pz(context): + return [games.PLANET_ZOO_PRE_1_6] + if is_pz16(context): + return [games.PLANET_ZOO_1_6] + if is_jwe(context): + return [games.JURASSIC_WORLD_EVOLUTION] return [games.UNKOWN_GAME] def set_game(context, game): if isinstance(game, str): game = games(game) - if game in {games.OLD}: - return set_old(context) + if game in {games.DISNEYLAND_ADVENTURE}: + return set_dla(context) + if game in {games.ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION}: + return set_ztuac(context) + if game in {games.PLANET_COASTER}: + return set_pc(context) + if game in {games.PLANET_ZOO_PRE_1_6}: + return set_pz(context) + if game in {games.PLANET_ZOO_1_6}: + return set_pz16(context) + if game in {games.JURASSIC_WORLD_EVOLUTION}: + return set_jwe(context) diff --git a/generated/formats/ovl/bitfield/VersionInfo.py b/generated/formats/ovl/bitfield/VersionInfo.py deleted file mode 100644 index 77bf28824..000000000 --- a/generated/formats/ovl/bitfield/VersionInfo.py +++ /dev/null @@ -1,25 +0,0 @@ -from generated.bitfield import BasicBitfield -from generated.bitfield import BitfieldMember - - -class VersionInfo(BasicBitfield): - - """ - Determines the format of the OVL file. - n.b. pos counts from the end! - """ - unk_1 = BitfieldMember(pos=2, mask=0x4, return_type=bool) - unk_2 = BitfieldMember(pos=4, mask=0x10, return_type=bool) - use_zlib = BitfieldMember(pos=7, mask=0x80, return_type=bool) - use_oodle = BitfieldMember(pos=9, mask=0x200, return_type=bool) - unk_3 = BitfieldMember(pos=13, mask=0x2000, return_type=bool) - is_jwe = BitfieldMember(pos=14, mask=0x4000, return_type=bool) - - def set_defaults(self): - pass - - def read(self, stream): - self._value = stream.read_uint() - - def write(self, stream): - stream.write_uint(self._value) diff --git a/generated/formats/ovl/compound/FixedString.py b/generated/formats/ovl/compound/FixedString.py deleted file mode 100644 index 45bb5b2a5..000000000 --- a/generated/formats/ovl/compound/FixedString.py +++ /dev/null @@ -1,32 +0,0 @@ -from generated.context import ContextReference - - -class FixedString: - - """ - Holds a string of a fixed size, given as an argument. - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - - diff --git a/generated/formats/ovl/compound/GenericHeader.py b/generated/formats/ovl/compound/GenericHeader.py deleted file mode 100644 index 0d0ab9b63..000000000 --- a/generated/formats/ovl/compound/GenericHeader.py +++ /dev/null @@ -1,91 +0,0 @@ -from generated.context import ContextReference -from generated.formats.ovl.bitfield.VersionInfo import VersionInfo -from generated.formats.ovl.compound.FixedString import FixedString - - -class GenericHeader: - - """ - Found at the beginning of every OVL file - """ - - context = ContextReference() - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.io_size = 0 - self.io_start = 0 - - # 'FRES' - self.fres = FixedString(self.context, 4, None) - - # if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC, 0x48 for JWE Switch, may be platform - self.version_flag = 0 - - # 0x12 = PC, 0x13 = JWE, PZ - self.version = 0 - - # endianness?, usually zero - self.bitswap = 0 - - # always = 1 - self.seventh_byte = 1 - - # determines compression format (none, zlib or oodle) and apparently type of data (additional fields) - self.user_version = VersionInfo() - self.set_defaults() - - def set_defaults(self): - self.fres = FixedString(self.context, 4, None) - self.version_flag = 0 - self.version = 0 - self.bitswap = 0 - self.seventh_byte = 1 - self.user_version = VersionInfo() - - def read(self, stream): - self.io_start = stream.tell() - self.fres = stream.read_type(FixedString, (self.context, 4, None)) - self.version_flag = stream.read_byte() - self.context.version_flag = self.version_flag - self.version = stream.read_byte() - self.context.version = self.version - self.bitswap = stream.read_byte() - self.seventh_byte = stream.read_byte() - self.user_version = stream.read_type(VersionInfo) - self.context.user_version = self.user_version - - self.io_size = stream.tell() - self.io_start - - def write(self, stream): - self.io_start = stream.tell() - stream.write_type(self.fres) - stream.write_byte(self.version_flag) - stream.write_byte(self.version) - stream.write_byte(self.bitswap) - stream.write_byte(self.seventh_byte) - stream.write_type(self.user_version) - - self.io_size = stream.tell() - self.io_start - - def get_info_str(self): - return f'GenericHeader [Size: {self.io_size}, Address: {self.io_start}] {self.name}' - - def get_fields_str(self): - s = '' - s += f'\n * fres = {self.fres.__repr__()}' - s += f'\n * version_flag = {self.version_flag.__repr__()}' - s += f'\n * version = {self.version.__repr__()}' - s += f'\n * bitswap = {self.bitswap.__repr__()}' - s += f'\n * seventh_byte = {self.seventh_byte.__repr__()}' - s += f'\n * user_version = {self.user_version.__repr__()}' - return s - - def __repr__(self): - s = self.get_info_str() - s += self.get_fields_str() - s += '\n' - return s diff --git a/generated/formats/ovl/compound/Header.py b/generated/formats/ovl/compound/Header.py index 539e79735..23b01353f 100644 --- a/generated/formats/ovl/compound/Header.py +++ b/generated/formats/ovl/compound/Header.py @@ -6,13 +6,13 @@ from generated.formats.ovl.compound.DependencyEntry import DependencyEntry from generated.formats.ovl.compound.DirEntry import DirEntry from generated.formats.ovl.compound.FileEntry import FileEntry -from generated.formats.ovl.compound.GenericHeader import GenericHeader from generated.formats.ovl.compound.MimeEntry import MimeEntry -from generated.formats.ovl.compound.PadAlign import PadAlign from generated.formats.ovl.compound.Triplet import Triplet from generated.formats.ovl.compound.UnknownEntry import UnknownEntry -from generated.formats.ovl.compound.ZStringBuffer import ZStringBuffer from generated.formats.ovl.compound.ZlibInfo import ZlibInfo +from generated.formats.ovl_base.compound.GenericHeader import GenericHeader +from generated.formats.ovl_base.compound.PadAlign import PadAlign +from generated.formats.ovl_base.compound.ZStringBuffer import ZStringBuffer class Header(GenericHeader): diff --git a/generated/formats/ovl/compound/PadAlign.py b/generated/formats/ovl/compound/PadAlign.py index 738a0ee74..5141f649b 100644 --- a/generated/formats/ovl/compound/PadAlign.py +++ b/generated/formats/ovl/compound/PadAlign.py @@ -1,24 +1,23 @@ - +# START_GLOBALS from generated.io import MAX_LEN from modules.formats.shared import get_padding_size ZERO = b"\x00" +# END_GLOBALS + + from generated.context import ContextReference class PadAlign: + """Automatically aligns to arg's start and pads so aligned with align""" - """ - Grabs 00 bytes only - """ +# START_CLASS context = ContextReference() - def set_defaults(self): - pass - def __init__(self, context, arg=None, template=None): # arg is reference object self.name = '' @@ -40,4 +39,3 @@ def get_pad(self, stream): def __repr__(self): return f"{self.data} Size: {len(self.data)}" - diff --git a/generated/formats/ovl/compound/ZStringBuffer.py b/generated/formats/ovl/compound/ZStringBuffer.py deleted file mode 100644 index a71885b33..000000000 --- a/generated/formats/ovl/compound/ZStringBuffer.py +++ /dev/null @@ -1,73 +0,0 @@ - -import logging -from generated.io import BinaryStream -from modules.formats.shared import get_padding - -ZERO = b"\x00" - - -from generated.context import ContextReference - - -class ZStringBuffer: - - """ - Holds a buffer of zero-terminated strings - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - self.strings = [] - - def read(self, stream): - self.data = stream.read(self.arg) - self.strings = self.data.split(ZERO) - - def write(self, stream): - stream.write(self.data) - - def get_str_at(self, pos): - end = self.data.find(ZERO, pos) - return self.data[pos:end].decode() - - def update_with(self, list_of_arrays): - """Updates this name buffer with a list of (array, attrib) whose elements have - offset: bytes relative to the start of the name block - [attrib]: string""" - logging.debug("Updating name buffer") - self.strings = [] - offset_dic = {} - with BinaryStream() as stream: - # for name in self.names: - for array, attrib in list_of_arrays: - for item in sorted(array, key=lambda i: getattr(i, attrib)): - name = getattr(item, attrib) - if name in offset_dic: - # known string, just get offset - address = offset_dic[name] - else: - # new string, store offset and write zstring - address = stream.tell() - self.strings.append(name) - offset_dic[name] = address - stream.write_zstring(name) - # store offset on item - item.offset = address - # get the actual result buffer - buffer_bytes = stream.getvalue() - - self.data = buffer_bytes + get_padding(len(buffer_bytes), alignment=8) - - def __repr__(self): - return str(self.strings) - diff --git a/generated/formats/ovl/ovl.xml b/generated/formats/ovl/ovl.xml index 7b6b1bd35..918545e71 100644 --- a/generated/formats/ovl/ovl.xml +++ b/generated/formats/ovl/ovl.xml @@ -2,129 +2,9 @@ - - - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo pre-1.6 - Planet Zoo 1.6+ - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + - - Determines the format of the OVL file. - n.b. pos counts from the end! - - - - - - - - - - - - Holds a buffer of zero-terminated strings - - - - Holds a string of a fixed size, given as an argument. - - - - Grabs 00 bytes only - - - - Found at the beginning of every OVL file - 'FRES' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC, 0x48 for JWE Switch, may be platform - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - determines compression format (none, zlib or oodle) and apparently type of data (additional fields) - + Found at the beginning of every OVL file diff --git a/generated/formats/tex/bitfield/VersionInfo.py b/generated/formats/tex/bitfield/VersionInfo.py deleted file mode 100644 index 8f4b289b4..000000000 --- a/generated/formats/tex/bitfield/VersionInfo.py +++ /dev/null @@ -1,25 +0,0 @@ -from generated.bitfield import BasicBitfield -from generated.bitfield import BitfieldMember - - -class VersionInfo(BasicBitfield): - - """ - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - """ - unk_1 = BitfieldMember(pos=2, mask=0x4, return_type=bool) - unk_2 = BitfieldMember(pos=4, mask=0x10, return_type=bool) - use_zlib = BitfieldMember(pos=7, mask=0x80, return_type=bool) - use_oodle = BitfieldMember(pos=9, mask=0x200, return_type=bool) - unk_3 = BitfieldMember(pos=13, mask=0x2000, return_type=bool) - is_jwe = BitfieldMember(pos=14, mask=0x4000, return_type=bool) - - def set_defaults(self): - pass - - def read(self, stream): - self._value = stream.read_uint() - - def write(self, stream): - stream.write_uint(self._value) diff --git a/generated/formats/tex/tex.xml b/generated/formats/tex/tex.xml index 1c55cb524..bb9273faa 100644 --- a/generated/formats/tex/tex.xml +++ b/generated/formats/tex/tex.xml @@ -2,93 +2,7 @@ - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Planet Zoo 1.6 - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - - + maps the OVL's dds type to name of compression format @@ -135,17 +49,6 @@ - - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - - - - - - - - Data struct for headers of type 3, read by data 0 of 3,7 frag. 16 bytes diff --git a/generated/formats/tex/versions.py b/generated/formats/tex/versions.py index 8db126fc6..6b08a7244 100644 --- a/generated/formats/tex/versions.py +++ b/generated/formats/tex/versions.py @@ -29,7 +29,7 @@ def set_pc(context): def is_pz(context): - if context.version in (19, 20) and context.user_version in (8340, 8724): + if context.version == 19 and context.user_version in (8340, 8724): return True @@ -58,7 +58,7 @@ def set_jwe(context): context.user_version._value = 24724 -games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO', 'Planet Zoo'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) +games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6+'), ('PLANET_ZOO_PRE_1_6', 'Planet Zoo pre-1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) def get_game(context): @@ -69,7 +69,7 @@ def get_game(context): if is_pc(context): return [games.PLANET_COASTER] if is_pz(context): - return [games.PLANET_ZOO] + return [games.PLANET_ZOO_PRE_1_6] if is_pz16(context): return [games.PLANET_ZOO_1_6] if is_jwe(context): @@ -86,7 +86,7 @@ def set_game(context, game): return set_ztuac(context) if game in {games.PLANET_COASTER}: return set_pc(context) - if game in {games.PLANET_ZOO}: + if game in {games.PLANET_ZOO_PRE_1_6}: return set_pz(context) if game in {games.PLANET_ZOO_1_6}: return set_pz16(context) diff --git a/generated/formats/voxelskirt/bitfield/VersionInfo.py b/generated/formats/voxelskirt/bitfield/VersionInfo.py deleted file mode 100644 index 8f4b289b4..000000000 --- a/generated/formats/voxelskirt/bitfield/VersionInfo.py +++ /dev/null @@ -1,25 +0,0 @@ -from generated.bitfield import BasicBitfield -from generated.bitfield import BitfieldMember - - -class VersionInfo(BasicBitfield): - - """ - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - """ - unk_1 = BitfieldMember(pos=2, mask=0x4, return_type=bool) - unk_2 = BitfieldMember(pos=4, mask=0x10, return_type=bool) - use_zlib = BitfieldMember(pos=7, mask=0x80, return_type=bool) - use_oodle = BitfieldMember(pos=9, mask=0x200, return_type=bool) - unk_3 = BitfieldMember(pos=13, mask=0x2000, return_type=bool) - is_jwe = BitfieldMember(pos=14, mask=0x4000, return_type=bool) - - def set_defaults(self): - pass - - def read(self, stream): - self._value = stream.read_uint() - - def write(self, stream): - stream.write_uint(self._value) diff --git a/generated/formats/voxelskirt/compound/FixedString.py b/generated/formats/voxelskirt/compound/FixedString.py deleted file mode 100644 index b35efc941..000000000 --- a/generated/formats/voxelskirt/compound/FixedString.py +++ /dev/null @@ -1,32 +0,0 @@ -from generated.context import ContextReference - - -class FixedString: - - """ - Holds a string of a fixed size, given as an argument. - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - # arg is byte count - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - - diff --git a/generated/formats/voxelskirt/compound/Header.py b/generated/formats/voxelskirt/compound/Header.py index d7f6096c3..c902c9a5f 100644 --- a/generated/formats/voxelskirt/compound/Header.py +++ b/generated/formats/voxelskirt/compound/Header.py @@ -1,79 +1,38 @@ -from generated.context import ContextReference -from generated.formats.voxelskirt.bitfield.VersionInfo import VersionInfo -from generated.formats.voxelskirt.compound.FixedString import FixedString +from generated.formats.ovl_base.compound.GenericHeader import GenericHeader from generated.formats.voxelskirt.compound.SizedStrData import SizedStrData -class Header: +class Header(GenericHeader): """ Found at the beginning of every OVL file """ - context = ContextReference() - def __init__(self, context, arg=None, template=None): self.name = '' - self._context = context + super().__init__(context, arg, template) self.arg = arg self.template = template self.io_size = 0 self.io_start = 0 - # 'VOXE' - self.magic = FixedString(self.context, 4, None) - - # if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - self.version_flag = 0 - - # 0x12 = PC, 0x13 = JWE, PZ - self.version = 0 - - # endianness?, usually zero - self.bitswap = 0 - - # always = 1 - self.seventh_byte = 1 - - # determines compression format (none, zlib or oodle) and apparently type of data (additional fields) - self.user_version = VersionInfo() - # always = 0 self.info = SizedStrData(self.context, None, None) self.set_defaults() def set_defaults(self): - self.magic = FixedString(self.context, 4, None) - self.version_flag = 0 - self.version = 0 - self.bitswap = 0 - self.seventh_byte = 1 - self.user_version = VersionInfo() self.info = SizedStrData(self.context, None, None) def read(self, stream): self.io_start = stream.tell() - self.magic = stream.read_type(FixedString, (self.context, 4, None)) - self.version_flag = stream.read_byte() - self.context.version_flag = self.version_flag - self.version = stream.read_byte() - self.context.version = self.version - self.bitswap = stream.read_byte() - self.seventh_byte = stream.read_byte() - self.user_version = stream.read_type(VersionInfo) - self.context.user_version = self.user_version + super().read(stream) self.info = stream.read_type(SizedStrData, (self.context, None, None)) self.io_size = stream.tell() - self.io_start def write(self, stream): self.io_start = stream.tell() - stream.write_type(self.magic) - stream.write_byte(self.version_flag) - stream.write_byte(self.version) - stream.write_byte(self.bitswap) - stream.write_byte(self.seventh_byte) - stream.write_type(self.user_version) + super().write(stream) stream.write_type(self.info) self.io_size = stream.tell() - self.io_start @@ -83,12 +42,7 @@ def get_info_str(self): def get_fields_str(self): s = '' - s += f'\n * magic = {self.magic.__repr__()}' - s += f'\n * version_flag = {self.version_flag.__repr__()}' - s += f'\n * version = {self.version.__repr__()}' - s += f'\n * bitswap = {self.bitswap.__repr__()}' - s += f'\n * seventh_byte = {self.seventh_byte.__repr__()}' - s += f'\n * user_version = {self.user_version.__repr__()}' + s += super().get_fields_str() s += f'\n * info = {self.info.__repr__()}' return s diff --git a/generated/formats/voxelskirt/compound/ZStringBuffer.py b/generated/formats/voxelskirt/compound/ZStringBuffer.py deleted file mode 100644 index a71885b33..000000000 --- a/generated/formats/voxelskirt/compound/ZStringBuffer.py +++ /dev/null @@ -1,73 +0,0 @@ - -import logging -from generated.io import BinaryStream -from modules.formats.shared import get_padding - -ZERO = b"\x00" - - -from generated.context import ContextReference - - -class ZStringBuffer: - - """ - Holds a buffer of zero-terminated strings - """ - - context = ContextReference() - - def set_defaults(self): - pass - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - self.strings = [] - - def read(self, stream): - self.data = stream.read(self.arg) - self.strings = self.data.split(ZERO) - - def write(self, stream): - stream.write(self.data) - - def get_str_at(self, pos): - end = self.data.find(ZERO, pos) - return self.data[pos:end].decode() - - def update_with(self, list_of_arrays): - """Updates this name buffer with a list of (array, attrib) whose elements have - offset: bytes relative to the start of the name block - [attrib]: string""" - logging.debug("Updating name buffer") - self.strings = [] - offset_dic = {} - with BinaryStream() as stream: - # for name in self.names: - for array, attrib in list_of_arrays: - for item in sorted(array, key=lambda i: getattr(i, attrib)): - name = getattr(item, attrib) - if name in offset_dic: - # known string, just get offset - address = offset_dic[name] - else: - # new string, store offset and write zstring - address = stream.tell() - self.strings.append(name) - offset_dic[name] = address - stream.write_zstring(name) - # store offset on item - item.offset = address - # get the actual result buffer - buffer_bytes = stream.getvalue() - - self.data = buffer_bytes + get_padding(len(buffer_bytes), alignment=8) - - def __repr__(self): - return str(self.strings) - diff --git a/generated/formats/voxelskirt/versions.py b/generated/formats/voxelskirt/versions.py index 8c2c5bd82..6b08a7244 100644 --- a/generated/formats/voxelskirt/versions.py +++ b/generated/formats/voxelskirt/versions.py @@ -1,6 +1,15 @@ from enum import Enum +def is_dla(context): + if context.version == 15: + return True + + +def set_dla(context): + context.version = 15 + + def is_ztuac(context): if context.version == 17: return True @@ -29,27 +38,40 @@ def set_pz(context): context.user_version._value = 8340 +def is_pz16(context): + if context.version == 20 and context.user_version in (8340, 8724): + return True + + +def set_pz16(context): + context.version = 20 + context.user_version._value = 8340 + + def is_jwe(context): - if context.version_flag == 1 and context.version == 19 and context.user_version in (24724, 25108): + if context.version == 19 and context.user_version in (24724, 25108): return True def set_jwe(context): - context.version_flag = 1 context.version = 19 context.user_version._value = 24724 -games = Enum('Games',[('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO', 'Planet Zoo'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) +games = Enum('Games',[('DISNEYLAND_ADVENTURE', 'Disneyland Adventure'), ('JURASSIC_WORLD_EVOLUTION', 'Jurassic World Evolution'), ('PLANET_COASTER', 'Planet Coaster'), ('PLANET_ZOO_1_6', 'Planet Zoo 1.6+'), ('PLANET_ZOO_PRE_1_6', 'Planet Zoo pre-1.6'), ('ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION', 'Zoo Tycoon Ultimate Animal Collection'), ('UNKNOWN_GAME', 'Unknown Game')]) def get_game(context): + if is_dla(context): + return [games.DISNEYLAND_ADVENTURE] if is_ztuac(context): return [games.ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION] if is_pc(context): return [games.PLANET_COASTER] if is_pz(context): - return [games.PLANET_ZOO] + return [games.PLANET_ZOO_PRE_1_6] + if is_pz16(context): + return [games.PLANET_ZOO_1_6] if is_jwe(context): return [games.JURASSIC_WORLD_EVOLUTION] return [games.UNKOWN_GAME] @@ -58,12 +80,16 @@ def get_game(context): def set_game(context, game): if isinstance(game, str): game = games(game) + if game in {games.DISNEYLAND_ADVENTURE}: + return set_dla(context) if game in {games.ZOO_TYCOON_ULTIMATE_ANIMAL_COLLECTION}: return set_ztuac(context) if game in {games.PLANET_COASTER}: return set_pc(context) - if game in {games.PLANET_ZOO}: + if game in {games.PLANET_ZOO_PRE_1_6}: return set_pz(context) + if game in {games.PLANET_ZOO_1_6}: + return set_pz16(context) if game in {games.JURASSIC_WORLD_EVOLUTION}: return set_jwe(context) diff --git a/generated/formats/voxelskirt/voxelskirt.xml b/generated/formats/voxelskirt/voxelskirt.xml index 7f0806238..e9682fe4a 100644 --- a/generated/formats/voxelskirt/voxelskirt.xml +++ b/generated/formats/voxelskirt/voxelskirt.xml @@ -2,123 +2,12 @@ - - - - - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Jurassic World Evolution - - - Commonly used version expressions. - ZTUAC - PC - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - - - - Null terminated string. - - - - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - - - - - - - + - - Holds a buffer of zero-terminated strings - - - - Holds a string of a fixed size, given as an argument. - - - + Found at the beginning of every OVL file - 'VOXE' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - determines compression format (none, zlib or oodle) and apparently type of data (additional fields) always = 0 diff --git a/source/formats/bani/bani.xml b/source/formats/bani/bani.xml index 9db8d25d9..b4865b66d 100644 --- a/source/formats/bani/bani.xml +++ b/source/formats/bani/bani.xml @@ -2,65 +2,7 @@ - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + A string of given length. @@ -101,7 +43,7 @@ Member 3,3 (bottom left) - + A 4x4 transformation matrix. The (1,1) element. The (2,1) element. diff --git a/generated/formats/ovl/bitfield/__init__.py b/source/formats/base/__init__.py similarity index 100% rename from generated/formats/ovl/bitfield/__init__.py rename to source/formats/base/__init__.py diff --git a/source/formats/base/base.xml b/source/formats/base/base.xml new file mode 100644 index 000000000..d736bd143 --- /dev/null +++ b/source/formats/base/base.xml @@ -0,0 +1,61 @@ + + + + + + All Operators except for unary not (!), parentheses, and member of (\) + NOTE: These can be ignored entirely by string substitution and dealt with directly. + NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. + + + + + + + + + + + + + + + + + + + + + An signed 8-bit integer. + + + An unsigned 8-bit integer. + + + + An unsigned 64-bit integer. + + + An unsigned 32-bit integer. + + + An unsigned 16-bit integer. + + + + A signed 32-bit integer. + + + + A signed 16-bit integer. + + + + An 8-bit character. + + + + A standard 32-bit floating point number. + + + diff --git a/source/formats/bnk/bnk.xml b/source/formats/bnk/bnk.xml index 703a33172..be3a63e83 100644 --- a/source/formats/bnk/bnk.xml +++ b/source/formats/bnk/bnk.xml @@ -2,30 +2,9 @@ - + - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - + @@ -33,42 +12,9 @@ Null terminated string. - - An unsigned 8-bit integer. - - - A signed 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - A signed 64-bit integer. - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - second Section of a soundback aux diff --git a/source/formats/fgm/fgm.xml b/source/formats/fgm/fgm.xml index 71b574f78..c37e53c1f 100644 --- a/source/formats/fgm/fgm.xml +++ b/source/formats/fgm/fgm.xml @@ -2,106 +2,7 @@ - - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Planet Zoo 1.6 - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - Zero terminated bytes. - - - - A signed 16-bit integer. - - - - A standard 32-bit floating point number. - - - - Determines the format of the OVL file. - n.b. pos counts from the end! - - - - - - - + 4 bytes @@ -110,7 +11,6 @@ - Sized str entry of 16 bytes @@ -153,16 +53,10 @@ - + Custom header struct This reads a whole custom FGM file - 'FGM ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - fragment count texture ref count byte count to check for quirks diff --git a/source/formats/manis/manis.xml b/source/formats/manis/manis.xml index 3eaa9544e..6d31ee927 100644 --- a/source/formats/manis/manis.xml +++ b/source/formats/manis/manis.xml @@ -2,9 +2,6 @@ - - - Commonly used version expressions. PZ @@ -20,65 +17,10 @@ - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - + - - A standard 32-bit floating point number. - + + A string of given length. @@ -86,13 +28,13 @@ The string itself. - - Grabs 00 bytes only - + + Grabs 00 bytes only + - - Grabs 00 bytes only - + + Grabs 00 bytes only + A string of given length. diff --git a/source/formats/matcol/matcol.xml b/source/formats/matcol/matcol.xml index 3bc82d0d4..35344d8ac 100644 --- a/source/formats/matcol/matcol.xml +++ b/source/formats/matcol/matcol.xml @@ -2,9 +2,6 @@ - - - Commonly used version expressions. PZ @@ -14,69 +11,14 @@ Global Tokens. NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - + - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - - - A signed 32-bit integer. - + - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + + Null terminated string. diff --git a/source/formats/ms2/compound/FixedString.py b/source/formats/ms2/compound/FixedString.py deleted file mode 100644 index 03333152c..000000000 --- a/source/formats/ms2/compound/FixedString.py +++ /dev/null @@ -1,22 +0,0 @@ -class FixedString: - """Holds a string of a fixed size, given as an argument.""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - # arg is byte count - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - diff --git a/source/formats/ms2/ms2.xml b/source/formats/ms2/ms2.xml index 9990ba328..8241e41dc 100644 --- a/source/formats/ms2/ms2.xml +++ b/source/formats/ms2/ms2.xml @@ -2,95 +2,13 @@ - - - - - - Old - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC PC, ZTUAC - PZ - PZ - JWE, 25108 is JWE on switch - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - A signed 8-bit integer. - - - - An unsigned 32-bit integer. - - - - An unsigned 64-bit integer. - - - - An unsigned 16-bit integer. - + - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + @@ -126,22 +44,10 @@ - - Holds a string of a fixed size, given as an argument. - - - - Null terminated string. - - Grabs 00 bytes only - - Holds a buffer of zero-terminated strings - - A string of given length. @@ -314,16 +220,10 @@ seems to be zeros - + Custom header struct This reads a whole custom mdl2 file - 'MS2 ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - index of this model inside the ms2 used to find bone info name of ms2 @@ -347,15 +247,9 @@ index into models array - + Custom header struct includes fragments but none of the 3 data buffers - 'MS2 ' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - diff --git a/source/formats/ovl/compound/FixedString.py b/source/formats/ovl/compound/FixedString.py deleted file mode 100644 index ac8f40017..000000000 --- a/source/formats/ovl/compound/FixedString.py +++ /dev/null @@ -1,22 +0,0 @@ -class FixedString: - """Holds a string of a fixed size, given as an argument.""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - diff --git a/source/formats/ovl/compound/PadAlign.py b/source/formats/ovl/compound/PadAlign.py index 22fb85659..5141f649b 100644 --- a/source/formats/ovl/compound/PadAlign.py +++ b/source/formats/ovl/compound/PadAlign.py @@ -7,11 +7,17 @@ # END_GLOBALS + +from generated.context import ContextReference + + class PadAlign: """Automatically aligns to arg's start and pads so aligned with align""" # START_CLASS + context = ContextReference() + def __init__(self, context, arg=None, template=None): # arg is reference object self.name = '' diff --git a/source/formats/ovl/compound/ZStringBuffer.py b/source/formats/ovl/compound/ZStringBuffer.py deleted file mode 100644 index f8065c55c..000000000 --- a/source/formats/ovl/compound/ZStringBuffer.py +++ /dev/null @@ -1,65 +0,0 @@ -# START_GLOBALS -import logging -from generated.io import BinaryStream -from modules.formats.shared import get_padding - -ZERO = b"\x00" - - -# END_GLOBALS - -class ZStringBuffer: - """Holds a buffer of zero-terminated strings""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - self.strings = [] - - def read(self, stream): - self.data = stream.read(self.arg) - self.strings = self.data.split(ZERO) - - def write(self, stream): - stream.write(self.data) - - def get_str_at(self, pos): - end = self.data.find(ZERO, pos) - return self.data[pos:end].decode() - - def update_with(self, list_of_arrays): - """Updates this name buffer with a list of (array, attrib) whose elements have - offset: bytes relative to the start of the name block - [attrib]: string""" - logging.debug("Updating name buffer") - self.strings = [] - offset_dic = {} - with BinaryStream() as stream: - # for name in self.names: - for array, attrib in list_of_arrays: - for item in sorted(array, key=lambda i: getattr(i, attrib)): - name = getattr(item, attrib) - if name in offset_dic: - # known string, just get offset - address = offset_dic[name] - else: - # new string, store offset and write zstring - address = stream.tell() - self.strings.append(name) - offset_dic[name] = address - stream.write_zstring(name) - # store offset on item - item.offset = address - # get the actual result buffer - buffer_bytes = stream.getvalue() - - self.data = buffer_bytes + get_padding(len(buffer_bytes), alignment=8) - - def __repr__(self): - return str(self.strings) diff --git a/source/formats/ovl/ovl.xml b/source/formats/ovl/ovl.xml index 7b6b1bd35..918545e71 100644 --- a/source/formats/ovl/ovl.xml +++ b/source/formats/ovl/ovl.xml @@ -2,129 +2,9 @@ - - - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo pre-1.6 - Planet Zoo 1.6+ - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - + - - Determines the format of the OVL file. - n.b. pos counts from the end! - - - - - - - - - - - - Holds a buffer of zero-terminated strings - - - - Holds a string of a fixed size, given as an argument. - - - - Grabs 00 bytes only - - - - Found at the beginning of every OVL file - 'FRES' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC, 0x48 for JWE Switch, may be platform - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - determines compression format (none, zlib or oodle) and apparently type of data (additional fields) - + Found at the beginning of every OVL file diff --git a/generated/formats/tex/bitfield/__init__.py b/source/formats/ovl_base/__init__.py similarity index 100% rename from generated/formats/tex/bitfield/__init__.py rename to source/formats/ovl_base/__init__.py diff --git a/generated/formats/tex/compound/FixedString.py b/source/formats/ovl_base/compound/FixedString.py similarity index 100% rename from generated/formats/tex/compound/FixedString.py rename to source/formats/ovl_base/compound/FixedString.py diff --git a/source/formats/ms2/compound/ZStringBuffer.py b/source/formats/ovl_base/compound/ZStringBuffer.py similarity index 100% rename from source/formats/ms2/compound/ZStringBuffer.py rename to source/formats/ovl_base/compound/ZStringBuffer.py index f8065c55c..c153ddb51 100644 --- a/source/formats/ms2/compound/ZStringBuffer.py +++ b/source/formats/ovl_base/compound/ZStringBuffer.py @@ -14,9 +14,9 @@ class ZStringBuffer: # START_CLASS def __init__(self, context, arg=None, template=None): - # arg is byte count self.name = '' self._context = context + # arg is byte count self.arg = arg self.template = template self.data = b"" diff --git a/generated/formats/voxelskirt/bitfield/__init__.py b/source/formats/ovl_base/compound/__init__.py similarity index 100% rename from generated/formats/voxelskirt/bitfield/__init__.py rename to source/formats/ovl_base/compound/__init__.py diff --git a/source/formats/ovl_base/ovl_base.xml b/source/formats/ovl_base/ovl_base.xml new file mode 100644 index 000000000..799b40525 --- /dev/null +++ b/source/formats/ovl_base/ovl_base.xml @@ -0,0 +1,81 @@ + + + + + + Commonly used version expressions. + Disneyland Adventure + ZTUAC + PC + PZ + PZ + JWE, 25108 is JWE on switch + + + + Global Tokens. + NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. + + + + + + + + + + + + Disneyland Adventure + Zoo Tycoon Ultimate Animal Collection + Planet Coaster + Planet Zoo pre-1.6 + Planet Zoo 1.6+ + Jurassic World Evolution + + + Determines the format of the OVL file. + n.b. pos counts from the end! + + + + + + + + + + + + An unsigned 8-bit integer. + + + + A string of given length. + + + + + + Holds a buffer of zero-terminated strings + + + + Holds a string of a fixed size, given as an argument. + + + + Grabs 00 bytes only + + + + Found at the beginning of every OVL file + 'FGM ' for fgm, 'FRES' for ovl, 'MANI' for manis, 'MS2 ' for ms2, 'VOXE' for voxelskirt + if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC, 0x48 for JWE Switch, may be platform + 0x12 = PC, 0x13 = JWE, PZ + endianness?, usually zero + always = 1 + determines compression format (none, zlib or oodle) and apparently type of data (additional fields) + + + diff --git a/source/formats/tex/compound/FixedString.py b/source/formats/tex/compound/FixedString.py deleted file mode 100644 index 03333152c..000000000 --- a/source/formats/tex/compound/FixedString.py +++ /dev/null @@ -1,22 +0,0 @@ -class FixedString: - """Holds a string of a fixed size, given as an argument.""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - # arg is byte count - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - diff --git a/source/formats/tex/tex.xml b/source/formats/tex/tex.xml index 1c55cb524..bb9273faa 100644 --- a/source/formats/tex/tex.xml +++ b/source/formats/tex/tex.xml @@ -2,93 +2,7 @@ - - - - Disneyland Adventure - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Planet Zoo 1.6 - Jurassic World Evolution - - - Commonly used version expressions. - Disneyland Adventure - ZTUAC - PC - PZ - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - - + maps the OVL's dds type to name of compression format @@ -135,17 +49,6 @@ - - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - - - - - - - - Data struct for headers of type 3, read by data 0 of 3,7 frag. 16 bytes diff --git a/source/formats/voxelskirt/compound/FixedString.py b/source/formats/voxelskirt/compound/FixedString.py deleted file mode 100644 index 03333152c..000000000 --- a/source/formats/voxelskirt/compound/FixedString.py +++ /dev/null @@ -1,22 +0,0 @@ -class FixedString: - """Holds a string of a fixed size, given as an argument.""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - self.name = '' - self._context = context - # arg is byte count - self.arg = arg - self.template = template - self.data = b"" - - def read(self, stream): - self.data = stream.read(self.arg) - - def write(self, stream): - stream.write(self.data) - - def __repr__(self): - return str(self.data) - diff --git a/source/formats/voxelskirt/compound/ZStringBuffer.py b/source/formats/voxelskirt/compound/ZStringBuffer.py deleted file mode 100644 index f8065c55c..000000000 --- a/source/formats/voxelskirt/compound/ZStringBuffer.py +++ /dev/null @@ -1,65 +0,0 @@ -# START_GLOBALS -import logging -from generated.io import BinaryStream -from modules.formats.shared import get_padding - -ZERO = b"\x00" - - -# END_GLOBALS - -class ZStringBuffer: - """Holds a buffer of zero-terminated strings""" - -# START_CLASS - - def __init__(self, context, arg=None, template=None): - # arg is byte count - self.name = '' - self._context = context - self.arg = arg - self.template = template - self.data = b"" - self.strings = [] - - def read(self, stream): - self.data = stream.read(self.arg) - self.strings = self.data.split(ZERO) - - def write(self, stream): - stream.write(self.data) - - def get_str_at(self, pos): - end = self.data.find(ZERO, pos) - return self.data[pos:end].decode() - - def update_with(self, list_of_arrays): - """Updates this name buffer with a list of (array, attrib) whose elements have - offset: bytes relative to the start of the name block - [attrib]: string""" - logging.debug("Updating name buffer") - self.strings = [] - offset_dic = {} - with BinaryStream() as stream: - # for name in self.names: - for array, attrib in list_of_arrays: - for item in sorted(array, key=lambda i: getattr(i, attrib)): - name = getattr(item, attrib) - if name in offset_dic: - # known string, just get offset - address = offset_dic[name] - else: - # new string, store offset and write zstring - address = stream.tell() - self.strings.append(name) - offset_dic[name] = address - stream.write_zstring(name) - # store offset on item - item.offset = address - # get the actual result buffer - buffer_bytes = stream.getvalue() - - self.data = buffer_bytes + get_padding(len(buffer_bytes), alignment=8) - - def __repr__(self): - return str(self.strings) diff --git a/source/formats/voxelskirt/compound/__init__.py b/source/formats/voxelskirt/compound/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/source/formats/voxelskirt/voxelskirt.xml b/source/formats/voxelskirt/voxelskirt.xml index 7f0806238..e9682fe4a 100644 --- a/source/formats/voxelskirt/voxelskirt.xml +++ b/source/formats/voxelskirt/voxelskirt.xml @@ -2,123 +2,12 @@ - - - - - Zoo Tycoon Ultimate Animal Collection - Planet Coaster - Planet Zoo - Jurassic World Evolution - - - Commonly used version expressions. - ZTUAC - PC - PZ - JWE, 25108 is JWE on switch - - - - Global Tokens. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - All Operators except for unary not (!), parentheses, and member of (\) - NOTE: These can be ignored entirely by string substitution and dealt with directly. - NOTE: These must be listed after the above tokens so that they replace last. For example, `verexpr` uses these tokens. - - - - - - - - - - - - - - - - - - - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - An unsigned 8-bit integer. - - - - An unsigned 64-bit integer. - - - An unsigned 32-bit integer. - - - An unsigned 16-bit integer. - - - - A signed 32-bit integer. - - - - A signed 16-bit integer. - - - - An 8-bit character. - - - - A standard 32-bit floating point number. - - - - Null terminated string. - - - - Determines the format of the OVL file held by a mesh. - n.b. pos counts from the end! - - - - - - - + - - Holds a buffer of zero-terminated strings - - - - Holds a string of a fixed size, given as an argument. - - - + Found at the beginning of every OVL file - 'VOXE' - if 0x08 then 64bit, 0x01 for JWE, PZ, 0x08 for PC - 0x12 = PC, 0x13 = JWE, PZ - endianness?, usually zero - always = 1 - determines compression format (none, zlib or oodle) and apparently type of data (additional fields) always = 0 From 902531544a893baa6bae108ba1f0e7542b0f4fc6 Mon Sep 17 00:00:00 2001 From: Candoran2 <45334438+Candoran2@users.noreply.github.com> Date: Fri, 5 Nov 2021 01:11:03 +0100 Subject: [PATCH 6/6] Added conversion in case the passed xml file (path) was not yet absolute path. --- codegen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codegen.py b/codegen.py index 240d26d08..9f61bf6a3 100644 --- a/codegen.py +++ b/codegen.py @@ -90,6 +90,7 @@ def load_xml(self, xml_file, parsed_xmls=None): except AttributeError: # if attribute error, assume it was a file path xml_path = xml_file + xml_path = os.path.realpath(xml_path) tree = ET.parse(xml_file) root = tree.getroot() self.generate_module_paths(root)