From ddc2e6f068f00b03de5edc21c3b30b8a763659d5 Mon Sep 17 00:00:00 2001 From: Armijn Hemel Date: Sat, 10 Apr 2021 22:12:01 +0200 Subject: [PATCH 01/13] add XAR format --- archive/xar_header.ksy | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 archive/xar_header.ksy diff --git a/archive/xar_header.ksy b/archive/xar_header.ksy new file mode 100644 index 000000000..094115c5a --- /dev/null +++ b/archive/xar_header.ksy @@ -0,0 +1,40 @@ +meta: + id: xar_header + title: eXtensible ARchiver + file-extension: + - pkg + - xar + - xip + xref: + mime: application/x-xar + pronom: fmt/600 + wikidata: Q1093556 + license: CC0-1.0 + ks-version: 0.9 + endian: be +doc-ref: https://github.com/mackyle/xar/wiki/xarformat +seq: + - id: magic + contents: 'xar!' + - id: header_size + type: u2 + - id: version + type: u2 + valid: 1 + - id: len_toc_compressed + -orig-id: toc_length_compressed + type: u8 + - id: toc_length_uncompressed + type: u8 + - id: checksum_algorithm + type: u4 + enum: checksum_algorithms + - id: toc + size: len_toc_compressed + process: zlib + doc: zlib compressed XML further describing the content of the archive +enums: + checksum_algorithms: + 0: none + 1: sha1 + 2: md5 From af86dd5d2dd253e1986798b9cfeb02990ad838c2 Mon Sep 17 00:00:00 2001 From: Armijn Hemel Date: Sat, 10 Apr 2021 22:52:32 +0200 Subject: [PATCH 02/13] xar: document variations, add support for versions with checksum_algorithm field 3 created with the XAR archiver referenced in the doc-ref --- archive/xar_header.ksy | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/archive/xar_header.ksy b/archive/xar_header.ksy index 094115c5a..04ddcea84 100644 --- a/archive/xar_header.ksy +++ b/archive/xar_header.ksy @@ -11,7 +11,20 @@ meta: wikidata: Q1093556 license: CC0-1.0 ks-version: 0.9 + encoding: UTF-8 endian: be +doc: | + From Wikipedia: + + "XAR (short for eXtensible ARchive format) is an open source file archiver + and the archiver’s file format. It was created within the OpenDarwin project + and is used in macOS X 10.5 and up for software installation routines, as + well as browser extensions in Safari 5.0 and up." + + It should be noted that there is a different version from Apple uses the + same version number as the (unmaintained) version described here, but that + supports additional checksums, but doesn't have checksum_name. +versions doc-ref: https://github.com/mackyle/xar/wiki/xarformat seq: - id: magic @@ -29,6 +42,10 @@ seq: - id: checksum_algorithm type: u4 enum: checksum_algorithms + - id: checksum_name + type: strz + size: 36 + if: checksum_algorithm == checksum_algorithms::other - id: toc size: len_toc_compressed process: zlib @@ -38,3 +55,4 @@ enums: 0: none 1: sha1 2: md5 + 3: other From 5ae77b1b0fca8fe3e2e38fd9de5e784768f37e7d Mon Sep 17 00:00:00 2001 From: Armijn Hemel Date: Mon, 12 Apr 2021 14:53:47 +0200 Subject: [PATCH 03/13] xar: differentiate between the regular header and the apple header (probably misnomers) --- archive/xar_header.ksy | 73 ++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/archive/xar_header.ksy b/archive/xar_header.ksy index 04ddcea84..e40aa2790 100644 --- a/archive/xar_header.ksy +++ b/archive/xar_header.ksy @@ -24,35 +24,66 @@ doc: | It should be noted that there is a different version from Apple uses the same version number as the (unmaintained) version described here, but that supports additional checksums, but doesn't have checksum_name. -versions doc-ref: https://github.com/mackyle/xar/wiki/xarformat seq: - id: magic contents: 'xar!' - - id: header_size + - id: len_header type: u2 - - id: version - type: u2 - valid: 1 - - id: len_toc_compressed - -orig-id: toc_length_compressed - type: u8 - - id: toc_length_uncompressed - type: u8 - - id: checksum_algorithm - type: u4 - enum: checksum_algorithms - - id: checksum_name - type: strz - size: 36 - if: checksum_algorithm == checksum_algorithms::other - - id: toc - size: len_toc_compressed - process: zlib - doc: zlib compressed XML further describing the content of the archive + - id: header + type: + switch-on: len_header + cases: + 28: apple_header + _: regular_header +types: + regular_header: + seq: + - id: version + type: u2 + valid: 1 + - id: toc_length_compressed + -orig-id: toc_length_compressed + type: u8 + - id: toc_length_uncompressed + type: u8 + - id: checksum_algorithm + type: u4 + enum: checksum_algorithms + - id: checksum_name + type: strz + size: 36 + if: checksum_algorithm == checksum_algorithms::other + - id: toc + size: len_toc_compressed + process: zlib + doc: zlib compressed XML further describing the content of the archive + apple_header: + seq: + - id: version + type: u2 + valid: 1 + - id: toc_length_compressed + -orig-id: toc_length_compressed + type: u8 + - id: toc_length_uncompressed + type: u8 + - id: checksum_algorithm + type: u4 + enum: checksum_algorithms_apple + - id: toc + size: len_toc_compressed + process: zlib + doc: zlib compressed XML further describing the content of the archive enums: checksum_algorithms: 0: none 1: sha1 2: md5 3: other + checksum_algorithms_apple: + 0: none + 1: sha1 + 2: md5 + 3: sha256 + 4: sha512 From 09501888b949a50a5f95e27b4db349b87ca83ff6 Mon Sep 17 00:00:00 2001 From: Armijn Hemel Date: Mon, 12 Apr 2021 15:04:58 +0200 Subject: [PATCH 04/13] xar: use correct variable --- archive/xar_header.ksy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/xar_header.ksy b/archive/xar_header.ksy index e40aa2790..0fd6f2573 100644 --- a/archive/xar_header.ksy +++ b/archive/xar_header.ksy @@ -42,7 +42,7 @@ types: - id: version type: u2 valid: 1 - - id: toc_length_compressed + - id: len_toc_compressed -orig-id: toc_length_compressed type: u8 - id: toc_length_uncompressed @@ -63,7 +63,7 @@ types: - id: version type: u2 valid: 1 - - id: toc_length_compressed + - id: len_toc_compressed -orig-id: toc_length_compressed type: u8 - id: toc_length_uncompressed From 633d2eff7d4513fba917860a846e73ee31e11071 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 18 Aug 2021 19:29:17 +0200 Subject: [PATCH 05/13] xar: rename from `xar_header` --- archive/{xar_header.ksy => xar.ksy} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename archive/{xar_header.ksy => xar.ksy} (99%) diff --git a/archive/xar_header.ksy b/archive/xar.ksy similarity index 99% rename from archive/xar_header.ksy rename to archive/xar.ksy index 0fd6f2573..304d40b1b 100644 --- a/archive/xar_header.ksy +++ b/archive/xar.ksy @@ -1,5 +1,5 @@ meta: - id: xar_header + id: xar title: eXtensible ARchiver file-extension: - pkg From c747ddac7324b7368cf8683e97c89060db3174c3 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 20 Aug 2021 14:17:18 +0200 Subject: [PATCH 06/13] xar: add `/meta/xref/justsolve` --- archive/xar.ksy | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/xar.ksy b/archive/xar.ksy index 304d40b1b..b47c59561 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -6,6 +6,7 @@ meta: - xar - xip xref: + justsolve: Xar_(Extensible_Archive) mime: application/x-xar pronom: fmt/600 wikidata: Q1093556 From 5dcd6c89423db7de7c0bbd61c360a0607e7edf41 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 21 Aug 2021 15:44:20 +0200 Subject: [PATCH 07/13] xar: restructure (`header` to substream), provide cksum alg name --- archive/xar.ksy | 114 +++++++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 40 deletions(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index b47c59561..7057713cb 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -27,39 +27,31 @@ doc: | supports additional checksums, but doesn't have checksum_name. doc-ref: https://github.com/mackyle/xar/wiki/xarformat seq: - - id: magic - contents: 'xar!' - - id: len_header - type: u2 + - id: header_prefix + type: file_header_prefix + doc: internal; access `_root.header` instead - id: header - type: - switch-on: len_header - cases: - 28: apple_header - _: regular_header + size: header_prefix.len_header - header_prefix._sizeof + type: file_header + - id: toc + size: header.len_toc_compressed + type: toc_type + process: zlib + doc: zlib compressed XML further describing the content of the archive +instances: + checksum_algorithm_other: + -orig-id: XAR_CKSUM_OTHER + value: 3 + doc-ref: https://github.com/mackyle/xar/blob/66d451d/xar/include/xar.h.in#L85 types: - regular_header: + file_header_prefix: seq: - - id: version + - id: magic + contents: 'xar!' + - id: len_header type: u2 - valid: 1 - - id: len_toc_compressed - -orig-id: toc_length_compressed - type: u8 - - id: toc_length_uncompressed - type: u8 - - id: checksum_algorithm - type: u4 - enum: checksum_algorithms - - id: checksum_name - type: strz - size: 36 - if: checksum_algorithm == checksum_algorithms::other - - id: toc - size: len_toc_compressed - process: zlib - doc: zlib compressed XML further describing the content of the archive - apple_header: + doc: internal; access `_root.header.len_header` instead + file_header: seq: - id: version type: u2 @@ -69,19 +61,61 @@ types: type: u8 - id: toc_length_uncompressed type: u8 - - id: checksum_algorithm + - id: checksum_algorithm_int type: u4 - enum: checksum_algorithms_apple - - id: toc - size: len_toc_compressed - process: zlib - doc: zlib compressed XML further describing the content of the archive + doc: internal; access `checksum_algorithm_name` instead + - id: checksum_alg_name + size-eos: true + type: strz + valid: + expr: _ != "" and _ != "none" + if: has_checksum_alg_name + doc: internal; access `checksum_algorithm_name` instead + instances: + checksum_algorithm_name: + value: | + has_checksum_alg_name ? checksum_alg_name + : checksum_algorithm_int == checksum_algorithms_apple::none.to_i ? "none" + : checksum_algorithm_int == checksum_algorithms_apple::sha1.to_i ? "sha1" + : checksum_algorithm_int == checksum_algorithms_apple::md5.to_i ? "md5" + : checksum_algorithm_int == checksum_algorithms_apple::sha256.to_i ? "sha256" + : checksum_algorithm_int == checksum_algorithms_apple::sha512.to_i ? "sha512" + : "" + doc: | + If it is not + + * `""` (empty string), indicating an unknown integer value (read + `checksum_algorithm_int` for debugging purposes to find out + what that value is), or + * `"none"`, indicating that the TOC checksum is not provided (in that + case, the `` property or its `style` attribute should be + missing, or the `style` attribute must be set to `"none"`), + + it must exactly match the `style` attribute value of the + `` property in the root node ``. See + + for reference. + + The `xar` (eXtensible ARchiver) program [uses OpenSSL's function + `EVP_get_digestbyname`]( + https://github.com/mackyle/xar/blob/66d451d/xar/lib/archive.c#L328 + ) to verify this value (if it's not `""` or `"none"`, of course). + So it's reasonable to assume that this can only have one of the values + that OpenSSL recognizes. + has_checksum_alg_name: + value: | + checksum_algorithm_int == _root.checksum_algorithm_other + and len_header >= 32 + and len_header % 4 == 0 + len_header: + value: _root.header_prefix.len_header + toc_type: + seq: + - id: xml_string + type: str + size-eos: true enums: - checksum_algorithms: - 0: none - 1: sha1 - 2: md5 - 3: other + # https://github.com/opensource-apple/cctools/blob/fdb4825/include/xar/xar.h#L63-L67 checksum_algorithms_apple: 0: none 1: sha1 From b1a77a05bb392ac647dbec864fa00aca4aa2a5fa Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 21 Aug 2021 19:45:33 +0200 Subject: [PATCH 08/13] xar: change `/meta/title` --- archive/xar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index 7057713cb..ef4822956 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -1,6 +1,6 @@ meta: id: xar - title: eXtensible ARchiver + title: XAR (eXtensible ARchive) file-extension: - pkg - xar From 8512fae22aa66014624dab4d97cb65940073183d Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 21 Aug 2021 19:50:20 +0200 Subject: [PATCH 09/13] xar: improve sentence in `/doc` --- archive/xar.ksy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index ef4822956..05ae700c2 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -22,9 +22,9 @@ doc: | and is used in macOS X 10.5 and up for software installation routines, as well as browser extensions in Safari 5.0 and up." - It should be noted that there is a different version from Apple uses the - same version number as the (unmaintained) version described here, but that - supports additional checksums, but doesn't have checksum_name. + It should be noted that there is a different version from Apple which uses + the same version number as the (unmaintained) version described here and + supports additional checksums, but doesn't have `checksum_alg_name`. doc-ref: https://github.com/mackyle/xar/wiki/xarformat seq: - id: header_prefix From f8d1e11b60c9bdf642c6edb0d92a42b368d7e67e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 22 Aug 2021 10:17:48 +0200 Subject: [PATCH 10/13] xar: update Apple Open Source GitHub mirror link --- archive/xar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index 05ae700c2..63f9fd707 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -115,7 +115,7 @@ types: type: str size-eos: true enums: - # https://github.com/opensource-apple/cctools/blob/fdb4825/include/xar/xar.h#L63-L67 + # https://github.com/apple-opensource/xar/blob/03d10ac/xar/include/xar.h.in#L67-L73 checksum_algorithms_apple: 0: none 1: sha1 From 5c52a6653dba10cbd5d83a27989c858922a13b49 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 22 Aug 2021 11:00:43 +0200 Subject: [PATCH 11/13] xar: remove sentence from `/doc`, add link to Wikipedia --- archive/xar.ksy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index 63f9fd707..09e58447f 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -15,16 +15,12 @@ meta: encoding: UTF-8 endian: be doc: | - From Wikipedia: + From [Wikipedia](https://en.wikipedia.org/wiki/Xar_(archiver)): "XAR (short for eXtensible ARchive format) is an open source file archiver and the archiver’s file format. It was created within the OpenDarwin project and is used in macOS X 10.5 and up for software installation routines, as well as browser extensions in Safari 5.0 and up." - - It should be noted that there is a different version from Apple which uses - the same version number as the (unmaintained) version described here and - supports additional checksums, but doesn't have `checksum_alg_name`. doc-ref: https://github.com/mackyle/xar/wiki/xarformat seq: - id: header_prefix From 2ba263e6df3dce153976ef35dae62774051b48aa Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 25 Aug 2021 17:26:30 +0200 Subject: [PATCH 12/13] xar: move "xar" file extension above "pkg" See https://github.com/kaitai-io/kaitai_struct_formats/pull/473#discussion_r693482279 --- archive/xar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index 09e58447f..c491f4253 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -2,8 +2,8 @@ meta: id: xar title: XAR (eXtensible ARchive) file-extension: - - pkg - xar + - pkg - xip xref: justsolve: Xar_(Extensible_Archive) From 9bcda90dc5b59d057bdc3f16073315bde88e4443 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 25 Aug 2021 17:31:32 +0200 Subject: [PATCH 13/13] xar: change "read" to "access" to avoid confusion --- archive/xar.ksy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/xar.ksy b/archive/xar.ksy index c491f4253..b9632dc72 100644 --- a/archive/xar.ksy +++ b/archive/xar.ksy @@ -80,7 +80,7 @@ types: doc: | If it is not - * `""` (empty string), indicating an unknown integer value (read + * `""` (empty string), indicating an unknown integer value (access `checksum_algorithm_int` for debugging purposes to find out what that value is), or * `"none"`, indicating that the TOC checksum is not provided (in that