From c3e87805fdb7caca9c05f58367749c8aa9b31de7 Mon Sep 17 00:00:00 2001 From: cen1 Date: Wed, 2 Oct 2024 10:08:07 +0200 Subject: [PATCH] Feature/jlcompress options (#203) * Initial support for extended options in JlCompress, provide a fixed dateTime for compressed files (last modified), mostly useful for reproducible archives and unit tests. * bump macos runners * fix winqtdeploy for debug builds --- .github/workflows/ci.yml | 5 +- .github/workflows/qt-zlib.yml | 2 +- .gitignore | 2 + cmake/windeployqt.cmake | 3 +- quazip/JlCompress.cpp | 152 +++++++++++++++++++++------------- quazip/JlCompress.h | 78 +++++++++++++++++ quazip/quazipnewinfo.cpp | 9 ++ quazip/quazipnewinfo.h | 10 ++- qztest/testjlcompress.cpp | 122 +++++++++++++++++++++++++++ qztest/testjlcompress.h | 4 + 10 files changed, 324 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 568ff122..15709cc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,9 +111,12 @@ jobs: strategy: fail-fast: false matrix: - macos_version: [11, 12] + macos_version: [14, 13] qt_version: [5.15.2, 6.6.2] shared: [ON, OFF] + exclude: + - macos_version: 14 + qt_version: 5.15.2 #Not available on macos-14 due to ARM arch steps: - name: Checkout diff --git a/.github/workflows/qt-zlib.yml b/.github/workflows/qt-zlib.yml index 8ca2ff55..c97f3d5b 100644 --- a/.github/workflows/qt-zlib.yml +++ b/.github/workflows/qt-zlib.yml @@ -180,7 +180,7 @@ jobs: - name: Run tests shell: cmd working-directory: ${{github.workspace}}/build - run: ctest --verbose -C Release + run: ctest --verbose --output-on-failure -C Release use-qt5-zlib-windows: if: true diff --git a/.gitignore b/.gitignore index 7d8b8869..461946c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ /doc /build +/build_release +/build_debug /lib *.tags *.user diff --git a/cmake/windeployqt.cmake b/cmake/windeployqt.cmake index dcbb2e76..67d219cd 100644 --- a/cmake/windeployqt.cmake +++ b/cmake/windeployqt.cmake @@ -8,7 +8,8 @@ function(windeployqt target) add_custom_command(TARGET ${target} POST_BUILD COMMAND "${_qt_bin_dir}/windeployqt.exe" --verbose 1 - --release + $<$:--debug> + $<$:--release> --no-plugins --no-translations --no-system-d3d-compiler diff --git a/quazip/JlCompress.cpp b/quazip/JlCompress.cpp index 803ecd1b..ca48492e 100644 --- a/quazip/JlCompress.cpp +++ b/quazip/JlCompress.cpp @@ -24,6 +24,7 @@ see quazip/(un)zip.h files for details. Basically it's the zlib license. */ #include "JlCompress.h" +#include bool JlCompress::copyData(QIODevice &inFile, QIODevice &outFile) { @@ -39,6 +40,10 @@ bool JlCompress::copyData(QIODevice &inFile, QIODevice &outFile) } bool JlCompress::compressFile(QuaZip* zip, QString fileName, QString fileDest) { + return compressFile(zip, fileName, fileDest, Options()); +} + +bool JlCompress::compressFile(QuaZip* zip, QString fileName, QString fileDest, const Options& options) { // zip: object where to add the file // fileName: real file name // fileDest: file name inside the zip object @@ -49,7 +54,12 @@ bool JlCompress::compressFile(QuaZip* zip, QString fileName, QString fileDest) { zip->getMode()!=QuaZip::mdAdd) return false; QuaZipFile outFile(zip); - if(!outFile.open(QIODevice::WriteOnly, QuaZipNewInfo(fileDest, fileName))) return false; + if (options.getDateTime().isNull()) { + if(!outFile.open(QIODevice::WriteOnly, QuaZipNewInfo(fileDest, fileName))) return false; + } + else { + if(!outFile.open(QIODevice::WriteOnly, QuaZipNewInfo(fileDest, fileName, options.getDateTime()))) return false; + } QFileInfo input(fileName); if (quazip_is_symlink(input)) { @@ -74,6 +84,10 @@ bool JlCompress::compressFile(QuaZip* zip, QString fileName, QString fileDest) { } bool JlCompress::compressSubDir(QuaZip* zip, QString dir, QString origDir, bool recursive, QDir::Filters filters) { + return compressSubDir(zip, dir, origDir, recursive, filters, Options()); +} + +bool JlCompress::compressSubDir(QuaZip* zip, QString dir, QString origDir, bool recursive, QDir::Filters filters, const Options& options) { // zip: object where to add the file // dir: current real directory // origDir: original real directory @@ -88,14 +102,20 @@ bool JlCompress::compressSubDir(QuaZip* zip, QString dir, QString origDir, bool if (!directory.exists()) return false; QDir origDirectory(origDir); - if (dir != origDir) { - QuaZipFile dirZipFile(zip); - if (!dirZipFile.open(QIODevice::WriteOnly, - QuaZipNewInfo(origDirectory.relativeFilePath(dir) + QLatin1String("/"), dir), nullptr, 0, 0)) { - return false; - } - dirZipFile.close(); - } + if (dir != origDir) { + QuaZipFile dirZipFile(zip); + std::unique_ptr qzni; + if (options.getDateTime().isNull()) { + qzni = std::make_unique(origDirectory.relativeFilePath(dir) + QLatin1String("/"), dir); + } + else { + qzni = std::make_unique(origDirectory.relativeFilePath(dir) + QLatin1String("/"), dir, options.getDateTime()); + } + if (!dirZipFile.open(QIODevice::WriteOnly, *qzni, nullptr, 0, 0)) { + return false; + } + dirZipFile.close(); + } // Whether to compress the subfolders, recursion if (recursive) { @@ -105,7 +125,7 @@ bool JlCompress::compressSubDir(QuaZip* zip, QString dir, QString origDir, bool if (!file.isDir()) // needed for Qt < 4.7 because it doesn't understand AllDirs continue; // Compress subdirectory - if(!compressSubDir(zip,file.absoluteFilePath(),origDir,recursive,filters)) return false; + if(!compressSubDir(zip,file.absoluteFilePath(),origDir,recursive,filters, options)) return false; } } @@ -119,7 +139,7 @@ bool JlCompress::compressSubDir(QuaZip* zip, QString dir, QString origDir, bool QString filename = origDirectory.relativeFilePath(file.absoluteFilePath()); // Compress the file - if (!compressFile(zip,file.absoluteFilePath(),filename)) return false; + if (!compressFile(zip,file.absoluteFilePath(),filename, options)) return false; } return true; @@ -204,6 +224,10 @@ bool JlCompress::removeFile(QStringList listFile) { } bool JlCompress::compressFile(QString fileCompressed, QString file) { + return compressFile(fileCompressed, file, JlCompress::Options()); +} + +bool JlCompress::compressFile(QString fileCompressed, QString file, const Options& options) { // Create zip QuaZip zip(fileCompressed); QDir().mkpath(QFileInfo(fileCompressed).absolutePath()); @@ -213,7 +237,7 @@ bool JlCompress::compressFile(QString fileCompressed, QString file) { } // Add file - if (!compressFile(&zip,file,QFileInfo(file).fileName())) { + if (!compressFile(&zip,file,QFileInfo(file).fileName(), options)) { QFile::remove(fileCompressed); return false; } @@ -229,33 +253,37 @@ bool JlCompress::compressFile(QString fileCompressed, QString file) { } bool JlCompress::compressFiles(QString fileCompressed, QStringList files) { - // Create zip - QuaZip zip(fileCompressed); - QDir().mkpath(QFileInfo(fileCompressed).absolutePath()); - if(!zip.open(QuaZip::mdCreate)) { - QFile::remove(fileCompressed); - return false; - } - - // Compress files - QFileInfo info; - for (int index = 0; index < files.size(); ++index ) { - const QString & file( files.at( index ) ); - info.setFile(file); - if (!info.exists() || !compressFile(&zip,file,info.fileName())) { - QFile::remove(fileCompressed); - return false; - } - } - - // Close zip - zip.close(); - if(zip.getZipError()!=0) { - QFile::remove(fileCompressed); - return false; - } + return compressFiles(fileCompressed, files, Options()); +} - return true; +bool JlCompress::compressFiles(QString fileCompressed, QStringList files, const Options& options) { + // Create zip + QuaZip zip(fileCompressed); + QDir().mkpath(QFileInfo(fileCompressed).absolutePath()); + if(!zip.open(QuaZip::mdCreate)) { + QFile::remove(fileCompressed); + return false; + } + + // Compress files + QFileInfo info; + for (int index = 0; index < files.size(); ++index ) { + const QString & file( files.at( index ) ); + info.setFile(file); + if (!info.exists() || !compressFile(&zip,file,info.fileName(), options)) { + QFile::remove(fileCompressed); + return false; + } + } + + // Close zip + zip.close(); + if(zip.getZipError()!=0) { + QFile::remove(fileCompressed); + return false; + } + + return true; } bool JlCompress::compressDir(QString fileCompressed, QString dir, bool recursive) { @@ -265,28 +293,34 @@ bool JlCompress::compressDir(QString fileCompressed, QString dir, bool recursive bool JlCompress::compressDir(QString fileCompressed, QString dir, bool recursive, QDir::Filters filters) { - // Create zip - QuaZip zip(fileCompressed); - QDir().mkpath(QFileInfo(fileCompressed).absolutePath()); - if(!zip.open(QuaZip::mdCreate)) { - QFile::remove(fileCompressed); - return false; - } - - // Add the files and subdirectories - if (!compressSubDir(&zip,dir,dir,recursive, filters)) { - QFile::remove(fileCompressed); - return false; - } - - // Close zip - zip.close(); - if(zip.getZipError()!=0) { - QFile::remove(fileCompressed); - return false; - } + return compressDir(fileCompressed, dir, recursive, filters, Options()); +} - return true; +bool JlCompress::compressDir(QString fileCompressed, QString dir, + bool recursive, QDir::Filters filters, const Options& options) +{ + // Create zip + QuaZip zip(fileCompressed); + QDir().mkpath(QFileInfo(fileCompressed).absolutePath()); + if(!zip.open(QuaZip::mdCreate)) { + QFile::remove(fileCompressed); + return false; + } + + // Add the files and subdirectories + if (!compressSubDir(&zip,dir,dir,recursive, filters, options)) { + QFile::remove(fileCompressed); + return false; + } + + // Close zip + zip.close(); + if(zip.getZipError()!=0) { + QFile::remove(fileCompressed); + return false; + } + + return true; } QString JlCompress::extractFile(QString fileCompressed, QString fileName, QString fileDest) { diff --git a/quazip/JlCompress.h b/quazip/JlCompress.h index fcb767b8..1c1432e3 100644 --- a/quazip/JlCompress.h +++ b/quazip/JlCompress.h @@ -42,6 +42,25 @@ see quazip/(un)zip.h files for details. Basically it's the zlib license. */ class QUAZIP_EXPORT JlCompress { public: + class Options { + public: + explicit Options(const QDateTime& dateTime = QDateTime()) + : m_dateTime(dateTime) {} + + QDateTime getDateTime() const { + return m_dateTime; + } + + void setDateTime(const QDateTime &dateTime) { + m_dateTime = dateTime; + } + + private: + // If set, used as last modified on file inside the archive. + // If compressing a directory, used for all files. + QDateTime m_dateTime; + }; + static bool copyData(QIODevice &inFile, QIODevice &outFile); static QStringList extractDir(QuaZip &zip, const QString &dir); static QStringList getFileList(QuaZip *zip); @@ -55,6 +74,15 @@ class QUAZIP_EXPORT JlCompress { \return true if success, false otherwise. */ static bool compressFile(QuaZip* zip, QString fileName, QString fileDest); + /// Compress a single file. + /** + \param zip Opened zip to compress the file to. + \param fileName The full path to the source file. + \param fileDest The full name of the file inside the archive. + \param options Options for fixed file timestamp, compression level, encryption.. + \return true if success, false otherwise. + */ + static bool compressFile(QuaZip* zip, QString fileName, QString fileDest, const Options& options); /// Compress a subdirectory. /** \param parentZip Opened zip containing the parent directory. @@ -67,6 +95,21 @@ class QUAZIP_EXPORT JlCompress { */ static bool compressSubDir(QuaZip* parentZip, QString dir, QString parentDir, bool recursive, QDir::Filters filters); + /// Compress a subdirectory. + /** + \param parentZip Opened zip containing the parent directory. + \param dir The full path to the directory to pack. + \param parentDir The full path to the directory corresponding to + the root of the ZIP. + \param recursive Whether to pack sub-directories as well or only + \param filters what to pack, filters are applied both when searching +* for subdirs (if packing recursively) and when looking for files to pack + \param options Options for fixed file timestamp, compression level, encryption.. + files. + \return true if success, false otherwise. + */ + static bool compressSubDir(QuaZip* parentZip, QString dir, QString parentDir, bool recursive, + QDir::Filters filters, const Options& options); /// Extract a single file. /** \param zip The opened zip archive to extract from. @@ -89,6 +132,14 @@ class QUAZIP_EXPORT JlCompress { \return true if success, false otherwise. */ static bool compressFile(QString fileCompressed, QString file); + /// Compress a single file with advanced options. + /** + \param fileCompressed The name of the archive. + \param file The file to compress. + \param options Options for fixed file timestamp, compression level, encryption.. + \return true if success, false otherwise. + */ + static bool compressFile(QString fileCompressed, QString file, const Options& options); /// Compress a list of files. /** \param fileCompressed The name of the archive. @@ -96,6 +147,14 @@ class QUAZIP_EXPORT JlCompress { \return true if success, false otherwise. */ static bool compressFiles(QString fileCompressed, QStringList files); + /// Compress a list of files. + /** + \param fileCompressed The name of the archive. + \param files The file list to compress. + \param options Options for fixed file timestamp, compression level, encryption.. + \return true if success, false otherwise. + */ + static bool compressFiles(QString fileCompressed, QStringList files, const Options& options); /// Compress a whole directory. /** Does not compress hidden files. See compressDir(QString, QString, bool, QDir::Filters). @@ -125,6 +184,25 @@ class QUAZIP_EXPORT JlCompress { */ static bool compressDir(QString fileCompressed, QString dir, bool recursive, QDir::Filters filters); + /** + * @brief Compress a whole directory. + * + * Unless filters are specified explicitly, packs + * only regular non-hidden files (and subdirs, if @c recursive is true). + * If filters are specified, they are OR-combined with + * %QDir::AllDirs|%QDir::NoDotAndDotDot when searching for dirs + * and with QDir::Files when searching for files. + * + * @param fileCompressed path to the resulting archive + * @param dir path to the directory being compressed + * @param recursive if true, then the subdirectories are packed as well + * @param filters what to pack, filters are applied both when searching + * for subdirs (if packing recursively) and when looking for files to pack + * @param options Options for fixed file timestamp, compression level, encryption.. + * @return true on success, false otherwise + */ + static bool compressDir(QString fileCompressed, QString dir, + bool recursive, QDir::Filters filters, const Options& options); /// Extract a single file. /** diff --git a/quazip/quazipnewinfo.cpp b/quazip/quazipnewinfo.cpp index d0ab1e63..b55bb1b3 100644 --- a/quazip/quazipnewinfo.cpp +++ b/quazip/quazipnewinfo.cpp @@ -105,6 +105,15 @@ QuaZipNewInfo::QuaZipNewInfo(const QString& name, const QString& file): } } +QuaZipNewInfo::QuaZipNewInfo(const QString& name, const QString& file, const QDateTime& dateTime): + name(name), dateTime(dateTime), internalAttr(0), externalAttr(0), uncompressedSize(0) +{ + QFileInfo info(file); + if (info.exists()) { + QuaZipNewInfo_setPermissions(this, info.permissions(), info.isDir(), quazip_is_symlink(info)); + } +} + void QuaZipNewInfo::setFileDateTime(const QString& file) { QFileInfo info(file); diff --git a/quazip/quazipnewinfo.h b/quazip/quazipnewinfo.h index bad70c27..ee53d9bb 100644 --- a/quazip/quazipnewinfo.h +++ b/quazip/quazipnewinfo.h @@ -94,10 +94,18 @@ struct QUAZIP_EXPORT QuaZipNewInfo { * is inaccessible (e. g. you do not have read permission for the * directory file in), uses current time and zero permissions. Other attributes are * initialized with zeros, comment and extra field with null values. - * * \sa setFileDateTime() **/ QuaZipNewInfo(const QString& name, const QString& file); + /// Constructs QuaZipNewInfo instance. + /** Initializes name with \a name and provided timestamp. Permissions are taken + * from the specified file. If the \a file does not exists or + * is inaccessible (e. g. you do not have read permission for the + * directory file in), uses zero permissions. Other attributes are + * initialized with zeros, comment and extra field with null values. + * \sa setFileDateTime() + **/ + QuaZipNewInfo(const QString& name, const QString& file, const QDateTime& dateTime); /// Initializes the new instance from existing file info. /** Mainly used when copying files between archives. * diff --git a/qztest/testjlcompress.cpp b/qztest/testjlcompress.cpp index f9cde6f0..c81fface 100644 --- a/qztest/testjlcompress.cpp +++ b/qztest/testjlcompress.cpp @@ -74,6 +74,61 @@ void TestJlCompress::compressFile() curDir.remove(zipName); } +void TestJlCompress::compressFileOptions_data() +{ + QTest::addColumn("zipName"); + QTest::addColumn("fileName"); + QTest::addColumn("dateTime"); + QTest::addColumn("sha256sum_unix"); // Due to extra data archives are not identical + QTest::addColumn("sha256sum_win"); + QTest::newRow("simple") << "jlsimplefile.zip" + << "test0.txt" + << QDateTime(QDate(2024, 9, 19), QTime(21, 0, 0), QTimeZone::utc()) + << "5eedd83aee92cf3381155d167fee54a4ef6e43b8bc7a979c903611d9aa28610a" + << "cb1847dff1a5c33a805efde2558fc74024ad4c64c8607f8b12903e4d92385955"; +} + +void TestJlCompress::compressFileOptions() +{ + QFETCH(QString, zipName); + QFETCH(QString, fileName); + QFETCH(QDateTime, dateTime); + QFETCH(QString, sha256sum_unix); + QFETCH(QString, sha256sum_win); + QDir curDir; + if (curDir.exists(zipName)) { + if (!curDir.remove(zipName)) + QFAIL("Can't remove zip file"); + } + if (!createTestFiles(QStringList() << fileName)) { + QFAIL("Can't create test file"); + } + + const JlCompress::Options options(dateTime); + QVERIFY(JlCompress::compressFile(zipName, "tmp/" + fileName, options)); + // get the file list and check it + QStringList fileList = JlCompress::getFileList(zipName); + QCOMPARE(fileList.count(), 1); + QVERIFY(fileList[0] == fileName); + // now test the QIODevice* overload of getFileList() + QFile zipFile(zipName); + QVERIFY(zipFile.open(QIODevice::ReadOnly)); + fileList = JlCompress::getFileList(zipName); + QCOMPARE(fileList.count(), 1); + QVERIFY(fileList[0] == fileName); + // Hash is computed on the resulting file externally, then hardcoded in the test data + // This should help detecting any library breakage since we compare against a well-known stable result + QString hash = QCryptographicHash::hash(zipFile.readAll(), QCryptographicHash::Sha256).toHex(); +#ifdef _WIN32 + QCOMPARE(hash, sha256sum_win); +#else + QCOMPARE(hash, sha256sum_unix); +#endif + zipFile.close(); + removeTestFiles(QStringList() << fileName); + curDir.remove(zipName); +} + void TestJlCompress::compressFiles_data() { QTest::addColumn("zipName"); @@ -163,6 +218,73 @@ void TestJlCompress::compressDir() curDir.remove(zipName); } +void TestJlCompress::compressDirOptions_data() +{ + QTest::addColumn("zipName"); + QTest::addColumn("fileNames"); + QTest::addColumn("expected"); + QTest::addColumn("dateTime"); + QTest::addColumn("sha256sum_unix"); + QTest::addColumn("sha256sum_win"); + QTest::newRow("simple") << "jldir.zip" + << (QStringList() << "test0.txt" << "testdir1/test1.txt" + << "testdir2/test2.txt" << "testdir2/subdir/test2sub.txt") + << (QStringList() << "test0.txt" + << "testdir1/" << "testdir1/test1.txt" + << "testdir2/" << "testdir2/test2.txt" + << "testdir2/subdir/" << "testdir2/subdir/test2sub.txt") + << QDateTime(QDate(2024, 9, 19), QTime(21, 0, 0), QTimeZone::utc()) + << "ed0d5921b2fc11b6b4cb214b3e43ea3ea28987d6ff8080faab54c4756de30af6" + << "1eba110a33718c07a4ddf3fa515d1b4c6e3f4fc912b2e29e5e32783e2cddf852"; +} + +void TestJlCompress::compressDirOptions() +{ + QFETCH(QString, zipName); + QFETCH(QStringList, fileNames); + QFETCH(QStringList, expected); + QFETCH(QDateTime, dateTime); + QFETCH(QString, sha256sum_unix); + QFETCH(QString, sha256sum_win); + QDir curDir; + if (curDir.exists(zipName)) { + if (!curDir.remove(zipName)) + QFAIL("Can't remove zip file"); + } + if (!createTestFiles(fileNames, -1, "compressDir_tmp")) { + QFAIL("Can't create test files"); + } +#ifdef Q_OS_WIN + for (int i = 0; i < fileNames.size(); ++i) { + if (fileNames.at(i).startsWith(".")) { + QString fn = "compressDir_tmp\\" + fileNames.at(i); + SetFileAttributesW(reinterpret_cast(fn.utf16()), + FILE_ATTRIBUTE_HIDDEN); + } + } +#endif + const JlCompress::Options options(dateTime); + QVERIFY(JlCompress::compressDir(zipName, "compressDir_tmp", true, QDir::Hidden, options)); + // get the file list and check it + QStringList fileList = JlCompress::getFileList(zipName); + fileList.sort(); + expected.sort(); + QCOMPARE(fileList, expected); + QFile zipFile(curDir.absoluteFilePath(zipName)); + if (!zipFile.open(QIODevice::ReadOnly)) { + QFAIL("Can't read output zip file"); + } + QString hash = QCryptographicHash::hash(zipFile.readAll(), QCryptographicHash::Sha256).toHex(); +#ifdef _WIN32 + QCOMPARE(hash, sha256sum_win); +#else + QCOMPARE(hash, sha256sum_unix); +#endif + zipFile.close(); + removeTestFiles(fileNames, "compressDir_tmp"); + curDir.remove(zipName); +} + void TestJlCompress::extractFile_data() { QTest::addColumn("zipName"); diff --git a/qztest/testjlcompress.h b/qztest/testjlcompress.h index 73be47c2..2e4697ef 100644 --- a/qztest/testjlcompress.h +++ b/qztest/testjlcompress.h @@ -41,10 +41,14 @@ class TestJlCompress: public QObject { private slots: void compressFile_data(); void compressFile(); + void compressFileOptions_data(); + void compressFileOptions(); void compressFiles_data(); void compressFiles(); void compressDir_data(); void compressDir(); + void compressDirOptions_data(); + void compressDirOptions(); void extractFile_data(); void extractFile(); void extractFiles_data();