diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d8136af..49f3bb13e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,12 @@ Version counting is based on semantic versioning (Major.Feature.Patch) ### YACReader +* Support for ComicBookDS (.cbds) comics. Only the pages are shown, not the previews stored next to them. * Show the full title of the comic in the window title: series, volume, number and title. ### YACReaderLibrary +* Support for ComicBookDS (.cbds) comics. * Show comic titles the same way in all views. The info panel, the current comic banner and continue reading show the full title: series, volume, number and title. * Show the series next to the title in the grid cells of reading lists, labels, smart lists and search results, so comics from different series are easy to tell apart. diff --git a/YACReaderLibrary/db/comic_model.cpp b/YACReaderLibrary/db/comic_model.cpp index 5dc242d02..406ceaabf 100644 --- a/YACReaderLibrary/db/comic_model.cpp +++ b/YACReaderLibrary/db/comic_model.cpp @@ -815,7 +815,7 @@ QVariant ComicModel::headerData(int section, Qt::Orientation orientation, if (ext.compare("cbr", Qt::CaseInsensitive) == 0) return QVariant(QIcon(":/images/comicRar.png")); - else if (ext.compare("cbz", Qt::CaseInsensitive) == 0) + else if (ext.compare("cbz", Qt::CaseInsensitive) == 0 || ext.compare("cbds", Qt::CaseInsensitive) == 0) return QVariant(QIcon(":/images/comicZip.png")); else if (Comic::fileIsEpub(fileName)) return QVariant(QIcon(":/images/comicZip.png")); diff --git a/YACReaderLibrary/initial_comic_info_extractor.cpp b/YACReaderLibrary/initial_comic_info_extractor.cpp index 05adac5bb..f5224a82b 100644 --- a/YACReaderLibrary/initial_comic_info_extractor.cpp +++ b/YACReaderLibrary/initial_comic_info_extractor.cpp @@ -1,5 +1,6 @@ #include "initial_comic_info_extractor.h" +#include "cbds_page_index.h" #include "comic.h" #include "compressed_archive.h" #include "cover_utils.h" @@ -132,8 +133,12 @@ void InitialComicInfoExtractor::extract() _numPages = epub.pageCount; coverArchiveIndex = epub.coverArchiveIndex; } else { - QList fileNames = FileComic::filter(order); - std::sort(fileNames.begin(), fileNames.end(), naturalSortLessThanCI); + QList fileNames = FileComic::filter(FileComic::cbdsPageCandidates(_fileSource, order)); + if (YACReaderCbds::isCbds(_fileSource, order)) { + YACReaderCbds::sortPages(fileNames); + } else { + std::sort(fileNames.begin(), fileNames.end(), naturalSortLessThanCI); + } _numPages = fileNames.size(); if (_coverPage > _numPages) { _coverPage = 1; diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index e98d94ff7..829a0249a 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -43,6 +43,15 @@ target_include_directories(epub_page_index PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) yacreader_apply_build_options(epub_page_index) target_link_libraries(epub_page_index PUBLIC Qt6::Core) +# --- ComicBookDS page index (selects the page entries of .cbds books) --- +add_library(cbds_page_index STATIC + cbds_page_index.h + cbds_page_index.cpp +) +target_include_directories(cbds_page_index PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +yacreader_apply_build_options(cbds_page_index) +target_link_libraries(cbds_page_index PUBLIC Qt6::Core naturalsort) + # --- concurrent_queue --- add_library(concurrent_queue STATIC concurrent_queue.h @@ -121,6 +130,7 @@ target_link_libraries(comic_backend PUBLIC pdf_backend_iface cbx_backend epub_page_index + cbds_page_index QsLog ) diff --git a/common/cbds_page_index.cpp b/common/cbds_page_index.cpp new file mode 100644 index 000000000..f57fa5708 --- /dev/null +++ b/common/cbds_page_index.cpp @@ -0,0 +1,74 @@ +#include "cbds_page_index.h" + +#include "qnaturalsorting.h" + +#include + +#include + +namespace { + +const QString bookIniName = QStringLiteral("ComicBookDS_book.ini"); +const QString pagesFolder = QStringLiteral("IMAGE"); + +// Leading separators are ignored, the names are relative to the archive root +// anyway. Directory entries (trailing separator) have no parts so they never match. +QStringList archivePathParts(QString name) +{ + name.replace('\\', '/'); + if (name.endsWith('/')) { + return { }; + } + return name.split('/', Qt::SkipEmptyParts); +} + +bool isBookIni(const QString &fileName) +{ + const QStringList parts = archivePathParts(fileName); + return parts.size() == 1 && parts.first().compare(bookIniName, Qt::CaseInsensitive) == 0; +} + +// Only files directly inside the top level IMAGE folder are pages, nested folders +// and entries trying to leave the archive root never match. +bool isPageEntry(const QString &fileName) +{ + const QStringList parts = archivePathParts(fileName); + return parts.size() == 2 && parts.first().compare(pagesFolder, Qt::CaseInsensitive) == 0 && parts.last() != QStringLiteral("..") && parts.last() != QStringLiteral("."); +} + +} + +bool YACReaderCbds::isCbds(const QString &path, const QStringList &fileNames) +{ + if (QFileInfo(path).suffix().compare(QStringLiteral("cbds"), Qt::CaseInsensitive) == 0) { + return true; + } + + return std::any_of(fileNames.cbegin(), fileNames.cend(), isBookIni); +} + +QStringList YACReaderCbds::pageCandidates(const QString &path, const QStringList &fileNames, const ImageFilter &acceptImage) +{ + if (!isCbds(path, fileNames)) { + return fileNames; + } + + QStringList pages; + for (const QString &fileName : fileNames) { + if (isPageEntry(fileName)) { + pages.append(fileName); + } + } + + // A book without images in IMAGE/ does not follow the known layout, falling + // back to every image in the archive is better than refusing to open it. + const bool hasImages = acceptImage ? std::any_of(pages.cbegin(), pages.cend(), acceptImage) : !pages.isEmpty(); + return hasImages ? pages : fileNames; +} + +void YACReaderCbds::sortPages(QStringList &pages) +{ + std::stable_sort(pages.begin(), pages.end(), [](const QString &left, const QString &right) { + return naturalSortLessThanCI(archivePathParts(left).join('/'), archivePathParts(right).join('/')); + }); +} diff --git a/common/cbds_page_index.h b/common/cbds_page_index.h new file mode 100644 index 000000000..80cdcf6ae --- /dev/null +++ b/common/cbds_page_index.h @@ -0,0 +1,32 @@ +#ifndef CBDS_PAGE_INDEX_H +#define CBDS_PAGE_INDEX_H + +#include +#include + +#include + +// ComicBookDS (.cbds) books are zip archives made for a Nintendo DS reader. +// Besides the pages in IMAGE/ they store several resized copies of every page +// (SMALL_N/, SMALL_R/, THMB_N/, THMB_R/) that must not be shown as pages. +// See docs/CBDS_FORMAT.md for the layout. +namespace YACReaderCbds { + +using ImageFilter = std::function; + +bool isCbds(const QString &path, const QStringList &fileNames); + +// Archive entries that may contain pages: the IMAGE/ entries of a CBDS book, or +// fileNames unchanged for any other archive. A book without any image in IMAGE/ +// (as told by acceptImage) falls back to fileNames too. The names are returned +// exactly as listed by the archive so they can be looked up in fileNames afterwards. +QStringList pageCandidates(const QString &path, const QStringList &fileNames, const ImageFilter &acceptImage = { }); + +// Natural order of the pages of a CBDS book. Page names are plain sequence +// numbers, so there is no spread detection, and the comparison ignores how the +// path is written (separator, case, leading separator) like pageCandidates() does. +void sortPages(QStringList &pages); + +} + +#endif // CBDS_PAGE_INDEX_H diff --git a/common/comic.cpp b/common/comic.cpp index 621d05867..d44d107c4 100644 --- a/common/comic.cpp +++ b/common/comic.cpp @@ -2,6 +2,7 @@ #include "QsLog.h" #include "bookmarks.h" //TODO desacoplar la dependencia con bookmarks +#include "cbds_page_index.h" #include "comic_db.h" #include "compressed_archive.h" #include "epub_page_index.h" @@ -78,7 +79,8 @@ const QStringList ComicArchiveExtensions = QStringList() << "*.cbr" << "*.7z" << "*.cb7" << "*.arj" - << "*.cbt"; + << "*.cbt" + << "*.cbds"; const QStringList LiteralComicArchiveExtensions = QStringList() << "cbr" << "cbz" << "epub" @@ -88,7 +90,8 @@ const QStringList LiteralComicArchiveExtensions = QStringList() << "cbr" << "7z" << "cb7" << "arj" - << "cbt"; + << "cbt" + << "cbds"; #else const QStringList ComicArchiveExtensions = QStringList() << "*.cbr" << "*.cbz" @@ -100,7 +103,8 @@ const QStringList ComicArchiveExtensions = QStringList() << "*.cbr" << "*.7z" << "*.cb7" #endif - << "*.cbt"; + << "*.cbt" + << "*.cbds"; const QStringList LiteralComicArchiveExtensions = QStringList() << "cbr" << "cbz" << "epub" @@ -111,7 +115,8 @@ const QStringList LiteralComicArchiveExtensions = QStringList() << "cbr" << "7z" << "cb7" #endif - << "cbt"; + << "cbt" + << "cbds"; #endif // use_unarr #ifndef NO_PDF const QStringList Comic::comicExtensions = QStringList() << ComicArchiveExtensions << "*.pdf"; @@ -479,6 +484,12 @@ YACReaderEpub::ScanInfo FileComic::epubScanInfo(const QStringList &fileNames, Co return YACReaderEpub::readScanInfo(fileNames, [&archive](int index) { return archive.getRawDataAtIndex(index); }, coverPage, [&supportedExtensions](const QString &fileName) { return isSupportedImage(fileName, supportedExtensions); }); } +QStringList FileComic::cbdsPageCandidates(const QString &path, const QStringList &fileNames) +{ + const QStringList supportedExtensions = Comic::getSupportedImageLiteralFormats(); + return YACReaderCbds::pageCandidates(path, fileNames, [&supportedExtensions](const QString &fileName) { return isSupportedImage(fileName, supportedExtensions); }); +} + // DELEGATE methods void FileComic::fileExtracted(int index, const QByteArray &rawData) { @@ -645,8 +656,12 @@ void FileComic::process() _pageArchiveIndexes.append(page.archiveIndex); } } else { - pageFileNames = filter(archiveFileNames); - comic_pages_sort(pageFileNames, YACReaderHeuristicSorting); + pageFileNames = filter(cbdsPageCandidates(_path, archiveFileNames)); + if (YACReaderCbds::isCbds(_path, archiveFileNames)) { + YACReaderCbds::sortPages(pageFileNames); + } else { + comic_pages_sort(pageFileNames, YACReaderHeuristicSorting); + } for (const QString &fileName : std::as_const(pageFileNames)) { _pageArchiveIndexes.append(archiveFileNames.indexOf(fileName)); } diff --git a/common/comic.h b/common/comic.h index 0f5620023..2c4b7374c 100644 --- a/common/comic.h +++ b/common/comic.h @@ -137,6 +137,7 @@ class FileComic : public Comic, public ExtractDelegate static bool isSupportedImage(const QString &fileName, const QStringList &supportedExtensions); static YACReaderEpub::PageIndex epubPageIndex(const QStringList &fileNames, CompressedArchive &archive); static YACReaderEpub::ScanInfo epubScanInfo(const QStringList &fileNames, CompressedArchive &archive, int coverPage); + static QStringList cbdsPageCandidates(const QString &path, const QStringList &fileNames); // ExtractDelegate void fileExtracted(int index, const QByteArray &rawData); diff --git a/docs/CBDS_FORMAT.md b/docs/CBDS_FORMAT.md new file mode 100644 index 000000000..77f124693 --- /dev/null +++ b/docs/CBDS_FORMAT.md @@ -0,0 +1,113 @@ +# ComicBookDS (`.cbds`) format + +ComicBookDS is a homebrew comic reader for the Nintendo DS (last version 3.0, 2010). Its books +use the `.cbds` extension and are made on a PC with PictoDS (a Java program for Windows, Linux +and macOS; `.cbds` output since 2.0 beta 4 in January 2007, last version 2.1 from August 2007; +earlier versions built a complete `.nds` program per book), with `pycbds` or with the open +source `cbdsconv` script. This document describes what YACReader relies on and where that +knowledge comes from. + +Sources: + +- 25 real `.cbds` books made by PictoDS (all `Version = 200`), inspected entry by entry. + They are not part of the repository. +- The project website (ComicBookDS user guide, PictoDS page and version + history). +- The `cbdsconv` converter (, GPL), which writes the format. +- Descriptions of ComicBookDS and PictoDS on gamebrew.org and fileinfo.com. + +## Container + +A `.cbds` file is a regular ZIP archive. The real books use Deflate with data descriptors +(general purpose flag bit 3), `cbdsconv` uses Python's `ZIP_DEFLATED`. No encryption, no +ZIP64, no comments. Every decompression backend of YACReader opens it by its `PK\x03\x04` +signature, nothing format specific is needed to read the container. + +## Layout + +``` +ComicBookDS_book.ini +IMAGE/1.jpg ... IMAGE/N.jpg pages +NAME/1.txt ... NAME/N.txt original file name of each page +SMALL_N/1.jpg ... SMALL_N/N.jpg preview, normal orientation +SMALL_R/1.jpg ... SMALL_R/N.jpg preview, rotated 90 degrees +THMB_N/1.jpg ... THMB_N/N.jpg thumbnail, normal orientation +THMB_R/1.jpg ... THMB_R/N.jpg thumbnail, rotated 90 degrees +``` + +- Pages are numbered from 1 without leading zeros. Entries are stored in lexicographic order + (`1, 10, 11, ..., 2, 20, ...`), so the archive order is **not** the reading order; natural + sorting of the names is. +- Every page exists in five resolutions. In the real books: `IMAGE` 700 px wide (about 900-1030 + px high), `SMALL_N` 140x192, `SMALL_R` 256x187, `THMB_N` 33x46, `THMB_R` 62x45. `cbdsconv` + targets 700x1054, 127x192, 30x46, 256x169 and 62x40, rotating the image by -90 degrees for + the `_R` variants. +- The extra sizes match the DS screens (256x192) and the modes of the reader: `SMALL_*` is the + whole-page overview shown on the touch screen, `THMB_*` the thumbnails of the library and + thumbnail modes (10 per screen). `_N` and `_R` are the two orientations the reader switches + between (portrait and landscape, R button). +- Only `IMAGE/` holds pages to display. All other folders are derived data of the reader. +- `NAME/N.txt` holds the source file name of page N (Latin-1 text, CRLF). It is not needed to + order the pages. +- All images are JPEG. No tiling or other quality levels were found. + +## `ComicBookDS_book.ini` + +Latin-1 text with CRLF line endings, `;` comments and `key = value` lines, no sections: + +| Key | Meaning | +|---|---| +| `CbCredits1..3` | Credit lines shown by the reader (title, author, provider), often `Unknown` | +| `LeftToRight` | Default reading direction of the book: `1` left to right (comic), `0` right to left (manga); the reader can switch it | +| `NbPages` | Number of pages | +| `Version` | Compatibility version, `200` in all known books | +| `iWidth`, `iHeight`, `iQuality`, `iSize` | Target size and JPEG quality of `IMAGE` | +| `oWidth`, `oHeight`, `oQuality`, `oSize` | Same for the `SMALL_*` previews | +| `thWidth`, `thHeight`, `thQuality`, `thSize` | Same for the `THMB_*` thumbnails | + +In the 21 intact PictoDS books `NbPages` matches the number of `IMAGE/` entries exactly. It is +still not something to rely on in general: `cbdsconv` writes the number of pages plus one and +skips the number of any page it fails to convert, so its books can have gaps in their numbering. + +## How YACReader reads it + +An archive is handled as a ComicBookDS book when its name ends in `.cbds` or when it has a +`ComicBookDS_book.ini` entry at its root. For such a book only the files directly inside the +top level `IMAGE/` folder are page candidates (see `common/cbds_page_index.cpp`). After that it +goes through the same path as any other comic: unsupported files are filtered out by extension +and the cover is the first page. The reader and the library sort the pages the same way, by the +natural (numeric) order of their names written as `IMAGE/`, whatever separator, case or +leading separator the archive uses. The reader's heuristic for spread files such as `1011.jpg` +is not applied, because ComicBookDS page names are plain sequence numbers. + +- The ini file is not read. The page list comes from the archive, which is the only reliable + source, and the optional metadata is not used yet. Broken, huge or missing ini files cannot + affect the result. +- A book without any supported image in `IMAGE/` falls back to the regular behaviour (every + image in the archive is a page). +- The file is never modified or converted. Pages are read from the archive into memory like for + CBZ. +- Only an entry of the form `IMAGE/` is a page. `\` is accepted as a separator and a + leading `/` or `\` is ignored, so names are always relative to the archive root. Nested + folders, `..`, drive letters such as `C:/IMAGE/...` and directory entries never match. The + fallback above returns the archive list unchanged, as for any other comic. +- YACReader does not extract archive entries to disk, so entry names cannot cause path traversal. +- Detecting a book renamed to another extension needs the ini entry in the archive listing. The + unarr backend leaves empty entries out of its listing, so a renamed book with an empty ini is + read as a regular archive there. Files with the `.cbds` extension are not affected. + +## Damaged books + +Some books in the wild are damaged: the file keeps its size, but from some point (in the books +seen, a 64 KiB boundary) to the end the data is zeros, including the ZIP central directory. Such +a file is not a valid ZIP archive and opens with an error, or only partially, depending on how +much the decompression backend can recover from the local headers. This is a data problem of the +individual file, not a format variant. + +## Unknown + +- Books from older ComicBookDS versions (before the `.cbds` extension and before `Version = 200`) + have not been seen. +- Whether other converters wrote different folder names, PNG pages or tiled pages is unknown. +- The use of `LeftToRight = 0` (all known books are left to right) has not been verified with a + real book; mapping it to YACReader's manga mode is a possible follow-up. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 20b3eade0..6c37e0c6f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,8 @@ add_subdirectory(pdf_render_size_test) add_subdirectory(folder_rename_test) add_subdirectory(library_item_actions_test) add_subdirectory(epub_page_index_test) +add_subdirectory(cbds_page_index_test) +add_subdirectory(cbds_comic_test) add_subdirectory(comic_files_manager_test) add_subdirectory(organize_files_test) add_subdirectory(yacreader_libraries_test) diff --git a/tests/cbds_comic_test/CMakeLists.txt b/tests/cbds_comic_test/CMakeLists.txt new file mode 100644 index 000000000..3656c00fd --- /dev/null +++ b/tests/cbds_comic_test/CMakeLists.txt @@ -0,0 +1,39 @@ +# ComicBookDS integration test: opens synthetic .cbds books through the +# configured decompression backend, the reader and the library scanner + +qt_add_executable(cbds_comic_test + main.cpp +) +yacreader_apply_build_options(cbds_comic_test) +target_link_libraries(cbds_comic_test PRIVATE + Qt6::Core + Qt6::Gui + Qt6::Sql + Qt6::Test + comic_backend + library_common +) +if(WIN32) + target_link_libraries(cbds_comic_test PRIVATE oleaut32 ole32) +endif() + +# The 7zip backend loads its library from /utils at runtime on +# Windows and macOS (Linux uses the installed one, the test skips without it) +if(DECOMPRESSION_BACKEND STREQUAL "7zip" AND (WIN32 OR APPLE)) + if(WIN32) + include(BackendHelpers) + yacreader_get_windows_arch_subdir(_cbds_test_7z_arch) + set(_cbds_test_7z_library "${PROJECT_SOURCE_DIR}/dependencies/7zip/win/${_cbds_test_7z_arch}/7z.dll") + else() + set(_cbds_test_7z_library "${PROJECT_SOURCE_DIR}/dependencies/7zip/macx/universal/7z.so") + endif() + get_filename_component(_cbds_test_7z_name "${_cbds_test_7z_library}" NAME) + add_custom_command(TARGET cbds_comic_test POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "$/utils" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_cbds_test_7z_library}" + "$/utils/${_cbds_test_7z_name}" + ) +endif() + +add_test(NAME cbds_comic_test COMMAND cbds_comic_test) diff --git a/tests/cbds_comic_test/main.cpp b/tests/cbds_comic_test/main.cpp new file mode 100644 index 000000000..09de1b4c3 --- /dev/null +++ b/tests/cbds_comic_test/main.cpp @@ -0,0 +1,551 @@ +#include "comic.h" +#include "compressed_archive.h" +#include "initial_comic_info_extractor.h" +#include "library_creator.h" +#include "yacreader_global.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +class CbdsComicTest : public QObject +{ + Q_OBJECT + +private slots: + void initTestCase(); + void readerShowsOnlyFullSizePagesInOrder_data(); + void readerShowsOnlyFullSizePagesInOrder(); + void readerAndLibraryAgreeOnPageNumbers(); + void pageOrderIgnoresPathSpelling(); + void fallsBackWhenImageHasNoPictures(); + void coverAndPageCountForTheLibrary(); + void worksWithoutTheIni(); + void ignoresBrokenAndHugeIni(); + void detectsRenamedBooksByContent(); + void leavesRegularComicsUntouched(); + void reportsCorruptAndEmptyArchives(); + void survivesTruncatedArchives(); + void libraryImportsAndRescansBooks(); + +private: + QTemporaryDir dir; +}; + +namespace { + +constexpr int pageCount = 12; +// Pages are told apart by their red channel, previews are pure green. +constexpr int previewGreen = 255; + +int pageRed(int page) +{ + return page * 20; +} + +quint32 crc32(const QByteArray &data) +{ + static const std::array table = [] { + std::array t { }; + for (quint32 i = 0; i < 256; ++i) { + quint32 c = i; + for (int k = 0; k < 8; ++k) { + c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; + } + t[i] = c; + } + return t; + }(); + + quint32 crc = 0xFFFFFFFFu; + for (const char byte : data) { + crc = table[(crc ^ static_cast(byte)) & 0xFF] ^ (crc >> 8); + } + return crc ^ 0xFFFFFFFFu; +} + +template +void append(QByteArray &out, T value) +{ + const T le = qToLittleEndian(value); + out.append(reinterpret_cast(&le), sizeof(T)); +} + +enum class ZipLayout { + Stored, + // Like the books written by PictoDS: Deflate with the sizes and CRC in a + // data descriptor after each entry (general purpose flag bit 3). + DeflateWithDataDescriptor, +}; + +// Raw Deflate stream: qCompress() adds a 4 byte length, a 2 byte zlib header +// and a 4 byte Adler-32 trailer around it. +QByteArray rawDeflate(const QByteArray &data) +{ + const QByteArray zlib = qCompress(data, 9); + return zlib.mid(6, zlib.size() - 10); +} + +// Minimal zip writer, enough to build test archives without depending on an +// archiving library. +QByteArray zipArchive(const QList> &entries, ZipLayout layout = ZipLayout::Stored) +{ + const bool deflate = layout == ZipLayout::DeflateWithDataDescriptor; + const quint16 flags = deflate ? 0x0008 : 0; + const quint16 method = deflate ? 8 : 0; + + QByteArray archive; + QByteArray centralDirectory; + + for (const auto &[name, data] : entries) { + const QByteArray encodedName = name.toUtf8(); + const quint32 crc = crc32(data); + const QByteArray stored = deflate ? rawDeflate(data) : data; + const auto offset = static_cast(archive.size()); + + append(archive, 0x04034b50); + append(archive, 20); // version needed + append(archive, flags); + append(archive, method); + append(archive, 0); // time + append(archive, 0x21); // date: 1980-01-01 + append(archive, deflate ? 0 : crc); + append(archive, deflate ? 0 : stored.size()); + append(archive, deflate ? 0 : data.size()); + append(archive, encodedName.size()); + append(archive, 0); + archive.append(encodedName); + archive.append(stored); + if (deflate) { + append(archive, 0x08074b50); + append(archive, crc); + append(archive, stored.size()); + append(archive, data.size()); + } + + append(centralDirectory, 0x02014b50); + append(centralDirectory, 20); // version made by + append(centralDirectory, 20); // version needed + append(centralDirectory, flags); + append(centralDirectory, method); + append(centralDirectory, 0); + append(centralDirectory, 0x21); + append(centralDirectory, crc); + append(centralDirectory, stored.size()); + append(centralDirectory, data.size()); + append(centralDirectory, encodedName.size()); + append(centralDirectory, 0); // extra + append(centralDirectory, 0); // comment + append(centralDirectory, 0); // disk + append(centralDirectory, 0); // internal attributes + append(centralDirectory, 0); // external attributes + append(centralDirectory, offset); + centralDirectory.append(encodedName); + } + + const auto centralDirectoryOffset = static_cast(archive.size()); + archive.append(centralDirectory); + append(archive, 0x06054b50); + append(archive, 0); + append(archive, 0); + append(archive, entries.size()); + append(archive, entries.size()); + append(archive, centralDirectory.size()); + append(archive, centralDirectoryOffset); + append(archive, 0); + return archive; +} + +QByteArray image(int red, int green) +{ + QImage img(16, 24, QImage::Format_RGB32); + img.fill(qRgb(red, green, 0)); + QByteArray data; + QBuffer buffer(&data); + buffer.open(QIODevice::WriteOnly); + img.save(&buffer, "PNG"); + return data; +} + +QByteArray bookIni(int pages) +{ + return QStringLiteral("; ComicBookDS ini file\r\n" + "CbCredits1 = Synthetic test book\r\n" + "CbCredits2 = Unknown\r\n" + "CbCredits3 = Unknown\r\n" + "LeftToRight = 1\r\n" + "NbPages = %1\r\n" + "Version = 200\r\n") + .arg(pages) + .toLatin1(); +} + +// Same layout as the books written by PictoDS: the ini, then each folder with +// its files numbered from 1, listed in the lexicographic order of real books. +QList> bookEntries(const QByteArray &ini = bookIni(pageCount)) +{ + QList> entries; + if (!ini.isNull()) { + entries.append({ "ComicBookDS_book.ini", ini }); + } + + QStringList numbers; + for (int page = 1; page <= pageCount; ++page) { + numbers.append(QString::number(page)); + } + numbers.sort(); + + for (const QString &number : std::as_const(numbers)) { + entries.append({ QString("IMAGE/%1.png").arg(number), image(pageRed(number.toInt()), 0) }); + } + for (const QString &number : std::as_const(numbers)) { + entries.append({ QString("NAME/%1.txt").arg(number), QString("original_%1.jpg\r\n").arg(number).toLatin1() }); + } + for (const QString folder : { "SMALL_N", "SMALL_R", "THMB_N", "THMB_R" }) { + for (const QString &number : std::as_const(numbers)) { + entries.append({ QString("%1/%2.png").arg(folder, number), image(0, previewGreen) }); + } + } + return entries; +} + +QString writeFile(const QString &path, const QByteArray &data) +{ + QFile file(path); + if (!file.open(QIODevice::WriteOnly) || file.write(data) != data.size()) { + return { }; + } + return path; +} + +struct OpenResult { + bool failed = false; + QList pages; +}; + +OpenResult openInReader(const QString &path) +{ + FileComic comic; + QSignalSpy errorSpy(&comic, static_cast(&Comic::errorOpening)); + QSignalSpy errorMessageSpy(&comic, static_cast(&Comic::errorOpening)); + QSignalSpy pagesSpy(&comic, static_cast(&Comic::numPages)); + + OpenResult result; + if (!comic.load(path, 0)) { + result.failed = true; + return result; + } + comic.process(); + + result.failed = !errorSpy.isEmpty() || !errorMessageSpy.isEmpty() || pagesSpy.isEmpty(); + if (!result.failed) { + for (int page = 0; page < static_cast(pagesSpy.first().first().toUInt()); ++page) { + result.pages.append(comic.getRawPage(page)); + } + } + return result; +} + +QColor colorOf(const QByteArray &data) +{ + QImage img; + if (!img.loadFromData(data)) { + return { }; + } + return img.pixelColor(img.width() / 2, img.height() / 2); +} + +} + +void CbdsComicTest::initTestCase() +{ + QVERIFY(dir.isValid()); + + const QString probe = writeFile(dir.filePath("probe.cbz"), zipArchive({ { "1.png", image(1, 0) } })); + CompressedArchive archive(probe); + if (!archive.toolsLoaded()) { + QSKIP("The decompression backend library is not deployed next to the test executable"); + } + QVERIFY(archive.isValid()); +} + +void CbdsComicTest::readerShowsOnlyFullSizePagesInOrder_data() +{ + QTest::addColumn("deflate"); + QTest::newRow("stored") << false; + QTest::newRow("deflate with data descriptor") << true; +} + +void CbdsComicTest::readerShowsOnlyFullSizePagesInOrder() +{ + QFETCH(bool, deflate); + const ZipLayout layout = deflate ? ZipLayout::DeflateWithDataDescriptor : ZipLayout::Stored; + const QString path = writeFile(dir.filePath(deflate ? "book-deflate.cbds" : "book-stored.cbds"), zipArchive(bookEntries(), layout)); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount); + for (int page = 0; page < pageCount; ++page) { + const QColor color = colorOf(result.pages.at(page)); + QVERIFY2(color.isValid(), qPrintable(QString("page %1 is not an image").arg(page + 1))); + QCOMPARE(color.green(), 0); + QCOMPARE(color.red(), pageRed(page + 1)); + } +} + +void CbdsComicTest::readerAndLibraryAgreeOnPageNumbers() +{ + // Page numbers are sequence numbers. A name like 1011 must not be taken for a + // spread of pages 10 and 11, the reader has to use the order the library uses. + QList> entries { { "ComicBookDS_book.ini", bookIni(pageCount + 1) } }; + for (int page = 1; page <= pageCount; ++page) { + entries.append({ QString("IMAGE/%1.png").arg(page), image(pageRed(page), 0) }); + } + entries.append({ "IMAGE/1011.png", image(255, 0) }); + const QString path = writeFile(dir.filePath("numbers.cbds"), zipArchive(entries)); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount + 1); + for (int page = 0; page < pageCount; ++page) { + QCOMPARE(colorOf(result.pages.at(page)).red(), pageRed(page + 1)); + } + QCOMPARE(colorOf(result.pages.last()).red(), 255); + + YACReader::InitialComicInfoExtractor secondPage(path, "", 2); + secondPage.extract(); + QCOMPARE(secondPage.getCoverImage().pixelColor(0, 0).red(), colorOf(result.pages.at(1)).red()); +} + +void CbdsComicTest::fallsBackWhenImageHasNoPictures() +{ + const QString path = writeFile(dir.filePath("noimages.cbds"), + zipArchive({ { "ComicBookDS_book.ini", bookIni(2) }, + { "IMAGE/readme.txt", "not a page" }, + { "2.png", image(pageRed(2), 0) }, + { "1.png", image(pageRed(1), 0) } })); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), 2); + QCOMPARE(colorOf(result.pages.first()).red(), pageRed(1)); + + YACReader::InitialComicInfoExtractor extractor(path); + extractor.extract(); + QCOMPARE(extractor.getNumPages(), 2); + QCOMPARE(extractor.getCoverImage().pixelColor(0, 0).red(), pageRed(1)); +} + +void CbdsComicTest::pageOrderIgnoresPathSpelling() +{ + const QString path = writeFile(dir.filePath("spelling.cbds"), + zipArchive({ { "/IMAGE/3.png", image(pageRed(3), 0) }, + { "image\\1.png", image(pageRed(1), 0) }, + { "IMAGE/2.png", image(pageRed(2), 0) } })); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), 3); + for (int page = 0; page < 3; ++page) { + QCOMPARE(colorOf(result.pages.at(page)).red(), pageRed(page + 1)); + } + + YACReader::InitialComicInfoExtractor extractor(path); + extractor.extract(); + QCOMPARE(extractor.getNumPages(), 3); + QCOMPARE(extractor.getCoverImage().pixelColor(0, 0).red(), pageRed(1)); +} + +void CbdsComicTest::coverAndPageCountForTheLibrary() +{ + const QString path = writeFile(dir.filePath("cover.cbds"), zipArchive(bookEntries(), ZipLayout::DeflateWithDataDescriptor)); + + YACReader::InitialComicInfoExtractor extractor(path); + extractor.extract(); + + QVERIFY(extractor.isFileSupported()); + QCOMPARE(extractor.getNumPages(), pageCount); + QVERIFY(extractor.hasValidCover()); + QCOMPARE(extractor.getCoverImage().pixelColor(0, 0).red(), pageRed(1)); + + YACReader::InitialComicInfoExtractor thirdPage(path, "", 3); + thirdPage.extract(); + QCOMPARE(thirdPage.getCoverImage().pixelColor(0, 0).red(), pageRed(3)); +} + +void CbdsComicTest::worksWithoutTheIni() +{ + const QString path = writeFile(dir.filePath("noini.cbds"), zipArchive(bookEntries(QByteArray()))); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount); + QCOMPARE(colorOf(result.pages.first()).red(), pageRed(1)); +} + +void CbdsComicTest::ignoresBrokenAndHugeIni() +{ + QByteArray garbage(8 * 1024 * 1024, '\0'); + for (int i = 0; i < garbage.size(); i += 7) { + garbage[i] = static_cast(i % 251); + } + const QString huge = writeFile(dir.filePath("hugeini.cbds"), zipArchive(bookEntries(garbage))); + const QString wrong = writeFile(dir.filePath("wrongini.cbds"), zipArchive(bookEntries("NbPages = -99999999999999999999\r\n[[[\r\n=\r\nLeftToRight"))); + + for (const QString &path : { huge, wrong }) { + const OpenResult result = openInReader(path); + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount); + } +} + +void CbdsComicTest::detectsRenamedBooksByContent() +{ + const QString path = writeFile(dir.filePath("renamed.cbz"), zipArchive(bookEntries())); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount); +} + +void CbdsComicTest::leavesRegularComicsUntouched() +{ + // Without the ini a .cbz is a regular comic, every image in it is a page. + QList> entries = bookEntries(QByteArray()); + const QString path = writeFile(dir.filePath("regular.cbz"), zipArchive(entries)); + + const OpenResult result = openInReader(path); + + QVERIFY(!result.failed); + QCOMPARE(result.pages.size(), pageCount * 5); + + const QString plain = writeFile(dir.filePath("plain.cbz"), zipArchive({ { "b/02.png", image(2, 0) }, { "a/01.png", image(1, 0) }, { "ComicInfo.xml", "" } })); + const OpenResult plainResult = openInReader(plain); + QVERIFY(!plainResult.failed); + QCOMPARE(plainResult.pages.size(), 2); + QCOMPARE(colorOf(plainResult.pages.first()).red(), 1); +} + +void CbdsComicTest::reportsCorruptAndEmptyArchives() +{ + QByteArray noise(64 * 1024, '\0'); + for (int i = 0; i < noise.size(); ++i) { + noise[i] = static_cast((i * 7919) % 256); + } + const QString corrupt = writeFile(dir.filePath("corrupt.cbds"), noise); + const QString empty = writeFile(dir.filePath("empty.cbds"), zipArchive({ })); + const QString iniOnly = writeFile(dir.filePath("inionly.cbds"), zipArchive({ { "ComicBookDS_book.ini", bookIni(0) } })); + const QString zeroBytes = writeFile(dir.filePath("zero.cbds"), QByteArray()); + + for (const QString &path : { corrupt, empty, iniOnly, zeroBytes }) { + QVERIFY2(openInReader(path).failed, qPrintable(path)); + + YACReader::InitialComicInfoExtractor extractor(path); + extractor.extract(); + QCOMPARE(extractor.getNumPages(), 0); + QVERIFY(!extractor.hasValidCover()); + } +} + +void CbdsComicTest::survivesTruncatedArchives() +{ + // Real damaged books end in zeros where the central directory should be. + QByteArray data = zipArchive(bookEntries()); + const int keep = data.size() / 3; + data = data.left(keep) + QByteArray(data.size() - keep, '\0'); + const QString path = writeFile(dir.filePath("truncated.cbds"), data); + + const OpenResult result = openInReader(path); + + // Backends differ in how much of a broken archive they recover; what matters + // is a controlled result and that only real pages are ever shown. + QVERIFY(result.pages.size() <= pageCount); + for (const QByteArray &page : result.pages) { + if (!page.isEmpty()) { + QCOMPARE(colorOf(page).green(), 0); + } + } + + YACReader::InitialComicInfoExtractor extractor(path); + extractor.extract(); + QVERIFY(extractor.getNumPages() <= pageCount); +} + +void CbdsComicTest::libraryImportsAndRescansBooks() +{ + QTemporaryDir libraryDir; + QVERIFY(libraryDir.isValid()); + QVERIFY(!writeFile(libraryDir.filePath("Synthetic.cbds"), zipArchive(bookEntries())).isEmpty()); + + const QString dataPath = YACReader::LibraryPaths::libraryDataPath(libraryDir.path()); + QSettings settings(dir.filePath("settings.ini"), QSettings::IniFormat); + + const auto scan = [&](bool create) { + LibraryCreator creator(&settings); + if (create) { + creator.createLibrary(libraryDir.path(), dataPath); + } else { + creator.updateLibrary(libraryDir.path(), dataPath); + } + creator.start(); + return creator.wait(60000); + }; + + struct Row { + qulonglong id; + QString fileName; + int numPages; + QString hash; + }; + const auto rows = [&] { + QList result; + const QString connection = QStringLiteral("cbds_comic_test"); + { + QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE", connection); + db.setDatabaseName(YACReader::LibraryPaths::libraryDatabasePath(libraryDir.path())); + if (db.open()) { + QSqlQuery query("SELECT c.id, c.fileName, ci.numPages, ci.hash FROM comic c INNER JOIN comic_info ci ON c.comicInfoId = ci.id", db); + while (query.next()) { + result.append({ query.value(0).toULongLong(), query.value(1).toString(), query.value(2).toInt(), query.value(3).toString() }); + } + } + } + QSqlDatabase::removeDatabase(connection); + return result; + }; + + QVERIFY(scan(true)); + const QList created = rows(); + QCOMPARE(created.size(), 1); + QCOMPARE(created.first().fileName, QString("Synthetic.cbds")); + QCOMPARE(created.first().numPages, pageCount); + + const QImage cover(YACReader::LibraryPaths::coverPathFromLibraryDataPath(dataPath, created.first().hash)); + QVERIFY(!cover.isNull()); + QVERIFY(qAbs(cover.pixelColor(cover.width() / 2, cover.height() / 2).red() - pageRed(1)) <= 4); + + QVERIFY(scan(false)); + const QList updated = rows(); + QCOMPARE(updated.size(), 1); + QCOMPARE(updated.first().id, created.first().id); + QCOMPARE(updated.first().numPages, pageCount); +} + +QTEST_GUILESS_MAIN(CbdsComicTest) + +#include "main.moc" diff --git a/tests/cbds_page_index_test/CMakeLists.txt b/tests/cbds_page_index_test/CMakeLists.txt new file mode 100644 index 000000000..76062a63a --- /dev/null +++ b/tests/cbds_page_index_test/CMakeLists.txt @@ -0,0 +1,12 @@ +# ComicBookDS page index test + +qt_add_executable(cbds_page_index_test + main.cpp +) +yacreader_apply_build_options(cbds_page_index_test) +target_link_libraries(cbds_page_index_test PRIVATE + Qt6::Core + Qt6::Test + cbds_page_index +) +add_test(NAME cbds_page_index_test COMMAND cbds_page_index_test) diff --git a/tests/cbds_page_index_test/main.cpp b/tests/cbds_page_index_test/main.cpp new file mode 100644 index 000000000..9e1354d35 --- /dev/null +++ b/tests/cbds_page_index_test/main.cpp @@ -0,0 +1,143 @@ +#include "cbds_page_index.h" + +#include +#include +#include + +class CbdsPageIndexTest : public QObject +{ + Q_OBJECT + +private slots: + void detectsBooksByIniOrSuffix(); + void keepsOnlyFullSizePages(); + void returnsNamesAsListedByTheArchive(); + void ignoresNestedAndEscapingEntries(); + void fallsBackWhenThereAreNoPages(); + void leavesOtherArchivesUntouched(); + void sortsPagesByNumber(); +}; + +namespace { + +QStringList bookEntries(int pages) +{ + QStringList entries { "ComicBookDS_book.ini" }; + const QStringList folders { "IMAGE", "NAME", "SMALL_N", "SMALL_R", "THMB_N", "THMB_R" }; + for (const QString &folder : folders) { + for (int page = 1; page <= pages; ++page) { + entries.append(QString("%1/%2.%3").arg(folder).arg(page).arg(folder == "NAME" ? "txt" : "jpg")); + } + } + return entries; +} + +} + +void CbdsPageIndexTest::detectsBooksByIniOrSuffix() +{ + QVERIFY(YACReaderCbds::isCbds("book.cbds", { })); + QVERIFY(YACReaderCbds::isCbds("/comics/BOOK.CBDS", { "IMAGE/1.jpg" })); + QVERIFY(YACReaderCbds::isCbds("renamed.cbz", { "IMAGE/1.jpg", "ComicBookDS_book.ini" })); + QVERIFY(YACReaderCbds::isCbds("renamed.zip", { "comicbookds_book.INI" })); + QVERIFY(YACReaderCbds::isCbds("renamed.zip", { "/ComicBookDS_book.ini" })); + + // Only the ini at the archive root marks a book. + QVERIFY(!YACReaderCbds::isCbds("comic.cbz", { "extras/ComicBookDS_book.ini", "IMAGE/1.jpg" })); + QVERIFY(!YACReaderCbds::isCbds("comic.cbz", { "IMAGE/1.jpg", "THMB_N/1.jpg" })); + QVERIFY(!YACReaderCbds::isCbds("comic.cbds.cbz", { "IMAGE/1.jpg" })); + + // Backends that list directories must not turn a folder with that name into a book. + QVERIFY(!YACReaderCbds::isCbds("comic.cbz", { "ComicBookDS_book.ini/", "IMAGE/1.jpg", "cover.jpg" })); + QVERIFY(!YACReaderCbds::isCbds("comic.cbz", { "ComicBookDS_book.ini\\", "IMAGE/1.jpg" })); +} + +void CbdsPageIndexTest::keepsOnlyFullSizePages() +{ + const QStringList entries = bookEntries(12); + + const QStringList pages = YACReaderCbds::pageCandidates("book.cbds", entries); + + QCOMPARE(pages.size(), 12); + for (const QString &page : pages) { + QVERIFY2(page.startsWith("IMAGE/"), qPrintable(page)); + } + QCOMPARE(pages.count("IMAGE/1.jpg"), 1); + QCOMPARE(pages.count("IMAGE/12.jpg"), 1); +} + +void CbdsPageIndexTest::returnsNamesAsListedByTheArchive() +{ + const QStringList entries { "ComicBookDS_book.ini", "image\\2.jpg", "Image/1.jpg", "/IMAGE/3.jpg", "THMB_R\\1.jpg" }; + + const QStringList pages = YACReaderCbds::pageCandidates("book.cbds", entries); + + QCOMPARE(pages, QStringList({ "image\\2.jpg", "Image/1.jpg", "/IMAGE/3.jpg" })); + for (const QString &page : pages) { + QVERIFY(entries.contains(page)); + } +} + +void CbdsPageIndexTest::ignoresNestedAndEscapingEntries() +{ + const QStringList entries { + "ComicBookDS_book.ini", + "IMAGE/1.jpg", + "IMAGE/..", + "IMAGE/.", + "IMAGE/", + "../IMAGE/2.jpg", + "IMAGE/../../outside.jpg", + "IMAGE/sub/3.jpg", + "IMAGE/folder.jpg/", + "other/IMAGE/4.jpg", + "C:/IMAGE/5.jpg", + }; + + QCOMPARE(YACReaderCbds::pageCandidates("book.cbds", entries), QStringList({ "IMAGE/1.jpg" })); +} + +void CbdsPageIndexTest::fallsBackWhenThereAreNoPages() +{ + const QStringList withoutPages { "ComicBookDS_book.ini", "pages/1.jpg", "pages/2.jpg" }; + QCOMPARE(YACReaderCbds::pageCandidates("book.cbds", withoutPages), withoutPages); + + QCOMPARE(YACReaderCbds::pageCandidates("empty.cbds", { }), QStringList()); + + // IMAGE/ entries that are not images do not make a book. + const auto isJpeg = [](const QString &name) { return name.endsWith(".jpg", Qt::CaseInsensitive); }; + const QStringList noImages { "ComicBookDS_book.ini", "IMAGE/readme.txt", "1.jpg", "2.jpg" }; + QCOMPARE(YACReaderCbds::pageCandidates("book.cbds", noImages, isJpeg), noImages); + QCOMPARE(YACReaderCbds::pageCandidates("book.cbds", { "IMAGE/1.jpg", "IMAGE/x.txt", "2.jpg" }, isJpeg), QStringList({ "IMAGE/1.jpg", "IMAGE/x.txt" })); +} + +void CbdsPageIndexTest::leavesOtherArchivesUntouched() +{ + // A regular comic that happens to use the same folder names keeps all its images. + QStringList entries = bookEntries(3); + entries.removeFirst(); + + QCOMPARE(YACReaderCbds::pageCandidates("comic.cbz", entries), entries); + QCOMPARE(YACReaderCbds::pageCandidates("comic.cbr", entries), entries); + QCOMPARE(YACReaderCbds::pageCandidates("comic.cb7", entries), entries); + + const QStringList plain { "001.jpg", "002.jpg", "ComicInfo.xml" }; + QCOMPARE(YACReaderCbds::pageCandidates("comic.cbz", plain), plain); +} + +void CbdsPageIndexTest::sortsPagesByNumber() +{ + // Archive order of real books, plus a number that looks like a spread of 10 and 11. + QStringList pages { "IMAGE/1.jpg", "IMAGE/10.jpg", "IMAGE/1011.jpg", "IMAGE/11.jpg", "IMAGE/2.jpg" }; + YACReaderCbds::sortPages(pages); + QCOMPARE(pages, QStringList({ "IMAGE/1.jpg", "IMAGE/2.jpg", "IMAGE/10.jpg", "IMAGE/11.jpg", "IMAGE/1011.jpg" })); + + // Every spelling pageCandidates() accepts sorts by the page number, the names stay as listed. + QStringList mixed { "/IMAGE/3.jpg", "image\\1.jpg", "IMAGE/2.jpg", "\\Image\\10.jpg" }; + YACReaderCbds::sortPages(mixed); + QCOMPARE(mixed, QStringList({ "image\\1.jpg", "IMAGE/2.jpg", "/IMAGE/3.jpg", "\\Image\\10.jpg" })); +} + +QTEST_GUILESS_MAIN(CbdsPageIndexTest) + +#include "main.moc"