From f29e1fdee46b99703e5990c5cdc53a5d02182aa4 Mon Sep 17 00:00:00 2001 From: derrod Date: Fri, 3 Mar 2023 12:37:02 +0100 Subject: [PATCH] updater: Switch to Zstandard for delta updates zstd is faster, less resource intensive, and produces smaller files than bsdiff + LZMA. Overall, it's a significant improvement for our workflow. --- UI/win-update/updater/CMakeLists.txt | 10 +- UI/win-update/updater/patch.cpp | 203 ++++----------------------- UI/win-update/updater/updater.cpp | 10 +- UI/win-update/updater/updater.hpp | 2 +- 4 files changed, 39 insertions(+), 186 deletions(-) diff --git a/UI/win-update/updater/CMakeLists.txt b/UI/win-update/updater/CMakeLists.txt index cad418b00..457705421 100644 --- a/UI/win-update/updater/CMakeLists.txt +++ b/UI/win-update/updater/CMakeLists.txt @@ -43,13 +43,7 @@ if(MSVC) endif() target_link_libraries( - updater - PRIVATE OBS::blake2 - OBS::lzma - zstd::libzstd_static - ${STATIC_ZLIB_PATH} - comctl32 - shell32 - winhttp) + updater PRIVATE OBS::blake2 zstd::libzstd_static ${STATIC_ZLIB_PATH} comctl32 + shell32 winhttp) set_target_properties(updater PROPERTIES FOLDER "frontend") diff --git a/UI/win-update/updater/patch.cpp b/UI/win-update/updater/patch.cpp index 7c034e152..c2cd70631 100644 --- a/UI/win-update/updater/patch.cpp +++ b/UI/win-update/updater/patch.cpp @@ -19,69 +19,8 @@ #include #include -#ifdef _MSC_VER -#define restrict __restrict -#include -#undef restrict -#else -#include -#endif - using namespace std; -#define MAX_BUF_SIZE 262144 -#define READ_BUF_SIZE 32768 - -/* ------------------------------------------------------------------------ */ - -class LZMAStream { - lzma_stream strm = {}; - bool initialized = false; - -public: - inline ~LZMAStream() - { - if (initialized) { - lzma_end(&strm); - } - } - - inline bool init_decoder() - { - lzma_ret ret = lzma_stream_decoder(&strm, 200 * 1024 * 1024, 0); - initialized = (ret == LZMA_OK); - return initialized; - } - - inline operator lzma_stream *() { return &strm; } - inline bool operator!() const { return !initialized; } - - inline lzma_stream *get() { return &strm; } -}; - -class File { - FILE *f = nullptr; - -public: - inline ~File() - { - if (f) - fclose(f); - } - - inline FILE **operator&() { return &f; } - inline operator FILE *() const { return f; } - inline bool operator!() const { return !f; } -}; - -/* ------------------------------------------------------------------------ */ - -struct bspatch_stream { - void *opaque; - int (*read)(const struct bspatch_stream *stream, void *buffer, - int length); -}; - /* ------------------------------------------------------------------------ */ static int64_t offtin(const uint8_t *buf) @@ -112,112 +51,15 @@ static int64_t offtin(const uint8_t *buf) /* ------------------------------------------------------------------------ */ -static int bspatch(const uint8_t *old, int64_t oldsize, uint8_t *newp, - int64_t newsize, struct bspatch_stream *stream) -{ - uint8_t buf[8]; - int64_t oldpos, newpos; - int64_t ctrl[3]; - int64_t i; - - oldpos = 0; - newpos = 0; - while (newpos < newsize) { - /* Read control data */ - for (i = 0; i <= 2; i++) { - if (stream->read(stream, buf, 8)) - return -1; - ctrl[i] = offtin(buf); - }; - - /* Sanity-check */ - if (newpos + ctrl[0] > newsize) - return -1; - - /* Read diff string */ - if (stream->read(stream, newp + newpos, (int)ctrl[0])) - return -1; - - /* Add old data to diff string */ - for (i = 0; i < ctrl[0]; i++) - if ((oldpos + i >= 0) && (oldpos + i < oldsize)) - newp[newpos + i] += old[oldpos + i]; - - /* Adjust pointers */ - newpos += ctrl[0]; - oldpos += ctrl[0]; - - /* Sanity-check */ - if (newpos + ctrl[1] > newsize) - return -1; - - /* Read extra string */ - if (stream->read(stream, newp + newpos, (int)ctrl[1])) - return -1; - - /* Adjust pointers */ - newpos += ctrl[1]; - oldpos += ctrl[2]; - }; - - return 0; -} - -/* ------------------------------------------------------------------------ */ - -struct patch_data { - HANDLE h; - lzma_stream *strm; - uint8_t buf[READ_BUF_SIZE]; -}; - -static int read_lzma(const struct bspatch_stream *stream, void *buffer, int len) -{ - if (!len) - return 0; - - patch_data *data = (patch_data *)stream->opaque; - HANDLE h = data->h; - lzma_stream *strm = data->strm; - - strm->avail_out = (size_t)len; - strm->next_out = (uint8_t *)buffer; - - for (;;) { - if (strm->avail_in == 0) { - DWORD read_size; - if (!ReadFile(h, data->buf, READ_BUF_SIZE, &read_size, - nullptr)) - return -1; - if (read_size == 0) - return -1; - - strm->avail_in = (size_t)read_size; - strm->next_in = data->buf; - } - - lzma_ret ret = lzma_code(strm, LZMA_RUN); - if (ret == LZMA_STREAM_END) - return 0; - if (ret != LZMA_OK) - return -1; - if (strm->avail_out == 0) - break; - } - - return 0; -} - -int ApplyPatch(const wchar_t *patchFile, const wchar_t *targetFile) +int ApplyPatch(ZSTD_DCtx *zstdCtx, const wchar_t *patchFile, + const wchar_t *targetFile) try { uint8_t header[24]; int64_t newsize; - struct bspatch_stream stream; bool success; WinHandle hPatch; WinHandle hTarget; - LZMAStream strm; /* --------------------------------- * * open patch and file to patch */ @@ -236,9 +78,15 @@ try { * read patch header */ DWORD read; + DWORD patchFileSize; + + patchFileSize = GetFileSize(hPatch, nullptr); + if (patchFileSize == INVALID_FILE_SIZE) + throw int(GetLastError()); + success = !!ReadFile(hPatch, header, sizeof(header), &read, nullptr); if (success && read == sizeof(header)) { - if (memcmp(header, "JIMSLEY/BSDIFF43", 16)) + if (memcmp(header, "BOUF//ZSTD//DICT", 16)) throw int(-4); } else { throw int(GetLastError()); @@ -258,6 +106,23 @@ try { throw int(-1); } + /* --------------------------------- * + * read remainder of patch file */ + + vector patchData; + try { + patchData.resize(patchFileSize - sizeof(header)); + } catch (...) { + throw int(-1); + } + + if (!ReadFile(hPatch, &patchData[0], patchFileSize - sizeof(header), + &read, nullptr)) + throw int(GetLastError()); + + if (read != (patchFileSize - sizeof(header))) + throw int(-1); + /* --------------------------------- * * read old file */ @@ -282,19 +147,11 @@ try { /* --------------------------------- * * patch to new file data */ - if (!strm.init_decoder()) - throw int(-10); + size_t result = ZSTD_decompress_usingDict( + zstdCtx, &newData[0], newData.size(), patchData.data(), + patchData.size(), oldData.data(), oldData.size()); - patch_data data; - data.h = hPatch; - data.strm = strm.get(); - - stream.read = read_lzma; - stream.opaque = &data; - - int ret = bspatch(oldData.data(), oldData.size(), newData.data(), - newData.size(), &stream); - if (ret != 0) + if (result != newsize || ZSTD_isError(result)) throw int(-9); /* --------------------------------- * diff --git a/UI/win-update/updater/updater.cpp b/UI/win-update/updater/updater.cpp index d735b2691..0337b69e6 100644 --- a/UI/win-update/updater/updater.cpp +++ b/UI/win-update/updater/updater.cpp @@ -992,7 +992,7 @@ static bool MoveInUseFileAway(update_t &file) return false; } -static bool UpdateFile(update_t &file) +static bool UpdateFile(ZSTD_DCtx *ctx, update_t &file) { wchar_t oldFileRenamedPath[MAX_PATH]; @@ -1049,7 +1049,7 @@ static bool UpdateFile(update_t &file) retryAfterMovingFile: if (file.patchable) { - error_code = ApplyPatch(file.tempPath.c_str(), + error_code = ApplyPatch(ctx, file.tempPath.c_str(), file.outputPath.c_str()); installed_ok = (error_code == 0); @@ -1099,9 +1099,10 @@ static bool UpdateFile(update_t &file) L"programs and try again.", curFileName); } else { + DWORD err = GetLastError(); Status(L"Update failed: Couldn't update %s " L"(error %d)", - curFileName, GetLastError()); + curFileName, err ? err : error_code); } file.state = STATE_INSTALL_FAILED; @@ -1147,6 +1148,7 @@ static bool updateThreadFailed = false; static bool UpdateWorker() { unique_lock ulock(updateMutex, defer_lock); + ZSTDDCtx zCtx; while (true) { ulock.lock(); @@ -1160,7 +1162,7 @@ static bool UpdateWorker() updateQueue.pop(); ulock.unlock(); - if (!UpdateFile(update)) { + if (!UpdateFile(zCtx, update)) { updateThreadFailed = true; return false; } else { diff --git a/UI/win-update/updater/updater.hpp b/UI/win-update/updater/updater.hpp index 5458d6dd0..694b1aaea 100644 --- a/UI/win-update/updater/updater.hpp +++ b/UI/win-update/updater/updater.hpp @@ -91,7 +91,7 @@ void StringToHash(const wchar_t *in, BYTE *out); bool CalculateFileHash(const wchar_t *path, BYTE *hash); -int ApplyPatch(LPCTSTR patchFile, LPCTSTR targetFile); +int ApplyPatch(ZSTD_DCtx *ctx, LPCTSTR patchFile, LPCTSTR targetFile); int DecompressFile(ZSTD_DCtx *ctx, LPCTSTR tempFile, size_t newSize); extern HWND hwndMain;