From ee884add8c0af4f15ab39487418c67b44b6e0162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lesimple?= Date: Wed, 24 Apr 2019 13:03:51 +0200 Subject: [PATCH] [Tools] implement hashfile generation on backup Add a configuration option "Full w/ hashfile" to the "verification" option menu, to enable hashfile generation when doing full verification of a backup. When enabled, during full backup verification we save the chunk's SHA256 digest in a hashfile next to the output file we're currently verifying. The performance impact is negligible between "Full verify" and "Full verify w/ hashfile", because we already compute the SHA256 of the chunks when verifying. We save the SHA256 per chunks (4 MB) because due to SE limitations, we can't compute the SHA256 of the whole partition (or rawnand). On the other hand a pure software implementation is way too slow to be bearable, even asm-optimized: between 15 and 90 seconds per 4 MB chunk for crc32/sha1/sha256, depending on the optimizations and the actual algorithm. The output hash file format is as follows: # chunksize: sha256_of_chunk_1 sha256_of_chunk_2 ... sha256_of_chunk_N --- bootloader/config/config.c | 19 +++++----- bootloader/frontend/fe_emmc_tools.c | 59 ++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/bootloader/config/config.c b/bootloader/config/config.c index 668340d..949ba69 100644 --- a/bootloader/config/config.c +++ b/bootloader/config/config.c @@ -438,11 +438,11 @@ void config_verification() gfx_clear_grey(0x1B); gfx_con_setpos(0, 0); - ment_t *ments = (ment_t *)malloc(sizeof(ment_t) * 6); - u32 *vr_values = (u32 *)malloc(sizeof(u32) * 3); - char *vr_text = (char *)malloc(64 * 3); + ment_t *ments = (ment_t *)malloc(sizeof(ment_t) * 7); + u32 *vr_values = (u32 *)malloc(sizeof(u32) * 4); + char *vr_text = (char *)malloc(64 * 4); - for (u32 j = 0; j < 3; j++) + for (u32 j = 0; j < 4; j++) { vr_values[j] = j; ments[j + 2].type = MENT_DATA; @@ -454,11 +454,12 @@ void config_verification() ments[1].type = MENT_CHGLINE; - memcpy(vr_text, " Disable (Fastest - Unsafe)", 28); - memcpy(vr_text + 64, " Sparse (Fast - Safe)", 23); - memcpy(vr_text + 128, " Full (Slow - Safe)", 23); + memcpy(vr_text, " None (Fastest - Unsafe)", 35); + memcpy(vr_text + 64, " Sparse (Fast - Safe)", 33); + memcpy(vr_text + 128, " Full (Slow - Safer)", 34); + memcpy(vr_text + 192, " Full w/ hashfiles (Slow - Safest)", 35); - for (u32 i = 0; i < 3; i++) + for (u32 i = 0; i < 4; i++) { if (h_cfg.verification != i) vr_text[64 * i] = ' '; @@ -467,7 +468,7 @@ void config_verification() ments[2 + i].caption = vr_text + (i * 64); } - memset(&ments[5], 0, sizeof(ment_t)); + memset(&ments[6], 0, sizeof(ment_t)); menu_t menu = {ments, "Backup & Restore verification", 0, 0}; u32 *temp_verification = (u32 *)tui_do_menu(&menu); diff --git a/bootloader/frontend/fe_emmc_tools.c b/bootloader/frontend/fe_emmc_tools.c index f3c77d2..a257c01 100644 --- a/bootloader/frontend/fe_emmc_tools.c +++ b/bootloader/frontend/fe_emmc_tools.c @@ -36,6 +36,9 @@ #define MIXD_BUF_ALIGNED 0xB7000000 #define NUM_SECTORS_PER_ITER 8192 // 4MB Cache. +#define OUT_FILENAME_SZ 80 +#define HASH_FILENAME_SZ (OUT_FILENAME_SZ + 11) // 11 == strlen(".sha256sums") +#define SHA256_SZ 0x20 extern sdmmc_t sd_sdmmc; extern sdmmc_storage_t sd_storage; @@ -49,17 +52,44 @@ extern void emmcsn_path_impl(char *path, char *sub_dir, char *filename, sdmmc_st static int _dump_emmc_verify(sdmmc_storage_t *storage, u32 lba_curr, char *outFilename, emmc_part_t *part) { FIL fp; + FIL hashFp; u8 sparseShouldVerify = 4; u32 btn = 0; u32 prevPct = 200; u32 sdFileSector = 0; int res = 0; + const char hexa[] = "0123456789abcdef"; - u8 hashEm[0x20]; - u8 hashSd[0x20]; + u8 hashEm[SHA256_SZ]; + u8 hashSd[SHA256_SZ]; if (f_open(&fp, outFilename, FA_READ) == FR_OK) { + if (h_cfg.verification == 3) + { + char hashFilename[HASH_FILENAME_SZ]; + strncpy(hashFilename, outFilename, OUT_FILENAME_SZ - 1); + strcat(hashFilename, ".sha256sums"); + + res = f_open(&hashFp, hashFilename, FA_CREATE_ALWAYS | FA_WRITE); + if (res) + { + f_close(&fp); + + gfx_con.fntsz = 16; + EPRINTFARGS("\nHash file could not be opened for write (error %d).\n\nAborting..\n", res); + return 1; + } + + char chunkSizeAscii[10]; + itoa(NUM_SECTORS_PER_ITER * NX_EMMC_BLOCKSIZE, chunkSizeAscii, 10); + chunkSizeAscii[9] = '\0'; + + f_puts("# chunksize: ", &hashFp); + f_puts(chunkSizeAscii, &hashFp); + f_puts("\n", &hashFp); + } + u32 totalSectorsVer = (u32)((u64)f_size(&fp) >> (u64)9); u8 *bufEm = (u8 *)EMMC_BUF_ALIGNED; @@ -76,7 +106,7 @@ static int _dump_emmc_verify(sdmmc_storage_t *storage, u32 lba_curr, char *outFi // Check every time or every 4. // Every 4 protects from fake sd, sector corruption and frequent I/O corruption. // Full provides all that, plus protection from extremely rare I/O corruption. - if ((h_cfg.verification & 2) || !(sparseShouldVerify % 4)) + if ((h_cfg.verification >= 2) || !(sparseShouldVerify % 4)) { if (!sdmmc_storage_read(storage, lba_curr, num, bufEm)) { @@ -109,6 +139,22 @@ static int _dump_emmc_verify(sdmmc_storage_t *storage, u32 lba_curr, char *outFi f_close(&fp); return 1; } + + if (h_cfg.verification == 3) + { + // Transform computed hash to readable hexadecimal + char hashStr[SHA256_SZ * 2 + 1]; + char *hashStrPtr = hashStr; + for (int i = 0; i < SHA256_SZ; i++) + { + *(hashStrPtr++) = hexa[hashSd[i] >> 4]; + *(hashStrPtr++) = hexa[hashSd[i] & 0x0F]; + } + hashStr[SHA256_SZ * 2] = '\0'; + + f_puts(hashStr, &hashFp); + f_puts("\n", &hashFp); + } } pct = (u64)((u64)(lba_curr - part->lba_start) * 100u) / (u64)(part->lba_end - part->lba_start); @@ -132,11 +178,13 @@ static int _dump_emmc_verify(sdmmc_storage_t *storage, u32 lba_curr, char *outFi msleep(1000); f_close(&fp); + f_close(&hashFp); return 0; } } f_close(&fp); + f_close(&hashFp); tui_pbar(0, gfx_con.y, pct, 0xFFCCCCCC, 0xFF555555); @@ -361,6 +409,7 @@ static int _dump_emmc_part(char *sd_path, sdmmc_storage_t *storage, emmc_part_t return 0; } + bytesWritten = 0; totalSize = (u64)((u64)totalSectors << 9); @@ -496,7 +545,7 @@ static void _dump_emmc_selected(emmcPartType_t dumpType) } int i = 0; - char sdPath[80]; + char sdPath[OUT_FILENAME_SZ]; // Create Restore folders, if they do not exist. emmcsn_path_impl(sdPath, "/restore", "", &storage); emmcsn_path_impl(sdPath, "/restore/partitions", "", &storage); @@ -855,7 +904,7 @@ static void _restore_emmc_selected(emmcPartType_t restoreType) } int i = 0; - char sdPath[80]; + char sdPath[OUT_FILENAME_SZ]; timer = get_tmr_s(); if (restoreType & PART_BOOT)