超大文件如何计算md5

超大文件如何计算md5

原理
对MD5算法简要的叙述可以为：MD5以512位分组来处理输入的信息，且每一分组又被划分为16个32位子分组，经过了一系列的处理后，算法的输出由四个32位分组组成，将这四个32位分组级联后将生成一个128位散列值。

根据于原理，可以分组计算，不影响最后的zhi

#include <stdio.h> #include <stdlib.h> #include <memory.h> #include <math.h> typedef unsigned char byte; typedef unsigned int uint; typedef uint md5_sub_array[16]; typedef uint md5_transorm_func(uint,uint,uint); typedef struct { uint a; uint b; uint c; uint d; md5_sub_array sub_array; }md5_transform_param; const double max_int = (double)0xffffffff + 1.0; const uint md5_transform_matrix[4][16][3] = { { { 0, 7, 1}, { 1,12, 2}, { 2,17, 3}, { 3,22, 4}, { 4, 7, 5}, { 5,12, 6}, { 6,17, 7}, { 7,22, 8}, { 8, 7, 9}, { 9,12,10}, {10,17,11}, {11,22,12}, {12, 7,13}, {13,12,14}, {14,17,15}, {15,22,16}, }, { { 1, 5,17}, { 6, 9,18}, {11,14,19}, { 0,20,20}, { 5, 5,21}, {10, 9,22}, {15,14,23}, { 4,20,24}, { 9, 5,25}, {14, 9,26}, { 3,14,27}, { 8,20,28}, {13, 5,29}, { 2, 9,30}, { 7,14,31}, {12,20,32}, }, { {5, 4, 33}, { 8,11,34}, {11,16,35},{14, 23,36}, {1, 4, 37}, { 4,11,38}, { 7,16,39},{10, 23,40}, {13,4, 41}, { 0,11,42}, { 3,16,43},{ 6, 23,44}, {9, 4, 45}, {12,11,46}, {15,16,47},{ 2, 23,48}, }, { { 0,6,49}, { 7,10,50}, {14,15,51},{ 5, 21,52}, {12,6,53}, { 3,10,54}, {10,15,55},{ 1, 21,56}, { 8,6,57}, {15,10,58}, { 6,15,59},{13, 21,60}, { 4,6,61}, {11,10,62}, { 2,15,63},{ 9, 21,64}, }, }; static uint md5_transform_array[65]; void md5_init() { int x; for(x = 1; x <= 64; x++) { md5_transform_array[x] = (uint)(max_int * fabs(sin(x))); } } uint f(uint x,uint y,uint z) { return ((x & y) | ((~x) & z)); } uint g(uint x,uint y,uint z) { return ((x & z) | (y & (~z))); } uint h(uint x,uint y,uint z) { return (x ^ y ^ z); } uint i(uint x,uint y,uint z) { return (y ^ (x | (~z))); } byte* md5_prepare_data(const byte* data,int len,int* new_len) { int rest,fill,size; byte* new_data; uint bit_len; // (1) 字节补齐 rest = len % 56; if (rest <= 56) fill = 56 - rest; else fill = (64 - rest) + 56; new_data = (byte*)malloc(len + fill + 8); if (null == new_data) return null; if (len > 0) memcpy(new_data,data,len); if (fill > 0) memset(new_data + len,0x80,1); if (fill > 1) memset(new_data + len + 1,0,fill - 1); size = fill + len; // (2) 附加数据的比特长度 bit_len = len * 8; // (64位二进制数表示的)比特长度的低32位 memset(new_data + size + 0,(bit_len & 0x000000ff), 1); memset(new_data + size + 1,(bit_len & 0x0000ff00) >> 8, 1); memset(new_data + size + 2,(bit_len & 0x00ff0000) >> 16,1); memset(new_data + size + 3,(bit_len & 0xff000000) >> 24,1); // 不考虑比特长度超出32位无符号数表示范围,所以高32位总是0 memset(new_data + size + 4,0,4); *new_len = size + 8; return new_data; } void md5_transform(md5_transform_param* param,int ring,md5_transorm_func func) { uint a,b,c,d,s,k,i; uint abcd[4]; uint *x,*t; int index; abcd[0] = param->a; abcd[1] = param->b; abcd[2] = param->c; abcd[3] = param->d; x = param->sub_array; t = md5_transform_array; for(index = 0; index < 16; index++) { a = abcd[(3 * index + 0) % 4]; b = abcd[(3 * index + 1) % 4]; c = abcd[(3 * index + 2) % 4]; d = abcd[(3 * index + 3) % 4]; k = md5_transform_matrix[ring][index][0]; s = md5_transform_matrix[ring][index][1]; i = md5_transform_matrix[ring][index][2]; a = a + func(b,c,d) + x[k] + t[i]; a = ( a << s) | ( a >> (32 - s)); // 循环左移 a = a + b; abcd[(3 * index + 0) % 4] = a; } param->a = abcd[0]; param->b = abcd[1]; param->c = abcd[2]; param->d = abcd[3]; } int md5(const byte* data,int len) { int x,y,new_len; md5_transform_param param; uint aa,bb,cc,dd; byte* buf; md5_init(); buf = md5_prepare_data(data,len,&new_len); if (buf == null) return -1; aa = 0x67452301; bb = 0xefcdab89; cc = 0x98badcfe; dd = 0x10325476; for(x = 0; x < new_len / 64; x++) { param.a = aa; param.b = bb; param.c = cc; param.d = dd; for(y = 0; y < 16; y++) { param.sub_array[y] = buf[64 * x + 4 * y + 0]; param.sub_array[y] += buf[64 * x + 4 * y + 1] << 8; param.sub_array[y] += buf[64 * x + 4 * y + 2] << 16; param.sub_array[y] += buf[64 * x + 4 * y + 3] << 24; } md5_transform(&param,0,f); md5_transform(&param,1,g); md5_transform(&param,2,h); md5_transform(&param,3,i); aa += param.a; bb += param.b; cc += param.c; dd += param.d; } printf("md5(\"%s\")=",data); printf("%02x%02x%02x%02x", (aa & 0x000000ff), (aa & 0x0000ff00) >> 8, (aa & 0x00ff0000) >> 16, (aa & 0xff000000) >> 24); printf("%02x%02x%02x%02x", (bb & 0x000000ff), (bb & 0x0000ff00) >> 8, (bb & 0x00ff0000) >> 16, (bb & 0xff000000) >> 24); printf("%02x%02x%02x%02x", (cc & 0x000000ff), (cc & 0x0000ff00) >> 8, (cc & 0x00ff0000) >> 16, (cc & 0xff000000) >> 24); printf("%02x%02x%02x%02x", (dd & 0x000000ff), (dd & 0x0000ff00) >> 8, (dd & 0x00ff0000) >> 16, (dd & 0xff000000) >> 24); printf("\n"); return 0; } int main() { md5("",0); md5("a",1); md5("abc",3); md5("message digest",14); md5("abcdefghijklmnopqrstuvwxyz",26); return 0; }