Skip to content

Commit

Permalink
fix(coder): incorrect hidden bits decoding in 32bit lib
Browse files Browse the repository at this point in the history
  • Loading branch information
@fumiama
fumiama committed Apr 5, 2024
1 parent cd15e63 commit 29e4019
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 38 deletions.
18 changes: 13 additions & 5 deletions base1432.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint32_t* vals = (uint32_t*)data;
const uint32_t* vals = (const uint32_t*)data;
uint32_t n = 0;
int32_t i = 0;
for(; i < outlen - 7; i+=7) { // n实际每次自增2
Expand Down Expand Up @@ -293,7 +293,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
sum |= shift & 0x003fff00;
valbuf.val = be32toh(sum);
memcpy(buf+i+4, valbuf.buf, 3);
} else if(offset--) {
} else if((*(uint8_t*)(&vals[n]) != '=') && offset--) {
int cnt = dlen-2-(int)n*(int)sizeof(uint32_t);
if (cnt > 4) cnt = 4;
memcpy(valbuf.buf, &vals[n], cnt);
Expand All @@ -312,6 +312,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
buf[i++] = ((sum & 0x000f0000) >> 12) | ((sum & 0xf0000000) >> 28);
if(offset--) {
buf[i] = (sum & 0x0f000000) >> 20;
if(*(uint8_t*)(&vals[n]) == '=') return outlen;
memcpy(valbuf.buf, &vals[n], dlen-2-(int)n*(int)sizeof(uint32_t));
#ifdef WORDS_BIGENDIAN
sum = __builtin_bswap32(valbuf.val);
Expand Down Expand Up @@ -351,7 +352,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint32_t* vals = (uint32_t*)data;
const uint32_t* vals = (const uint32_t*)data;
uint32_t n = 0;
int32_t i = 0;
for(; i <= outlen - 7; i+=7) { // n实际每次自增2
Expand All @@ -371,6 +372,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
sum |= shift & 0x003fff00;
*(uint32_t*)(buf+i+4) = be32toh(sum);
}
if(*(uint8_t*)(&vals[n]) == '=') return outlen;
if(offset--) {
// 这里有读取越界
#ifdef WORDS_BIGENDIAN
Expand All @@ -387,6 +389,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
buf[i++] = ((sum & 0x000f0000) >> 12) | ((sum & 0xf0000000) >> 28);
if(offset--) {
buf[i] = (sum & 0x0f000000) >> 20;
if(*(uint8_t*)(&vals[n]) == '=') return outlen;
// 这里有读取越界
#ifdef WORDS_BIGENDIAN
sum = __builtin_bswap32(vals[n]);
Expand Down Expand Up @@ -426,7 +429,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint32_t* vals = (uint32_t*)data;
const uint32_t* vals = (const uint32_t*)data;
uint32_t n = 0;
int32_t i = 0;
for(; i < outlen-7; i+=7) { // n实际每次自增2
Expand All @@ -448,14 +451,19 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
}
register uint32_t sum = 0;
register uint32_t shift = htobe32(vals[n++]);
if(((shift>>24)&0xff) == 0x3d) return outlen;
if(((shift>>24)&0xff) < 0x4e) shift |= 0xff000000;
if(((shift>> 8)&0xff) < 0x4e) shift |= 0x0000ff00;
shift -= 0x4e004e00;
shift <<= 2;
sum |= shift & 0xfffc0000;
shift <<= 2;
sum |= shift & 0x0003fff0;
shift = htobe32(vals[n++]);
shift = htobe32(vals[n]);
if(((shift>>24)&0xff) == 0x3d) {
*(uint32_t*)(buf+i) = be32toh(sum);
return outlen;
}
if(((shift>>24)&0xff) < 0x4e) shift |= 0xff000000;
if(((shift>> 8)&0xff) < 0x4e) shift |= 0x0000ff00;
shift -= 0x4e004e00;
Expand Down
10 changes: 6 additions & 4 deletions base1464.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint64_t* vals = (uint64_t*)data;
const uint64_t* vals = (const uint64_t*)data;
uint64_t n = 0;
int64_t i = 0;
for(; i < outlen - 7; n++, i+=7) {
Expand Down Expand Up @@ -256,7 +256,7 @@ int base16384_decode_safe(const char* data, int dlen, char* buf) {
sum |= shift & 0x00000000003fff00;
valbuf.val = be64toh(sum);
memcpy(buf+i, valbuf.buf, 7);
} else if(offset--) {
} else if((*(uint8_t*)(&vals[n]) != '=') && offset--) {
memcpy(valbuf.buf, &vals[n], dlen-2-(int)n*(int)sizeof(uint64_t));
#ifdef WORDS_BIGENDIAN
register uint64_t sum = __builtin_bswap64(valbuf.val) - 0x000000000000004e;
Expand Down Expand Up @@ -303,7 +303,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint64_t* vals = (uint64_t*)data;
const uint64_t* vals = (const uint64_t*)data;
uint64_t n = 0;
int64_t i = 0;
for(; i <= outlen - 7; n++, i+=7) {
Expand All @@ -319,6 +319,7 @@ int base16384_decode(const char* data, int dlen, char* buf) {
sum |= shift & 0x00000000003fff00;
*(uint64_t*)(buf+i) = be64toh(sum);
}
if(*(uint8_t*)(&vals[n]) == '=') return outlen;
if(offset--) {
// 这里有读取越界
#ifdef WORDS_BIGENDIAN
Expand Down Expand Up @@ -366,7 +367,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
}
}
outlen = outlen / 8 * 7 + offset;
uint64_t* vals = (uint64_t*)data;
const uint64_t* vals = (const uint64_t*)data;
uint64_t n = 0;
int64_t i = 0;
for(; i < outlen-7; n++, i+=7) {
Expand All @@ -384,6 +385,7 @@ int base16384_decode_unsafe(const char* data, int dlen, char* buf) {
}
register uint64_t sum = 0;
register uint64_t shift = htobe64(vals[n]);
if(((shift>>56)&0xff) == 0x3d) return outlen;
if(((shift>>56)&0xff) < 0x4e) shift |= 0xff00000000000000;
if(((shift>>40)&0xff) < 0x4e) shift |= 0x0000ff0000000000;
if(((shift>>24)&0xff) < 0x4e) shift |= 0x00000000ff000000;
Expand Down
6 changes: 2 additions & 4 deletions base16384.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,10 @@ typedef enum base16384_err_t base16384_err_t;

// disable 0xFEFF file header in encode
#define BASE16384_FLAG_NOHEADER (1<<0)
// enable sum check when using stdin or inputsize > _BASE16384_ENCBUFSZ
// enable sum check when using stdin or stdout or inputsize > _BASE16384_ENCBUFSZ
#define BASE16384_FLAG_SUM_CHECK_ON_REMAIN (1<<1)
// forcely do sumcheck without checking decoded data length
// forcely do sumcheck without checking data length
#define BASE16384_FLAG_DO_SUM_CHECK_FORCELY (1<<2)
// initial sum value used in BASE16384_FLAG_SUM_CHECK_ON_REMAIN
#define BASE16384_SIMPLE_SUM_INIT_VALUE (0x8e29c213)

/**
* @brief calculate the exact encoded size
Expand Down
36 changes: 20 additions & 16 deletions binary.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,15 @@
#define htobe64(x) (x)
#endif
#else
#define be16toh(x) _byteswap_ushort(x)
#define be32toh(x) _byteswap_ulong(x)
#define be16toh(x) _bitswap_ushort(x)
#define be32toh(x) _bitswap_ulong(x)
#ifdef IS_64BIT_PROCESSOR
#define be64toh(x) _byteswap_uint64(x)
#define be64toh(x) _bitswap_uint64(x)
#endif
#define htobe16(x) _byteswap_ushort(x)
#define htobe32(x) _byteswap_ulong(x)
#define htobe16(x) _bitswap_ushort(x)
#define htobe32(x) _bitswap_ulong(x)
#ifdef IS_64BIT_PROCESSOR
#define htobe64(x) _byteswap_uint64(x)
#define htobe64(x) _bitswap_uint64(x)
#endif
#endif
#endif
Expand All @@ -87,6 +87,9 @@
// leftrotate function definition
#define LEFTROTATE(x, c) (((x) << (c)) | ((x) >> (sizeof(x)*8 - (c))))

// initial sum value used in BASE16384_FLAG_SUM_CHECK_ON_REMAIN
#define BASE16384_SIMPLE_SUM_INIT_VALUE (0x8e29c213)

static inline uint32_t calc_sum(uint32_t sum, size_t cnt, const char* encbuf) {
size_t i;
uint32_t buf;
Expand All @@ -99,20 +102,21 @@ static inline uint32_t calc_sum(uint32_t sum, size_t cnt, const char* encbuf) {
return sum;
}

static inline uint32_t calc_and_embed_sum(uint32_t sum, size_t cnt, char* encbuf) {
sum = calc_sum(sum, cnt, encbuf);
if(cnt%7) { // last encode
*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
}
return sum;
}

static inline int check_sum(uint32_t sum, uint32_t sum_read_raw, int offset) {
int shift = (int[]){0, 26, 20, 28, 22, 30, 24}[offset%7];
offset = offset%7;
if(!offset--) return 0; // no remain bits, pass
// offset 1: 0011 1111 1100 0000 remain: 3*2 bits
// offset 2: 0011 1111 1111 1111 0011 0000 0000 0000 remain: 6*2 bits
// offset 3: 0011 1111 1111 0000 remain: 2*2 bits
// offset 4: 0011 1111 1111 1111 0011 1100 0000 0000 remain: 5*2 bits
// offset 5: 0011 1111 1111 1100 remain: 1*2 bits
// offset 6: 0011 1111 1111 1111 0011 1111 0000 0000 remain: 4*2 bits
// encode: 0415263 (6-1) per 3bits, thus 0x021ab3
int shift = sizeof(uint32_t)*8 - ((0x021ab3>>(offset*3))&0x07)*2;
uint32_t sum_read = be32toh(sum_read_raw) >> shift;
sum >>= shift;
#ifdef DEBUG
fprintf(stderr, "offset: %d, mysum: %08x, sumrd: %08x\n", offset, sum, sum_read);
fprintf(stderr, "shift: %d, offset: %d, mysum: %08x, sumrd: %08x\n", shift, offset+1, sum, sum_read);
#endif
return sum != sum_read;
}
Expand Down
38 changes: 29 additions & 9 deletions file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ static inline off_t get_file_size(const char* filepath) {
goto base16384_##method##_file_detailed_cleanup; \
}

#define do_sum_check(flag) ((flag)&(BASE16384_FLAG_DO_SUM_CHECK_FORCELY|BASE16384_FLAG_SUM_CHECK_ON_REMAIN))

base16384_err_t base16384_encode_file_detailed(const char* input, const char* output, char* encbuf, char* decbuf, int flag) {
off_t inputsize;
FILE *fp = NULL, *fpo;
Expand Down Expand Up @@ -98,7 +100,15 @@ base16384_err_t base16384_encode_file_detailed(const char* input, const char* ou
if(n > 0) cnt++;
else break;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
if(do_sum_check(flag)) {
sum = calc_sum(sum, cnt, encbuf);
if(cnt%7) { // last encode
*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
#ifdef DEBUG
fprintf(stderr, "writesum: %08x\n", sum);
#endif
}
}
n = base16384_encode_unsafe(encbuf, cnt, decbuf);
if(n && fwrite(decbuf, n, 1, fpo) <= 0) {
goto_base16384_file_detailed_cleanup(encode, base16384_err_write_file, {});
Expand Down Expand Up @@ -157,7 +167,12 @@ base16384_err_t base16384_encode_fp_detailed(FILE* input, FILE* output, char* en
if(n > 0) cnt++;
else break;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
if(do_sum_check(flag)) {
sum = calc_sum(sum, cnt, encbuf);
if(cnt%7) { // last encode
*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
}
}
n = base16384_encode_unsafe(encbuf, cnt, decbuf);
if(n && fwrite(decbuf, n, 1, output) <= 0) {
return base16384_err_write_file;
Expand All @@ -184,7 +199,12 @@ base16384_err_t base16384_encode_fd_detailed(int input, int output, char* encbuf
if(n > 0) cnt++;
else break;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_and_embed_sum(sum, cnt, encbuf);
if(do_sum_check(flag)) {
sum = calc_sum(sum, cnt, encbuf);
if(cnt%7) { // last encode
*(uint32_t*)(&encbuf[cnt]) = htobe32(sum);
}
}
n = base16384_encode_unsafe(encbuf, cnt, decbuf);
if(n && write(output, decbuf, n) < n) {
return base16384_err_write_file;
Expand Down Expand Up @@ -269,10 +289,10 @@ base16384_err_t base16384_decode_file_detailed(const char* input, const char* ou
goto_base16384_file_detailed_cleanup(decode, base16384_err_write_file, {});
}
total_decoded_len += cnt;
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, cnt, encbuf);
if(do_sum_check(flag)) sum = calc_sum(sum, cnt, encbuf);
last_encbuf_cnt = cnt;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
if(do_sum_check(flag)
&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
&& last_decbuf_cnt > 2
&& decbuf[last_decbuf_cnt-2] == '='
Expand Down Expand Up @@ -346,10 +366,10 @@ base16384_err_t base16384_decode_fp_detailed(FILE* input, FILE* output, char* en
return base16384_err_write_file;
}
total_decoded_len += cnt;
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, cnt, encbuf);
if(do_sum_check(flag)) sum = calc_sum(sum, cnt, encbuf);
last_encbuf_cnt = cnt;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
if(do_sum_check(flag)
&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
&& last_decbuf_cnt > 2
&& decbuf[last_decbuf_cnt-2] == '='
Expand Down Expand Up @@ -425,10 +445,10 @@ base16384_err_t base16384_decode_fd_detailed(int input, int output, char* encbuf
return base16384_err_write_file;
}
total_decoded_len += n;
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN) sum = calc_sum(sum, n, encbuf);
if(do_sum_check(flag)) sum = calc_sum(sum, n, encbuf);
last_encbuf_cnt = n;
}
if(flag&BASE16384_FLAG_SUM_CHECK_ON_REMAIN
if(do_sum_check(flag)
&& (flag&BASE16384_FLAG_DO_SUM_CHECK_FORCELY || total_decoded_len >= _BASE16384_ENCBUFSZ)
&& last_decbuf_cnt > 2
&& decbuf[last_decbuf_cnt-2] == '='
Expand Down

0 comments on commit 29e4019

Please sign in to comment.