Skip to content

Commit

Permalink
Fix warnings from recent overflow checks
Browse files Browse the repository at this point in the history 
see #832
  • Loading branch information
@detonin
detonin committed Sep 16, 2016
1 parent d8ae42d commit 2596634
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 24 deletions.
23 changes: 13 additions & 10 deletions src/lib/openjp2/dwt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void

OPJ_INT32 rw; /* width of the resolution level computed */
OPJ_INT32 rh; /* height of the resolution level computed */
OPJ_UINT32 l_data_size;
size_t l_data_size;

opj_tcd_resolution_t * l_cur_res = 0;
opj_tcd_resolution_t * l_last_res = 0;
Expand All @@ -410,12 +410,13 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions);

/* overflow check */
if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}

bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32));
l_data_size *= sizeof(OPJ_INT32);
bj = (OPJ_INT32*)opj_malloc(l_data_size);
/* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
/* in that case, so do not error out */
if (l_data_size != 0 && ! bj) {
Expand Down Expand Up @@ -578,21 +579,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres

OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);

OPJ_UINT32 mr; /* max resolution */
size_t mr; /* max resolution */

if (numres == 1U) {
return OPJ_TRUE;
}

mr = opj_dwt_max_resolution(tr, numres);
mr = opj_dwt_max_resolution(tr, numres);

/* overflow check */
if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
if (mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}

h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32));
mr *= sizeof(OPJ_INT32);
h.mem = (OPJ_INT32*)opj_aligned_malloc(mr);
if (! h.mem){
/* FIXME event manager error callback */
return OPJ_FALSE;
Expand Down Expand Up @@ -865,7 +867,7 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32

OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);

OPJ_UINT32 mr; /* max resolution */
size_t mr; /* max resolution */

mr = opj_dwt_max_resolution(res, numres);

Expand All @@ -876,12 +878,13 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
}
mr += 5U;

if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) {
if (mr > (SIZE_MAX / sizeof(opj_v4_t))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}

h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t));
mr *= sizeof(opj_v4_t);
h.wavelet = (opj_v4_t*) opj_aligned_malloc(mr);
if (!h.wavelet) {
/* FIXME event manager error callback */
return OPJ_FALSE;
Expand Down
28 changes: 14 additions & 14 deletions src/lib/openjp2/t1.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1166,37 +1166,37 @@ static OPJ_BOOL opj_t1_allocate_buffers(
OPJ_UINT32 w,
OPJ_UINT32 h)
{
OPJ_UINT32 flagssize;
size_t flagssize;

/* encoder uses tile buffer, so no need to allocate */
if (!t1->encoder) {
OPJ_UINT32 datasize;
size_t datasize;

/* Overflow check */
if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
datasize = w * h;
datasize = (size_t) w * h;

/* Overflow check */
if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}

if(datasize > t1->datasize){
if(datasize > (size_t) t1->datasize){
opj_aligned_free(t1->data);
t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32));
t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32));
if(!t1->data){
/* FIXME event manager error callback */
return OPJ_FALSE;
}
t1->datasize = datasize;
t1->datasize = (OPJ_UINT32) datasize;
}
/* memset first arg is declared to never be null by gcc */
if (t1->data != NULL) {
memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32));
memset(t1->data, 0, datasize * sizeof(OPJ_INT32));
}
}

Expand All @@ -1210,24 +1210,24 @@ static OPJ_BOOL opj_t1_allocate_buffers(
/* FIXME event manager error callback */
return OPJ_FALSE;
}
flagssize = t1->flags_stride * (h + 2U);
flagssize = (size_t) t1->flags_stride * (h + 2U);

/* Overflow check */
if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}

if(flagssize > t1->flagssize){
if(flagssize > (size_t) t1->flagssize){
opj_aligned_free(t1->flags);
t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t));
t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
if(!t1->flags){
/* FIXME event manager error callback */
return OPJ_FALSE;
}
t1->flagssize = flagssize;
t1->flagssize = (OPJ_UINT32) flagssize;
}
memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t));
memset(t1->flags, 0, flagssize * sizeof(opj_flag_t));

t1->w = w;
t1->h = h;
Expand Down

0 comments on commit 2596634

Please sign in to comment.