fixing overrelease and out-of-bounds write, fixes #13725

[wxWidgets.git] / src / common / quantize.cpp

diff --git a/src/common/quantize.cpp b/src/common/quantize.cpp
index b792d1483389a2f07780de9d01ed1da419c9b066..cbd53cc46838d09712fa128776cccb38ff4409fd 100644 (file)
--- a/src/common/quantize.cpp
+++ b/src/common/quantize.cpp
@@ -1,5 +1,5 @@
 /////////////////////////////////////////////////////////////////////////////
-// Name:        quantize.cpp
+// Name:        src/common/quantize.cpp
 // Purpose:     wxQuantize implementation
 // Author:      Julian Smart
 // Modified by:
@@ -30,10 +30,6 @@
 
 /* modified by Vaclav Slavik for use as jpeglib-independent module */
 
-#if defined(__GNUG__) && !defined(NO_GCC_PRAGMA)
-#pragma implementation "quantize.h"
-#endif
-
 // For compilers that support precompilation, includes "wx/wx.h".
 #include "wx/wxprec.h"
 
@@ -41,15 +37,17 @@
 #pragma hdrstop
 #endif
 
+#if wxUSE_IMAGE
+
+#include "wx/quantize.h"
+
 #ifndef WX_PRECOMP
     #include "wx/palette.h"
+    #include "wx/image.h"
 #endif
 
-#include "wx/image.h"
-#include "wx/quantize.h"
-
 #ifdef __WXMSW__
-#include "wx/msw/private.h"
+    #include "wx/msw/private.h"
 #endif
 
 #include <stdlib.h>
@@ -75,7 +73,9 @@
 
 typedef unsigned short UINT16;
 typedef signed short INT16;
+#if !(defined(__WATCOMC__) && (defined(__WXMSW__) || defined(__WXMOTIF__)))
 typedef signed int INT32;
+#endif
 
 typedef unsigned char JSAMPLE;
 typedef JSAMPLE *JSAMPROW;
@@ -1114,13 +1114,13 @@ pass2_fs_dither (j_decompress_ptr cinfo,
       dir = -1;
       dir3 = -3;
       errorptr = cquantize->fserrors + (width+1)*3; /* => entry after last column */
-      cquantize->on_odd_row = FALSE; /* flip for next time */
+      cquantize->on_odd_row = false; /* flip for next time */
     } else {
       /* work left to right in this row */
       dir = 1;
       dir3 = 3;
       errorptr = cquantize->fserrors; /* => entry before first real column */
-      cquantize->on_odd_row = TRUE; /* flip for next time */
+      cquantize->on_odd_row = true; /* flip for next time */
     }
     /* Preset error values: no error propagated to first pixel from left */
     cur0 = cur1 = cur2 = 0;
@@ -1280,7 +1280,7 @@ finish_pass1 (j_decompress_ptr cinfo)
   cinfo->colormap = cquantize->sv_colormap;
   select_colors(cinfo, cquantize->desired);
   /* Force next pass to zero the color index table */
-  cquantize->needs_zeroed = TRUE;
+  cquantize->needs_zeroed = true;
 }
 
 
@@ -1300,21 +1300,17 @@ start_pass_2_quant (j_decompress_ptr cinfo, bool is_pre_scan)
 {
   my_cquantize_ptr cquantize = (my_cquantize_ptr) cinfo->cquantize;
   hist3d histogram = cquantize->histogram;
-  int i;
 
   if (is_pre_scan) {
     /* Set up method pointers */
     cquantize->pub.color_quantize = prescan_quantize;
     cquantize->pub.finish_pass = finish_pass1;
-    cquantize->needs_zeroed = TRUE; /* Always zero histogram */
+    cquantize->needs_zeroed = true; /* Always zero histogram */
   } else {
     /* Set up method pointers */
     cquantize->pub.color_quantize = pass2_fs_dither;
     cquantize->pub.finish_pass = finish_pass2;
 
-    /* Make sure color count is acceptable */
-    i = cinfo->actual_number_of_colors;
-
     {
       size_t arraysize = (size_t) ((cinfo->output_width + 2) *
                    (3 * sizeof(FSERROR)));
@@ -1326,17 +1322,17 @@ start_pass_2_quant (j_decompress_ptr cinfo, bool is_pre_scan)
       /* Make the error-limit table if we didn't already. */
       if (cquantize->error_limiter == NULL)
     init_error_limit(cinfo);
-      cquantize->on_odd_row = FALSE;
+      cquantize->on_odd_row = false;
     }
 
   }
   /* Zero the histogram or inverse color map, if necessary */
   if (cquantize->needs_zeroed) {
-    for (i = 0; i < HIST_C0_ELEMS; i++) {
+    for (int i = 0; i < HIST_C0_ELEMS; i++) {
       memset((void  *) histogram[i], 0,
         HIST_C1_ELEMS*HIST_C2_ELEMS * sizeof(histcell));
     }
-    cquantize->needs_zeroed = FALSE;
+    cquantize->needs_zeroed = false;
   }
 }
 
@@ -1351,7 +1347,7 @@ new_color_map_2_quant (j_decompress_ptr cinfo)
   my_cquantize_ptr cquantize = (my_cquantize_ptr) cinfo->cquantize;
 
   /* Reset the inverse color map */
-  cquantize->needs_zeroed = TRUE;
+  cquantize->needs_zeroed = true;
 }
 
 
@@ -1378,7 +1374,7 @@ jinit_2pass_quantizer (j_decompress_ptr cinfo)
   for (i = 0; i < HIST_C0_ELEMS; i++) {
     cquantize->histogram[i] = (hist2d) malloc(HIST_C1_ELEMS*HIST_C2_ELEMS * sizeof(histcell));
   }
-  cquantize->needs_zeroed = TRUE; /* histogram is garbage now */
+  cquantize->needs_zeroed = true; /* histogram is garbage now */
 
   /* Allocate storage for the completed colormap, if required.
    * We do this now since it is  storage and may affect
@@ -1467,11 +1463,11 @@ void wxQuantize::DoQuantize(unsigned w, unsigned h, unsigned char **in_rows, uns
     cquantize = (my_cquantize_ptr) dec.cquantize;
 
 
-    cquantize->pub.start_pass(&dec, TRUE);
+    cquantize->pub.start_pass(&dec, true);
     cquantize->pub.color_quantize(&dec, in_rows, out_rows, h);
     cquantize->pub.finish_pass(&dec);
 
-    cquantize->pub.start_pass(&dec, FALSE);
+    cquantize->pub.start_pass(&dec, false);
     cquantize->pub.color_quantize(&dec, in_rows, out_rows, h);
     cquantize->pub.finish_pass(&dec);
 
@@ -1508,8 +1504,6 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
 
 {
     int i;
-    int w = src.GetWidth();
-    int h = src.GetHeight();
 
     int windowsSystemColourCount = 20;
 
@@ -1527,8 +1521,9 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
 #endif
 
     // create rows info:
+    int h = src.GetHeight();
+    int w = src.GetWidth();
     unsigned char **rows = new unsigned char *[h];
-    h = src.GetHeight(), w = src.GetWidth();
     unsigned char *imgdt = src.GetData();
     for (i = 0; i < h; i++)
         rows[i] = imgdt + 3/*RGB*/ * w * i;
@@ -1551,7 +1546,7 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
 
     if (flags & wxQUANTIZE_FILL_DESTINATION_IMAGE)
     {
-        if (!dest.Ok())
+        if (!dest.IsOk())
             dest.Create(w, h);
 
         imgdt = dest.GetData();
@@ -1572,7 +1567,7 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
             // We need to shift the palette entries up
             // to make room for the Windows system colours.
             for (i = 0; i < w * h; i++)
-                data8bit[i] = data8bit[i] + paletteShift;
+                data8bit[i] = (unsigned char)(data8bit[i] + paletteShift);
         }
 #endif
         *eightBitData = data8bit;
@@ -1581,7 +1576,7 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
         delete[] data8bit;
 
 #if wxUSE_PALETTE
-    // Make a wxWindows palette
+    // Make a wxWidgets palette
     if (pPalette)
     {
         unsigned char* r = new unsigned char[256];
@@ -1626,9 +1621,11 @@ bool wxQuantize::Quantize(const wxImage& src, wxImage& dest,
         delete[] g;
         delete[] b;
     }
-#endif // wxUSE_PALETTE
+#else // !wxUSE_PALETTE
+    wxUnusedVar(pPalette);
+#endif // wxUSE_PALETTE/!wxUSE_PALETTE
 
-    return TRUE;
+    return true;
 }
 
 // This version sets a palette in the destination image so you don't
@@ -1642,7 +1639,7 @@ bool wxQuantize::Quantize(const wxImage& src,
 {
     wxPalette* palette = NULL;
     if ( !Quantize(src, dest, & palette, desiredNoColours, eightBitData, flags) )
-        return FALSE;
+        return false;
 
 #if wxUSE_PALETTE
     if (palette)
@@ -1652,6 +1649,8 @@ bool wxQuantize::Quantize(const wxImage& src,
     }
 #endif // wxUSE_PALETTE
 
-    return TRUE;
+    return true;
 }
 
+#endif
+    // wxUSE_IMAGE