Avoid signed integer overflow

...as seen at

> /tools/source/misc/fix16.cxx:79:53: runtime error: signed integer overflow: -2147483648 - 1 cannot be represented in type 'int'
>     #0 0x7fc2f9f50a60 in mask(int) /tools/source/misc/fix16.cxx:79:53
>     #1 0x7fc2f9f505fd in fix16_div(int, int) /tools/source/misc/fix16.cxx:122:34
>     #2 0x7fc2f00d12e0 in vcl::fixedDiv(int, int) /vcl/source/fontsubset/sft.cxx:127:12
>     #3 0x7fc2f00d11c1 in vcl::fixedMulDiv(int, int, int) /vcl/source/fontsubset/sft.cxx:135:12
>     #4 0x7fc2f00d0851 in vcl::GetCompoundTTOutline(vcl::AbstractTrueTypeFont*, unsigned int, std::__debug::vector<vcl::ControlPoint, std::allocator<vcl::ControlPoint> >&, vcl::(anonymous namespace)::TTGlyphMetrics*, std::__debug::vector<unsigned int, std::allocator<unsigned int> >&) /vcl/source/fontsubset/sft.cxx:544:91
>     #5 0x7fc2f00af5f1 in vcl::GetTTGlyphOutline(vcl::AbstractTrueTypeFont*, unsigned int, std::__debug::vector<vcl::ControlPoint, std::allocator<vcl::ControlPoint> >&, vcl::(anonymous namespace)::TTGlyphMetrics*, std::__debug::vector<unsigned int, std::allocator<unsigned int> >*) /vcl/source/fontsubset/sft.cxx:621:15
>     #6 0x7fc2f00aea34 in vcl::GetTTGlyphPoints(vcl::AbstractTrueTypeFont*, unsigned int, std::__debug::vector<vcl::ControlPoint, std::allocator<vcl::ControlPoint> >&) /vcl/source/fontsubset/sft.cxx:1239:12
>     #7 0x7fc2f00b92de in vcl::GetTTRawGlyphData(vcl::AbstractTrueTypeFont*, unsigned int) /vcl/source/fontsubset/sft.cxx:1667:9
>     #8 0x7fc2f00b23ed in vcl::CreateTTFromTTGlyphs(vcl::AbstractTrueTypeFont*, std::__debug::vector<unsigned char, std::allocator<unsigned char> >&, unsigned short const*, unsigned char const*, int) /vcl/source/fontsubset/sft.cxx:1374:32
>     #9 0x7fc2f00bc71a in vcl::CreateTTFfontSubset(vcl::AbstractTrueTypeFont&, std::__debug::vector<unsigned char, std::allocator<unsigned char> >&, unsigned int const*, unsigned char const*, int, FontSubsetInfo&) /vcl/source/fontsubset/sft.cxx:1549:13
>     #10 0x7fc2eff57836 in vcl::font::PhysicalFontFace::CreateFontSubset(std::__debug::vector<unsigned char, std::allocator<unsigned char> >&, unsigned int const*, unsigned char const*, int, FontSubsetInfo&) const /vcl/source/font/PhysicalFontFace.cxx:367:12
>     #11 0x7fc2ee4e0472 in vcl::PDFWriterImpl::emitFonts() /vcl/source/gdi/pdfwriter_impl.cxx:3152:24
>     #12 0x7fc2ee4e9cb9 in vcl::PDFWriterImpl::emitResources() /vcl/source/gdi/pdfwriter_impl.cxx:3363:10
>     #13 0x7fc2ee53f91d in vcl::PDFWriterImpl::emitCatalog() /vcl/source/gdi/pdfwriter_impl.cxx:5288:10
>     #14 0x7fc2ee567ae3 in vcl::PDFWriterImpl::emit() /vcl/source/gdi/pdfwriter_impl.cxx:6341:10
>     #15 0x7fc2ee3d3f2a in vcl::PDFWriter::Emit() /vcl/source/gdi/pdfwriter.cxx:53:29
>     #16 0x7fc249b4b82a in PDFExport::Export(rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /filter/source/pdf/pdfexport.cxx:1127:39
>     #17 0x7fc249bb2782 in PDFFilter::implExport(com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /filter/source/pdf/pdffilter.cxx:182:24
>     #18 0x7fc249bb342b in PDFFilter::filter(com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /filter/source/pdf/pdffilter.cxx:247:23
>     #19 0x7fc2a51ebec6 in SfxObjectShell::ExportTo(SfxMedium&) /sfx2/source/doc/objstor.cxx:2945:25
>     #20 0x7fc2a51d7bec in SfxObjectShell::SaveTo_Impl(SfxMedium&, SfxItemSet const*) /sfx2/source/doc/objstor.cxx:1943:19
>     #21 0x7fc2a5213f83 in SfxObjectShell::PreDoSaveAs_Impl(rtl::OUString const&, rtl::OUString const&, SfxItemSet const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /sfx2/source/doc/objstor.cxx:3459:39
>     #22 0x7fc2a520b709 in SfxObjectShell::CommonSaveAs_Impl(INetURLObject const&, rtl::OUString const&, SfxItemSet&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /sfx2/source/doc/objstor.cxx:3249:9
>     #23 0x7fc2a5147a8b in SfxObjectShell::APISaveAs_Impl(std::basic_string_view<char16_t, std::char_traits<char16_t> >, SfxItemSet&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /sfx2/source/doc/objserv.cxx:320:19
>     #24 0x7fc2a543024c in SfxBaseModel::impl_store(rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&, bool) /sfx2/source/doc/sfxbasemodel.cxx:3231:42
>     #25 0x7fc2a54374f4 in SfxBaseModel::storeToURL(rtl::OUString const&, com::sun::star::uno::Sequence<com::sun::star::beans::PropertyValue> const&) /sfx2/source/doc/sfxbasemodel.cxx:1813:13
>     #26 0x7fc2c74053ae in ScPDFExportTest::exportToPDF(com::sun::star::uno::Reference<com::sun::star::frame::XModel> const&, ScRange const&) /sc/qa/extras/scpdfexport.cxx:179:16
>     #27 0x7fc2c7437e0f in ScPDFExportTest::testForcepoint97() /sc/qa/extras/scpdfexport.cxx:915:5

during CppunitTest_sc_pdf_export
(<https://ci.libreoffice.org/job/lo_ubsan/3346/>)

Change-Id: I3074479ea3267417a73ecafa81b4a9898f19d1a9
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/175469
Reviewed-by: Stephan Bergmann <[email protected]>
Tested-by: Jenkins

tools/source/misc/fix16.cxx

@@ -76,7 +76,7 @@ fix16_t fix16_mul(fix16_t inArg0, fix16_t inArg1)
     return result;
 }
 
-static uint32_t mask(int bits) { return (1 << bits) - 1; }
+static uint32_t mask(int bits) { return (1U << bits) - 1; }
 
 /* 32-bit implementation of fix16_div. Fastest version for e.g. ARM Cortex M3.
  * Performs 32-bit divisions repeatedly to reduce the remainder. For this to