Merge pull request #214 from r-wasm/gdal

Preliminary support for geospatial stack

.github/workflows/deploy.yml

@@ -35,6 +35,8 @@ jobs:
           cp /opt/flang/emfc ./host/bin/emfc
       - name: Setup Emscripten PATH
         run: echo "/opt/emsdk:/opt/emsdk/upstream/emscripten" >> $GITHUB_PATH
+      - name: Set Emscripten EM_NODE_JS
+        run: echo "EM_NODE_JS=$(which node)" >> $GITHUB_ENV
       - name: Set the webR CDN URL as the BASE_URL
         run: echo "BASE_URL=https://webr.r-wasm.org/${{ github.ref_name }}/" > "$HOME/.webr-config.mk"
         shell: bash

.github/workflows/npm.yml

@@ -26,6 +26,8 @@ jobs:
           cp /opt/flang/emfc ./host/bin/emfc
       - name: Setup Emscripten PATH
         run: echo "/opt/emsdk:/opt/emsdk/upstream/emscripten" >> $GITHUB_PATH
+      - name: Set Emscripten EM_NODE_JS
+        run: echo "EM_NODE_JS=$(which node)" >> $GITHUB_ENV
       - name: Build webR
         env:
           EMSDK: /opt/emsdk

.github/workflows/test.yml

@@ -25,6 +25,8 @@ jobs:
           cp /opt/flang/emfc ./host/bin/emfc
       - name: Setup Emscripten PATH
         run: echo "/opt/emsdk:/opt/emsdk/upstream/emscripten" >> $GITHUB_PATH
+      - name: Set Emscripten EM_NODE_JS
+        run: echo "EM_NODE_JS=$(which node)" >> $GITHUB_ENV
       - name: Build webR
         env:
           EMSDK: /opt/emsdk

R/Makefile

@@ -30,8 +30,17 @@ WASM_OPT ?= -Oz
 WASM_OPT_LDADD ?= $(WASM_OPT)
 WASM_OPT_LDADD += -s DECLARE_ASM_MODULE_EXPORTS=0
 
+WASM_CPPFLAGS := $(WASM_CPPFLAGS)
+WASM_CPPFLAGS += -fwasm-exceptions
+WASM_CPPFLAGS += -s SUPPORT_LONGJMP=wasm
+
 WASM_CFLAGS := $(WASM_CFLAGS)
-WASM_CFLAGS += -fPIC -fno-exceptions -fno-rtti $(WASM_OPT)
+WASM_CFLAGS += $(WASM_OPT)
+WASM_CFLAGS += -fPIC
+
+WASM_LDFLAGS := $(WASM_LDFLAGS)
+WASM_LDFLAGS += -fwasm-exceptions
+WASM_LDFLAGS += -s SUPPORT_LONGJMP=wasm
 
 .PHONY: all
 all: R
@@ -93,16 +102,19 @@ $(BUILD)/state/R-$(R_VERSION)/r-stage1: $(BUILD)/state/R-$(R_VERSION)/r-stage1-c
 	touch $@
 
 STAGE2_CPPFLAGS := $(STAGE2_CPPFLAGS)
+STAGE2_CPPFLAGS += $(WASM_CPPFLAGS)
 STAGE2_CPPFLAGS += -I$(WASM)/include
 STAGE2_CPPFLAGS += -DEXPEL_OLD_TO_NEW=1
 STAGE2_CPPFLAGS += -s USE_BZIP2=1
 STAGE2_CPPFLAGS += -s USE_ZLIB=1
 STAGE2_CPPFLAGS += -s USE_FREETYPE=1
 
 STAGE2_CFLAGS := $(STAGE2_CFLAGS)
-STAGE2_CFLAGS += $(STAGE2_CPPFLAGS) $(WASM_CFLAGS)
+STAGE2_CFLAGS += $(WASM_CFLAGS)
+STAGE2_CFLAGS += $(STAGE2_CPPFLAGS)
 
 STAGE2_LDFLAGS := $(STAGE2_LDFLAGS)
+STAGE2_LDFLAGS += $(WASM_LDFLAGS)
 STAGE2_LDFLAGS += -L$(WASM)/lib
 STAGE2_LDFLAGS +=-s USE_FREETYPE=1
 

libs/Makefile

@@ -9,20 +9,48 @@ TOOLS = $(WEBR_ROOT)/tools
 HOST = $(WEBR_ROOT)/host
 WASM = $(WEBR_ROOT)/wasm
 
-export EM_PKG_CONFIG_PATH = $(WASM)/lib/pkgconfig
+EM_LIBS := -s USE_BZIP2=1
+EM_LIBS += -s USE_ZLIB=1
 
 WASM_OPT ?= -Oz
 WASM_OPT_LDADD ?= $(WASM_OPT)
 
+WASM_COMMON_FLAGS := $(WASM_OPT)
+WASM_COMMON_FLAGS += -fPIC
+WASM_COMMON_FLAGS += -fwasm-exceptions
+WASM_COMMON_FLAGS += -s SUPPORT_LONGJMP=wasm
+
 WASM_CFLAGS := $(WASM_CFLAGS)
-WASM_CFLAGS += -fPIC -fno-exceptions -fno-rtti $(WASM_OPT)
-WASM_CFLAGS += -s USE_BZIP2=1 -s USE_ZLIB=1
+WASM_CFLAGS += $(WASM_COMMON_FLAGS)
+
+WASM_CXXFLAGS := $(WASM_CXXFLAGS)
+WASM_CXXFLAGS += $(WASM_COMMON_FLAGS)
+
+WASM_CPPFLAGS := $(WASM_CPPFLAGS)
+WASM_CPPFLAGS += -I$(WASM)/include
+WASM_CPPFLAGS += $(EM_LIBS)
+
+WASM_LDFLAGS := $(WASM_LDFLAGS)
+WASM_LDFLAGS += -L$(WASM)/lib
+WASM_LDFLAGS += $(EM_LIBS)
+WASM_LDFLAGS += -fwasm-exceptions
+WASM_LDFLAGS += -s SUPPORT_LONGJMP=wasm
+
+export CPPFLAGS = $(WASM_CPPFLAGS)
+export CFLAGS = $(WASM_CFLAGS)
+export CXXFLAGS = $(WASM_CXXFLAGS)
+export LDFLAGS = $(WASM_LDFLAGS)
+export EM_PKG_CONFIG_PATH = $(WASM)/lib/pkgconfig
 
 include recipes/**/targets.mk
 include recipes/**/rules.mk
 
-all: $(DEFAULT_WASM_LIBS) $(WASM)/usr/share/fonts
-.DEFAULT_GOAL := all
+.PHONY: default
+default: $(DEFAULT_WASM_LIBS) $(WASM)/usr/share/fonts
+.DEFAULT_GOAL := default
+
+.PHONY: all
+all: default $(OPTIONAL_WASM_LIBS)
 
 $(EM_PKG_CONFIG_PATH)/%.pc: recipes/**/%.pc
 	mkdir -p $(EM_PKG_CONFIG_PATH)
@@ -32,6 +60,7 @@ $(EM_PKG_CONFIG_PATH)/%.pc: recipes/**/%.pc
 clean:
 	rm -rf $(DOWNLOAD) $(BUILD)
 	rm -f $(DEFAULT_WASM_LIBS)
+	rm -f $(OPTIONAL_WASM_LIBS)
 	rm -f $(EM_PKG_CONFIG_PATH)/*.pc
 
 # Print Makefile variable

libs/recipes/cairo/rules.mk

@@ -16,8 +16,8 @@ $(CAIRO_WASM_LIB): $(CAIRO_TARBALL) $(PIXMAN_WASM_LIB) $(FC_DEPS)
 	cp -r "$(WEBR_ROOT)/patches/cairo-$(CAIRO_VERSION)/." \
 	  "$(BUILD)/cairo-$(CAIRO_VERSION)/patches"
 	cd $(BUILD)/cairo-$(CAIRO_VERSION)/build && quilt push -a && \
-	  CFLAGS="$(WASM_CFLAGS) -DCAIRO_NO_MUTEX=1" \
-	  LDFLAGS="-sUSE_FREETYPE=1 -sUSE_PTHREADS=0" \
+	  CFLAGS="$(CFLAGS) -DCAIRO_NO_MUTEX=1" \
+	  LDFLAGS="$(LDFLAGS) -sUSE_FREETYPE=1 -sUSE_PTHREADS=0" \
 	  emconfigure ../configure \
 	    ax_cv_c_float_words_bigendian=no \
 	    --enable-shared=no \

libs/recipes/fontconfig/rules.mk

@@ -13,8 +13,7 @@ $(FC_DEPS): $(FC_TARBALL) $(LIBXML2_WASM_LIB) $(EM_PKG_CONFIG_PATH)/freetype2.pc
 	mkdir -p $(BUILD)/fontconfig-$(FC_VERSION)/build
 	tar -C $(BUILD) -xf $(FC_TARBALL) --exclude=fcobjshash.h
 	cd $(BUILD)/fontconfig-$(FC_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
-	  LDFLAGS="-sUSE_FREETYPE=1 -sUSE_PTHREADS=0" \
+	  LDFLAGS="$(LDFLAGS) -sUSE_FREETYPE=1 -sUSE_PTHREADS=0" \
 	  PTHREAD_CFLAGS=" " \
 	  emconfigure ../configure \
 	    ac_cv_func_fstatfs=no \

libs/recipes/gdal/rules.mk

@@ -0,0 +1,45 @@
+GDAL_VERSION = 3.6.4
+GDAL_TARBALL = $(DOWNLOAD)/gdal-$(GDAL_VERSION).tar.gz
+GDAL_URL = https://github.com/OSGeo/gdal/releases/download/v$(GDAL_VERSION)/gdal-$(GDAL_VERSION).tar.gz
+
+.PHONY: gdal
+gdal: $(GDAL_WASM_LIB)
+
+$(GDAL_TARBALL):
+	mkdir -p $(DOWNLOAD)
+	wget $(GDAL_URL) -O $@
+
+# https://gdal.org/development/building_from_source.html
+
+# There is one configure helper that makes an erroneous conclusion
+# about the signature of an iconv function because building the test
+# file fails with an unrelated Emscripten error about a missing symbol.
+# To work around, we just fix the setting directly in `cpl_config.h`.
+
+# The `CONFIG_DEP_LIBS` field in `gdal-config` needs two edits. First
+# remove `-lproj` to avoid duplicates in build command that include
+# both `gdal-config --dep-libs` and `pkg-config proj --libs`.
+# Emscripten expands those flags to the `libproj.a` archive file,
+# which causes duplicate symbols errors.
+#
+# Second, remove the `-L/path/to/emscripten/sysroot/wasm32-emscripten`
+# flag. This directory contains non-fPIC libraries. Not sure why the
+# `pic` subfolder isn't selected.
+
+$(GDAL_WASM_LIB): $(GDAL_TARBALL) $(PROJ_WASM_LIB)
+	mkdir -p $(BUILD)/gdal-$(GDAL_VERSION)/build
+	tar -C $(BUILD) -xf $(GDAL_TARBALL)
+	cd $(BUILD)/gdal-$(GDAL_VERSION)/build && \
+	  emcmake cmake --debug-find \
+	    -DCMAKE_FIND_ROOT_PATH=$(WASM) \
+	    -DCMAKE_INSTALL_PREFIX:PATH=$(WASM) \
+	    -DBUILD_TESTING=OFF \
+	    -DBUILD_APPS=OFF \
+	    -DGDAL_ENABLE_DRIVER_HDF5=OFF \
+	    -DGDAL_USE_HDF4=OFF \
+	    -DGDAL_USE_HDF5=OFF \
+	    .. && \
+	  sed -i.bak 's/#define ICONV_CPP_CONST const/#define ICONV_CPP_CONST/' port/cpl_config.h && \
+	  emmake make install && \
+	  sed -i.bak 's/\(^CONFIG_DEP_LIBS=.*\) -lproj\(.*\)/\1\2/' $(WASM)/bin/gdal-config && \
+	  sed -i.bak 's/\(^CONFIG_DEP_LIBS=.*\)-L.*wasm32-emscripten\(.*\)/\1 \2/' $(WASM)/bin/gdal-config

libs/recipes/gdal/targets.mk

@@ -0,0 +1,2 @@
+GDAL_WASM_LIB = $(WASM)/lib/libgdal.a
+OPTIONAL_WASM_LIBS += $(GDAL_WASM_LIB)

libs/recipes/openssl/rules.mk

@@ -13,7 +13,6 @@ $(OPENSSL_WASM_LIB): $(OPENSSL_TARBALL)
 	mkdir -p $(BUILD)/openssl-openssl-$(OPENSSL_VERSION)/build
 	tar -C $(BUILD) -xf $(OPENSSL_TARBALL)
 	cd $(BUILD)/openssl-openssl-$(OPENSSL_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../Configure \
 	    darwin-i386 \
 	    CC="cc" \

libs/recipes/openssl/targets.mk

@@ -1 +1,2 @@
 OPENSSL_WASM_LIB = $(WASM)/lib/libssl.a
+OPTIONAL_WASM_LIBS += $(OPENSSL_WASM_LIB)

libs/recipes/pcre/rules.mk

@@ -13,7 +13,6 @@ $(PCRE_WASM_LIB): $(PCRE_TARBALL)
 	mkdir -p $(BUILD)/pcre2-$(PCRE_VERSION)/build
 	tar -C $(BUILD) -xf $(PCRE_TARBALL)
 	cd $(BUILD)/pcre2-$(PCRE_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../configure \
 	    --enable-shared=no \
 	    --enable-static=yes \

libs/recipes/pixman/rules.mk

@@ -15,7 +15,6 @@ $(PIXMAN_WASM_LIB): $(PIXMAN_TARBALL) $(LIBPNG_WASM_LIB)
 	sed -i.bak 's/support_for_pthreads=yes/support_for_pthreads=no/g' \
 	  $(BUILD)/pixman-$(PIXMAN_VERSION)/configure
 	cd $(BUILD)/pixman-$(PIXMAN_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../configure \
 	    --enable-shared=no \
 	    --enable-static=yes \

libs/recipes/png/rules.mk

@@ -13,7 +13,6 @@ $(LIBPNG_WASM_LIB): $(LIBPNG_TARBALL) $(EM_PKG_CONFIG_PATH)/zlib.pc
 	mkdir -p $(BUILD)/libpng-$(LIBPNG_VERSION)/build
 	tar -C $(BUILD) -xf $(LIBPNG_TARBALL)
 	cd $(BUILD)/libpng-$(LIBPNG_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../configure \
 	    --enable-shared=no \
 	    --enable-static=yes \

libs/recipes/proj/rules.mk

@@ -0,0 +1,37 @@
+PROJ_VERSION = 9.2.0
+PROJ_TARBALL = $(DOWNLOAD)/proj-$(PROJ_VERSION).tar.gz
+PROJ_URL = https://github.com/OSGeo/PROJ/releases/download/$(PROJ_VERSION)/proj-$(PROJ_VERSION).tar.gz
+
+.PHONY: proj
+proj: $(PROJ_WASM_LIB)
+
+$(PROJ_TARBALL):
+	mkdir -p $(DOWNLOAD)
+	wget -q -O $@ $(PROJ_URL)
+
+# https://github.com/OSGeo/PROJ/blob/master/docs/source/install.rst
+
+# Somehow `-DBUILD_APPS` doesn't work and we need to turn off each app
+# individually. TIFF support is disabled but we probably should enable
+# it once we have built libtiff as a wasm lib.
+$(PROJ_WASM_LIB): $(PROJ_TARBALL) $(SQLITE3_WASM_LIB)
+	mkdir -p $(BUILD)/proj-$(PROJ_VERSION)/build
+	tar -C $(BUILD) -xf $(PROJ_TARBALL)
+	cd $(BUILD)/proj-$(PROJ_VERSION)/build && \
+	  emcmake cmake \
+	    -DCMAKE_INSTALL_PREFIX:PATH="$(WASM)" \
+	    -DSQLITE3_INCLUDE_DIR="$(WASM)/include" \
+	    -DSQLITE3_LIBRARY="$(SQLITE3_WASM_LIB)" \
+	    -DENABLE_TIFF=OFF \
+	    -DENABLE_CURL=OFF \
+	    -DBUILD_APPS=OFF \
+	    -DBUILD_CCT=OFF \
+	    -DBUILD_CS2CS=OFF \
+	    -DBUILD_GEOD=OFF \
+	    -DBUILD_GIE=OFF \
+	    -DBUILD_PROJ=OFF \
+	    -DBUILD_PROJINFO=OFF \
+	    -DBUILD_PROJSYNC=OFF \
+	    -DBUILD_TESTING=OFF \
+	    .. && \
+	  emmake make install

libs/recipes/proj/targets.mk

@@ -0,0 +1,2 @@
+PROJ_WASM_LIB = $(WASM)/lib/libproj.a
+OPTIONAL_WASM_LIBS += $(PROJ_WASM_LIB)

libs/recipes/sqlite3/rules.mk

@@ -0,0 +1,22 @@
+SQLITE3_VERSION = 3.41.2
+SQLITE3_VERSION2 = 3410200
+SQLITE3_FILE_NAME = sqlite-autoconf-$(SQLITE3_VERSION2)
+SQLITE3_TARBALL = $(DOWNLOAD)/$(SQLITE3_FILE_NAME).tar.gz
+SQLITE3_URL = https://sqlite.org/2023/$(SQLITE3_FILE_NAME).tar.gz
+
+.PHONY: sqlite3
+sqlite3: $(SQLITE3_WASM_LIB)
+
+$(SQLITE3_TARBALL):
+	mkdir -p $(DOWNLOAD)
+	wget $(SQLITE3_URL) -O $@
+
+$(SQLITE3_WASM_LIB): $(SQLITE3_TARBALL)
+	mkdir -p $(BUILD)/$(SQLITE3_FILE_NAME)/build
+	tar -C $(BUILD) -xf $(SQLITE3_TARBALL)
+	cd $(BUILD)/$(SQLITE3_FILE_NAME)/build && \
+	  emconfigure ../configure \
+	    --enable-shared=no \
+	    --enable-static=yes \
+	    --prefix=$(WASM) && \
+	  emmake make install

libs/recipes/sqlite3/targets.mk

@@ -0,0 +1,2 @@
+SQLITE3_WASM_LIB = $(WASM)/lib/libsqlite3.a
+OPTIONAL_WASM_LIBS += $(SQLITE3_WASM_LIB)

libs/recipes/xml2/rules.mk

@@ -15,7 +15,6 @@ $(LIBXML2_WASM_LIB): $(LIBXML2_TARBALL) $(XZ_WASM_LIB)
 	mkdir -p $(BUILD)/libxml2-$(LIBXML2_VERSION)/build
 	tar -C $(BUILD) -xf $(LIBXML2_TARBALL)
 	cd $(BUILD)/libxml2-$(LIBXML2_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../configure \
 	    --enable-shared=no \
 	    --enable-static=yes \

libs/recipes/xz/rules.mk

@@ -13,7 +13,6 @@ $(XZ_WASM_LIB): $(XZ_TARBALL)
 	mkdir -p $(BUILD)/xz-$(XZ_VERSION)/build
 	tar -C $(BUILD) -xf $(XZ_TARBALL)
 	cd $(BUILD)/xz-$(XZ_VERSION)/build && \
-	  CFLAGS="$(WASM_CFLAGS)" \
 	  emconfigure ../configure \
 	    --enable-shared=no \
 	    --enable-static=yes \