Enable ICU locale provider in PG build

.github/workflows/build-and-test.yml

@@ -39,6 +39,16 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade cibuildwheel
+      - name: Setup MSYS2 (Windows)
+        if: ${{ startsWith(matrix.os, 'windows') }}
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: MINGW64
+          path-type: inherit
+          install: >-
+            base-devel
+            mingw-w64-x86_64-toolchain
+            pkg-config
       - name: Restore postgres build from cache
         if: ${{ !startsWith(matrix.os, 'ubuntu') }}
         id: restore-postgres
@@ -55,6 +65,9 @@ jobs:
         if: ${{ !startsWith(matrix.os, 'ubuntu') && !steps.restore-postgres.outputs.cache-hit }}
         # Run `make` now so that we may cache the postgres build before running cibuildwheel
         # (On Linux cibuildwheel runs in a docker container, so this won't work)
+        # On Windows, route make through MSYS2 bash so autoconf doesn't embed a
+        # space-containing Git-Bash `sh.exe` path into ICU's generated sub-Makefiles.
+        shell: ${{ startsWith(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
         env:
           MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment-target }}
         run: make

pgbuild/Makefile

@@ -4,7 +4,7 @@ INSTALL_PREFIX := $(shell pwd)/$(INSTALL_RELPATH)
 BUILD := $(shell pwd)/pgbuild/
 
 .PHONY: all
-all: pgvector postgres
+all: pgvector postgres icu
 
 ### postgres
 POSTGRES_VERSION := 16.11
@@ -21,9 +21,23 @@ $(POSTGRES_SRC)/configure: $(POSTGRES_SRC).tar.gz
 	touch $(POSTGRES_SRC)/configure
 
 ## configure
-$(POSTGRES_BLD)/config.status: $(POSTGRES_SRC)/configure
+# Point postgres at the in-tree ICU build (see icu target below) and embed an
+# rpath so installed binaries resolve libicu* from pginstall/lib at runtime.
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+  PG_LDFLAGS := -Wl,-rpath,@loader_path/../lib
+else ifeq ($(UNAME_S),Linux)
+  PG_LDFLAGS := -Wl,-rpath,\$$ORIGIN/../lib
+else
+  PG_LDFLAGS :=
+endif
+
+$(POSTGRES_BLD)/config.status: $(POSTGRES_SRC)/configure $(INSTALL_PREFIX)/lib/pkgconfig/icu-i18n.pc
 	mkdir -p $(POSTGRES_BLD)
-	cd $(POSTGRES_BLD) && ../$(POSTGRES_SRC)/configure --prefix=$(INSTALL_PREFIX) --without-icu
+	cd $(POSTGRES_BLD) && \
+		PKG_CONFIG_PATH=$(INSTALL_PREFIX)/lib/pkgconfig \
+		LDFLAGS="$(PG_LDFLAGS)" \
+		../$(POSTGRES_SRC)/configure --prefix=$(INSTALL_PREFIX) --with-icu
 
 ## build
 # https://stackoverflow.com/questions/68379786/
@@ -67,10 +81,48 @@ $(INSTALL_PREFIX)/lib/vector.so: $(PGVECTOR_DIR)/Makefile $(INSTALL_PREFIX)/bin/
 .PHONY: pgvector
 pgvector: postgres $(INSTALL_PREFIX)/lib/vector.so
 
+### icu
+# Built from source into $(INSTALL_PREFIX) so the wheel is self-contained.
+# On macOS we rewrite the dylibs' install_names to @rpath/... after install so
+# downstream binaries (postgres, pgvector) that link against them pick up a
+# relocatable reference instead of the absolute $(INSTALL_PREFIX) path.
+ICU_VERSION := 78.3
+ICU_URL := https://github.com/unicode-org/icu/releases/download/release-$(ICU_VERSION)/icu4c-$(ICU_VERSION)-sources.tgz
+ICU_DIR := icu-$(ICU_VERSION)
+
+$(ICU_DIR).tgz:
+	curl -L -o $(ICU_DIR).tgz $(ICU_URL)
+
+$(ICU_DIR)/source/configure: $(ICU_DIR).tgz
+	mkdir -p $(ICU_DIR)
+	tar xzf $(ICU_DIR).tgz -C $(ICU_DIR) --strip-components=1
+	touch $(ICU_DIR)/source/configure
+
+$(INSTALL_PREFIX)/lib/pkgconfig/icu-i18n.pc: $(ICU_DIR)/source/configure
+	mkdir -p $(INSTALL_PREFIX)
+	cd $(ICU_DIR)/source && ./configure --prefix=$(INSTALL_PREFIX) --disable-static --enable-shared
+	unset MAKELEVEL && unset MAKEFLAGS && unset MFLAGS && $(MAKE) -C $(ICU_DIR)/source -j
+	unset MAKELEVEL && unset MAKEFLAGS && unset MFLAGS && $(MAKE) -C $(ICU_DIR)/source install
+ifeq ($(UNAME_S),Darwin)
+	# Rewrite absolute install_names to @rpath-relative so downstream binaries
+	# embed relocatable LC_LOAD_DYLIB entries.
+	@for dylib in $(INSTALL_PREFIX)/lib/libicu*.dylib; do \
+		[ -L "$$dylib" ] && continue; \
+		install_name_tool -id "@rpath/$$(basename $$dylib)" "$$dylib"; \
+		for dep in $$(otool -L "$$dylib" | awk 'NR>1 {print $$1}' | grep libicu); do \
+			install_name_tool -change "$$dep" "@rpath/$$(basename $$dep)" "$$dylib"; \
+		done; \
+	done
+endif
+
+.PHONY: icu
+icu: $(INSTALL_PREFIX)/lib/pkgconfig/icu-i18n.pc
+
 ### other
 .PHONY: clean clean-all
 clean:
 	rm -rf $(INSTALL_PREFIX)
 	rm -rf postgresql-*
 	rm -rf pgvector-*
-	rm -rf *.tar.gz
+	rm -rf icu-*
+	rm -rf *.tar.gz *.tgz

pyproject.toml

@@ -62,6 +62,10 @@ test-command = "bash -x {project}/cibuildwheel_test.bash {project}"
 
 [tool.cibuildwheel.linux]
 before-all = "yum install -y readline-devel; make"
+# ICU is vendored into pginstall/lib/ by pgbuild/Makefile; postgres resolves it
+# at runtime via RUNPATH=$ORIGIN/../lib. Tell auditwheel not to graft these
+# sonames into .libs/ (it would double-bundle and break our relative rpath).
+repair-wheel-command = "auditwheel repair --exclude libicui18n.so.78 --exclude libicuuc.so.78 --exclude libicudata.so.78 -w {dest_dir} {wheel}"
 
 [tool.isort]
 force_single_line = false

src/pixeltable_pgserver/postgres_server.py

@@ -148,6 +148,8 @@ def ensure_pgdata_inited(self) -> None:
                     '--auth=trust',
                     '--auth-local=trust',
                     '--encoding=utf8',
+                    '--locale-provider=icu',
+                    '--icu-locale=und',
                     '-U',
                     self.postgres_user,
                     '-D',

tests/test_pgserver.py

@@ -43,6 +43,30 @@ def _check_sqlalchemy_works(srv: PostgresServer, driver: str | None = None) -> N
         assert result[0] == 1
 
 
+def _check_utf8(srv: PostgresServer) -> None:
+    database_name = 'testdb'
+    uri = srv.get_uri(database_name, 'psycopg')
+
+    if not database_exists(uri):
+        create_database(uri)
+
+    engine = sa.create_engine(uri)
+    conn = engine.connect()
+
+    with conn.begin():
+        # German sharp s -> SS, Spanish ñ -> Ñ, accented é -> É
+        cur = conn.execute(sa.text("select upper('ßñé');"))
+        result = cur.fetchone()
+        assert result
+        assert result[0] == 'SSÑÉ'
+
+        # Chinese characters are recognized as POSIX alphabetic
+        cur = conn.execute(sa.text(r"select '中文' ~ '^[[:alpha:]]+$';"))
+        result = cur.fetchone()
+        assert result
+        assert result[0] is True
+
+
 def _check_time_zones(srv: PostgresServer) -> None:
     # Check that time zone information was properly compiled
     database_name = 'testdb'
@@ -290,6 +314,10 @@ def test_pgvector(tmp_postgres: PostgresServer) -> None:
     assert ret.strip() == 'CREATE EXTENSION'
 
 
+def test_utf8(tmp_postgres: PostgresServer) -> None:
+    _check_utf8(tmp_postgres)
+
+
 def test_start_failure_log(caplog: pytest.LogCaptureFixture) -> None:
     """Test server log contents are shown in python log when failures"""
     with tempfile.TemporaryDirectory() as tmpdir: