Skip to content

Commit 374a3ce

Browse files
committed
Merge remote-tracking branch 'origin/main' into feat/v17-full-api-merge
# Conflicts: # versions/17/package.json
2 parents 9e294a4 + 70ac314 commit 374a3ce

25 files changed

Lines changed: 522 additions & 102 deletions

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,7 @@ jobs:
5151
run: pnpm install
5252

5353
- name: Build WASM 🏗
54-
run: |
55-
if [ "${{ matrix.package.name }}" = "v13" ]; then
56-
# Download prebuilt WASM for v13 since it can't build in CI
57-
mkdir -p wasm
58-
curl -o v13.tgz "https://registry.npmjs.org/@libpg-query/v13/-/v13-13.5.7.tgz"
59-
tar -xzf v13.tgz --strip-components=1 package/wasm
60-
rm v13.tgz
61-
else
62-
pnpm run build
63-
fi
54+
run: pnpm run build
6455
working-directory: ${{ matrix.package.path }}
6556

6657
- name: Upload WASM Artifacts 📦

‎templates/full/index.ts‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ interface WasmModule {
131131
stringToUTF8: (str: string, ptr: number, len: number) => void;
132132
UTF8ToString: (ptr: number) => string;
133133
getValue: (ptr: number, type: string) => number;
134+
HEAPU8: Uint8Array;
134135
}
135136

136137
let wasmModule: WasmModule;
@@ -156,20 +157,28 @@ function awaitInit<T extends (...args: any[]) => Promise<any>>(fn: T): T {
156157
}) as T;
157158
}
158159

160+
const SURROGATE = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]/g;
161+
159162
function stringToPtr(str: string): number {
160163
ensureLoaded();
161164
if (typeof str !== 'string') {
162165
throw new TypeError(`Expected a string, got ${typeof str}`);
163166
}
164-
const len = wasmModule.lengthBytesUTF8(str) + 1;
165-
const ptr = wasmModule._malloc(len);
166-
try {
167-
wasmModule.stringToUTF8(str, ptr, len);
168-
return ptr;
169-
} catch (error) {
170-
wasmModule._free(ptr);
171-
throw error;
167+
// libpg_query takes a NUL-terminated C string: an embedded NUL would end the
168+
// parse early and silently drop everything after it.
169+
if (str.includes('\0')) {
170+
throw new Error('Query must not contain a NUL (U+0000) character');
171+
}
172+
// lengthBytesUTF8 and stringToUTF8 disagree on unpaired surrogates, which cut
173+
// the query short. Write each as U+FFFD, the same bytes a Postgres driver sends.
174+
const wellFormed = str.replace(SURROGATE, (s) => (s.length === 2 ? s : '\uFFFD'));
175+
const size = wasmModule.lengthBytesUTF8(wellFormed) + 1;
176+
const ptr = wasmModule._malloc(size);
177+
if (!ptr) {
178+
throw new Error('Failed to allocate memory for query');
172179
}
180+
wasmModule.stringToUTF8(wellFormed, ptr, size);
181+
return ptr;
173182
}
174183

175184
function ptrToString(ptr: number): string {

‎templates/full/libpg-query.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ declare module './libpg-query.js' {
1010
lengthBytesUTF8: (str: string) => number;
1111
stringToUTF8: (str: string, ptr: number, len: number) => void;
1212
UTF8ToString: (ptr: number) => string;
13+
HEAPU8: Uint8Array;
1314
}
1415

1516
const PgQueryModule: () => Promise<WasmModule>;

‎templates/index.ts‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -145,20 +145,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
145145
}) as T;
146146
}
147147

148+
const SURROGATE = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]/g;
149+
148150
function stringToPtr(str: string): number {
149151
ensureLoaded();
150152
if (typeof str !== 'string') {
151153
throw new TypeError(`Expected a string, got ${typeof str}`);
152154
}
153-
const len = wasmModule.lengthBytesUTF8(str) + 1;
154-
const ptr = wasmModule._malloc(len);
155-
try {
156-
wasmModule.stringToUTF8(str, ptr, len);
157-
return ptr;
158-
} catch (error) {
159-
wasmModule._free(ptr);
160-
throw error;
155+
// libpg_query takes a NUL-terminated C string: an embedded NUL would end the
156+
// parse early and silently drop everything after it.
157+
if (str.includes('\0')) {
158+
throw new Error('Query must not contain a NUL (U+0000) character');
159+
}
160+
// lengthBytesUTF8 and stringToUTF8 disagree on unpaired surrogates, which cut
161+
// the query short. Write each as U+FFFD, the same bytes a Postgres driver sends.
162+
const wellFormed = str.replace(SURROGATE, (s) => (s.length === 2 ? s : '\uFFFD'));
163+
const size = wasmModule.lengthBytesUTF8(wellFormed) + 1;
164+
const ptr = wasmModule._malloc(size);
165+
if (!ptr) {
166+
throw new Error('Failed to allocate memory for query');
161167
}
168+
wasmModule.stringToUTF8(wellFormed, ptr, size);
169+
return ptr;
162170
}
163171

164172
function ptrToString(ptr: number): string {

‎versions/13/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"wasm:rebuild": "pnpm wasm:make rebuild",
3131
"wasm:clean": "pnpm wasm:make clean",
3232
"wasm:clean-cache": "pnpm wasm:make clean-cache",
33-
"test": "node --test test/parsing.test.js test/errors.test.js"
33+
"test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
3434
},
3535
"author": "Constructive <developers@constructive.io>",
3636
"license": "MIT",
Lines changed: 57 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,59 @@
1-
--- a/Makefile
2-
+++ b/Makefile
3-
@@ -36,6 +36,11 @@ OBJ_FILES := $(filter-out $(NOT_OBJ_FILES), $(SRC_FILES:.c=.o))
4-
override CFLAGS += -g -I. -I./vendor -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
1+
Backport of the PostgreSQL 14 s_lock.h fallback that uses __sync_lock_test_and_set()
2+
when there is no platform-specific TAS. PG13 has none for wasm32, so without this
3+
Emscripten hits "#error PostgreSQL does not have native spinlock support".
4+
5+
diff --git a/src/postgres/include/storage/s_lock.h b/src/postgres/include/storage/s_lock.h
6+
index 6b368a5..d8d0bff 100644
7+
--- a/src/postgres/include/storage/s_lock.h
8+
+++ b/src/postgres/include/storage/s_lock.h
9+
@@ -748,6 +748,50 @@ tas(volatile slock_t *lock)
10+
typedef unsigned char slock_t;
11+
#endif
512

6-
override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib
713
+
8-
+# Disable spinlocks when building with Emscripten
9-
+ifdef EMSCRIPTEN
10-
+override PG_CONFIGURE_FLAGS += --disable-spinlocks
11-
+endif
12-
13-
override TEST_CFLAGS += -I. -I./vendor -g
14-
override TEST_LDFLAGS += -pthread
15-
@@ -149,6 +154,9 @@ extract_source: $(PGDIR)
16-
echo "#undef LOCALE_T_IN_XLOCALE" >> ./src/postgres/include/pg_config.h
17-
echo "#undef WCSTOMBS_L_IN_XLOCALE" >> ./src/postgres/include/pg_config.h
18-
# Support 32-bit systems without reconfiguring
19-
+ifdef EMSCRIPTEN
20-
+ echo "#undef HAVE_SPINLOCKS" >> ./src/postgres/include/pg_config.h
21-
+endif
22-
echo "#undef PG_INT128_TYPE" >> ./src/postgres/include/pg_config.h
23-
# Support gcc earlier than 4.6.0 without reconfiguring
24-
echo "#undef HAVE__STATIC_ASSERT" >> ./src/postgres/include/pg_config.h
14+
+/*
15+
+ * If we have no platform-specific knowledge, but we found that the compiler
16+
+ * provides __sync_lock_test_and_set(), use that. Prefer the int-width
17+
+ * version over the char-width version if we have both, on the rather dubious
18+
+ * grounds that that's known to be more likely to work in the ARM ecosystem.
19+
+ * (But we dealt with ARM above.)
20+
+ */
21+
+#if !defined(HAS_TEST_AND_SET)
22+
+
23+
+#if defined(HAVE_GCC__SYNC_INT32_TAS)
24+
+#define HAS_TEST_AND_SET
25+
+
26+
+#define TAS(lock) tas(lock)
27+
+
28+
+typedef int slock_t;
29+
+
30+
+static __inline__ int
31+
+tas(volatile slock_t *lock)
32+
+{
33+
+ return __sync_lock_test_and_set(lock, 1);
34+
+}
35+
+
36+
+#define S_UNLOCK(lock) __sync_lock_release(lock)
37+
+
38+
+#elif defined(HAVE_GCC__SYNC_CHAR_TAS)
39+
+#define HAS_TEST_AND_SET
40+
+
41+
+#define TAS(lock) tas(lock)
42+
+
43+
+typedef char slock_t;
44+
+
45+
+static __inline__ int
46+
+tas(volatile slock_t *lock)
47+
+{
48+
+ return __sync_lock_test_and_set(lock, 1);
49+
+}
50+
+
51+
+#define S_UNLOCK(lock) __sync_lock_release(lock)
52+
+
53+
+#endif /* HAVE_GCC__SYNC_INT32_TAS */
54+
+
55+
+#endif /* !defined(HAS_TEST_AND_SET) */
56+
+
57+
/*
58+
* Default implementation of S_UNLOCK() for gcc/icc.
59+
*

‎versions/13/src/index.ts‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -152,20 +152,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
152152
}) as T;
153153
}
154154

155+
const SURROGATE = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]/g;
156+
155157
function stringToPtr(str: string): number {
156158
ensureLoaded();
157159
if (typeof str !== 'string') {
158160
throw new TypeError(`Expected a string, got ${typeof str}`);
159161
}
160-
const len = wasmModule.lengthBytesUTF8(str) + 1;
161-
const ptr = wasmModule._malloc(len);
162-
try {
163-
wasmModule.stringToUTF8(str, ptr, len);
164-
return ptr;
165-
} catch (error) {
166-
wasmModule._free(ptr);
167-
throw error;
162+
// libpg_query takes a NUL-terminated C string: an embedded NUL would end the
163+
// parse early and silently drop everything after it.
164+
if (str.includes('\0')) {
165+
throw new Error('Query must not contain a NUL (U+0000) character');
166+
}
167+
// lengthBytesUTF8 and stringToUTF8 disagree on unpaired surrogates, which cut
168+
// the query short. Write each as U+FFFD, the same bytes a Postgres driver sends.
169+
const wellFormed = str.replace(SURROGATE, (s) => (s.length === 2 ? s : '\uFFFD'));
170+
const size = wasmModule.lengthBytesUTF8(wellFormed) + 1;
171+
const ptr = wasmModule._malloc(size);
172+
if (!ptr) {
173+
throw new Error('Failed to allocate memory for query');
168174
}
175+
wasmModule.stringToUTF8(wellFormed, ptr, size);
176+
return ptr;
169177
}
170178

171179
function ptrToString(ptr: number): string {
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
const { describe, it, before } = require('node:test');
2+
const assert = require('node:assert/strict');
3+
const { execFileSync } = require('node:child_process');
4+
const query = require('../');
5+
6+
const LONE_HIGH = '\uD800';
7+
const LONE_LOW = '\uDC00';
8+
9+
function firstLiteral(result) {
10+
const aConst = result.stmts[0].stmt.SelectStmt.targetList[0].ResTarget.val.A_Const;
11+
return aConst.sval ? aConst.sval.sval : aConst.val.String.str;
12+
}
13+
14+
describe('Query input encoding', () => {
15+
before(async () => {
16+
await query.parse('SELECT 1');
17+
});
18+
19+
for (const count of [1, 2, 4, 8, 16, 64]) {
20+
it(`parses every statement after ${count} unpaired high surrogates`, () => {
21+
const result = query.parseSync(`SELECT '${LONE_HIGH.repeat(count)}'; SELECT 2`);
22+
assert.equal(result.stmts.length, 2);
23+
});
24+
}
25+
26+
it('parses every statement after unpaired low surrogates', async () => {
27+
const result = await query.parse(`SELECT '${LONE_LOW.repeat(8)}'; SELECT 2`);
28+
assert.equal(result.stmts.length, 2);
29+
});
30+
31+
it('reads an unpaired surrogate as U+FFFD, as a Postgres driver sends it', () => {
32+
const result = query.parseSync(`SELECT '${LONE_HIGH}x${LONE_LOW}'`);
33+
assert.equal(firstLiteral(result), '\uFFFDx\uFFFD');
34+
});
35+
36+
it('keeps valid surrogate pairs intact', () => {
37+
const result = query.parseSync("SELECT '\u{1F600}'; SELECT 2");
38+
assert.equal(result.stmts.length, 2);
39+
assert.equal(firstLiteral(result), '\u{1F600}');
40+
});
41+
42+
it('rejects an embedded NUL instead of dropping the rest of the query', async () => {
43+
assert.throws(() => query.parseSync('SELECT 1\u0000; SELECT 2'), /NUL/);
44+
await assert.rejects(query.parse('SELECT 1\u0000; SELECT 2'), /NUL/);
45+
});
46+
47+
it('loads and parses without a global TextEncoder, as under Jest jsdom', () => {
48+
const script = `delete globalThis.TextEncoder; delete globalThis.TextDecoder;
49+
require(${JSON.stringify(require.resolve('../'))})
50+
.parse("SELECT '\\uD800'; SELECT 2")
51+
.then((r) => process.stdout.write(String(r.stmts.length)));`;
52+
assert.equal(execFileSync(process.execPath, ['-e', script], { encoding: 'utf8' }), '2');
53+
});
54+
});

‎versions/14/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"wasm:rebuild": "pnpm wasm:make rebuild",
3131
"wasm:clean": "pnpm wasm:make clean",
3232
"wasm:clean-cache": "pnpm wasm:make clean-cache",
33-
"test": "node --test test/parsing.test.js test/errors.test.js"
33+
"test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
3434
},
3535
"author": "Constructive <developers@constructive.io>",
3636
"license": "MIT",

‎versions/14/src/index.ts‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -152,20 +152,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
152152
}) as T;
153153
}
154154

155+
const SURROGATE = /[\uD800-\uDBFF][\uDC00-\uDFFF]|[\uD800-\uDFFF]/g;
156+
155157
function stringToPtr(str: string): number {
156158
ensureLoaded();
157159
if (typeof str !== 'string') {
158160
throw new TypeError(`Expected a string, got ${typeof str}`);
159161
}
160-
const len = wasmModule.lengthBytesUTF8(str) + 1;
161-
const ptr = wasmModule._malloc(len);
162-
try {
163-
wasmModule.stringToUTF8(str, ptr, len);
164-
return ptr;
165-
} catch (error) {
166-
wasmModule._free(ptr);
167-
throw error;
162+
// libpg_query takes a NUL-terminated C string: an embedded NUL would end the
163+
// parse early and silently drop everything after it.
164+
if (str.includes('\0')) {
165+
throw new Error('Query must not contain a NUL (U+0000) character');
166+
}
167+
// lengthBytesUTF8 and stringToUTF8 disagree on unpaired surrogates, which cut
168+
// the query short. Write each as U+FFFD, the same bytes a Postgres driver sends.
169+
const wellFormed = str.replace(SURROGATE, (s) => (s.length === 2 ? s : '\uFFFD'));
170+
const size = wasmModule.lengthBytesUTF8(wellFormed) + 1;
171+
const ptr = wasmModule._malloc(size);
172+
if (!ptr) {
173+
throw new Error('Failed to allocate memory for query');
168174
}
175+
wasmModule.stringToUTF8(wellFormed, ptr, size);
176+
return ptr;
169177
}
170178

171179
function ptrToString(ptr: number): string {

0 commit comments

Comments
 (0)