From 6d4d327598702b5508531bbcf0e5057a8c54f3fc Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 9 Sep 2026 08:21:28 -0400 Subject: [PATCH 1/2] fix(copy): raise CSV record limit to 64 MiB --- README.md | 1 + docs/postgres-compatibility.md | 2 +- server/conn_copy.go | 10 ++++++- server/conn_test.go | 51 +++++++++++++++++++++++++++++----- 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 741c4040a..a7bd38dc0 100644 --- a/README.md +++ b/README.md @@ -865,6 +865,7 @@ COPY tablename FROM STDIN WITH CSV HEADER; ``` This works with psql's `\copy` command and programmatic COPY operations from PostgreSQL drivers. +Individual text and CSV records may be up to 64 MiB; larger records are rejected by the DuckDB CSV reader. ## Graceful Shutdown diff --git a/docs/postgres-compatibility.md b/docs/postgres-compatibility.md index b1d134f8f..58d6bb7b4 100644 --- a/docs/postgres-compatibility.md +++ b/docs/postgres-compatibility.md @@ -78,7 +78,7 @@ why. | ON CONFLICT DO NOTHING | 🟡 | `dml_test.go::TestDMLInsertOnConflict` (subtest skipped) | | | MERGE (user-facing) | ⛔ | — | Not a PostgreSQL compatibility target; Duckgres only uses MERGE internally for DuckLake ON CONFLICT emulation | | TRUNCATE | ✅ | `ddl_test.go::TestDDLTruncate` | | -| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON` | Escape sequences stored literally (documented DuckDB CSV-parser limitation) | +| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON`; `conn_test.go::TestBuildDuckDBCopyFromSQLAllowsLargeRecord` | Escape sequences stored literally (documented DuckDB CSV-parser limitation); individual records are limited to 64 MiB | | COPY … TO STDOUT | 🟡 | `copy_test.go::TestCopyToStdout` (skipped under lib/pq); `conn_test.go::TestCopyToStdoutRegex`; client-compat `psycopg` COPY suite | Integration skip is a lib/pq driver limitation, not a Duckgres gap | | COPY binary format | 🟡 | `conn_test.go::TestShouldHandleCopyBeforeTranspile`; `types_test.go` encode/decode | Unit-level only | diff --git a/server/conn_copy.go b/server/conn_copy.go index 2f17e278f..c1582eb4a 100644 --- a/server/conn_copy.go +++ b/server/conn_copy.go @@ -64,6 +64,8 @@ type CopyFromOptions struct { IsBinary bool // True if FORMAT binary } +const copyMaxLineSizeBytes = 64 * 1024 * 1024 + // ParseCopyFromOptions extracts options from a COPY FROM STDIN command func ParseCopyFromOptions(query string) (*CopyFromOptions, error) { upperQuery := strings.ToUpper(query) @@ -164,7 +166,13 @@ func BuildDuckDBCopyFromSQL(tableName, columnList, filePath string, opts *CopyFr // STRICT_MODE FALSE allows reading rows that don't strictly comply with CSV standard // PARALLEL FALSE avoids "Parallel CSV Reader does not support full read" errors // on files streamed from COPY FROM STDIN (temp files with no seek support for sniffing) - copyOptions := []string{"FORMAT CSV", "AUTO_DETECT FALSE", "STRICT_MODE FALSE", "PARALLEL FALSE", "MAX_LINE_SIZE 10485760"} + copyOptions := []string{ + "FORMAT CSV", + "AUTO_DETECT FALSE", + "STRICT_MODE FALSE", + "PARALLEL FALSE", + fmt.Sprintf("MAX_LINE_SIZE %d", copyMaxLineSizeBytes), + } if opts.HasHeader { copyOptions = append(copyOptions, "HEADER") } diff --git a/server/conn_test.go b/server/conn_test.go index 460084a39..8af4dbfe1 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -12,6 +12,7 @@ import ( "io" "math" "net" + "os" "slices" "strings" "testing" @@ -2673,7 +2674,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { HasHeader: false, NullString: "\\N", }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER '\t')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL '\\N', DELIMITER '\t')", }, { name: "CSV with header", @@ -2686,7 +2687,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "with column list", @@ -2699,7 +2700,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "custom NULL string", @@ -2712,7 +2713,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "NA", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "empty NULL string", @@ -2725,7 +2726,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "schema qualified table", @@ -2738,7 +2739,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", + want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", }, { name: "CSV with custom escape character", @@ -2752,7 +2753,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { Quote: `"`, Escape: `\`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", }, } @@ -2766,6 +2767,42 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { } } +func TestBuildDuckDBCopyFromSQLAllowsLargeRecord(t *testing.T) { + const payloadSize = 11 << 20 + + db, err := sql.Open("duckdb", ":memory:") + if err != nil { + t.Fatalf("open duckdb: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err = db.Exec("CREATE TABLE copy_large_record_test (payload TEXT)"); err != nil { + t.Fatalf("create table: %v", err) + } + + path := t.TempDir() + "/large-record.tsv" + payload := strings.Repeat("x", payloadSize) + if err = os.WriteFile(path, []byte(payload+"\n"), 0o600); err != nil { + t.Fatalf("write COPY input: %v", err) + } + + copySQL := BuildDuckDBCopyFromSQL("copy_large_record_test", "", path, &CopyFromOptions{ + Delimiter: "\t", + NullString: "\\N", + }) + if _, err = db.Exec(copySQL); err != nil { + t.Fatalf("COPY 11 MiB record: %v", err) + } + + var storedSize int64 + if err = db.QueryRow("SELECT length(payload) FROM copy_large_record_test").Scan(&storedSize); err != nil { + t.Fatalf("query copied payload size: %v", err) + } + if storedSize != payloadSize { + t.Fatalf("copied payload size = %d, want %d", storedSize, payloadSize) + } +} + func TestCopyCommandTypeDetection(t *testing.T) { c := &clientConn{} From b7fbe20db1ff230d06bd68bfca3ae62064ad170b Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 9 Sep 2026 08:59:12 -0400 Subject: [PATCH 2/2] fix(copy): bound CSV buffer memory --- README.md | 2 +- docs/postgres-compatibility.md | 2 +- server/conn_copy.go | 4 +++- server/conn_test.go | 17 ++++++++++------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a7bd38dc0..0aa2c5a24 100644 --- a/README.md +++ b/README.md @@ -865,7 +865,7 @@ COPY tablename FROM STDIN WITH CSV HEADER; ``` This works with psql's `\copy` command and programmatic COPY operations from PostgreSQL drivers. -Individual text and CSV records may be up to 64 MiB; larger records are rejected by the DuckDB CSV reader. +Individual text and CSV records may be up to 16 MiB; larger records are rejected by the DuckDB CSV reader. ## Graceful Shutdown diff --git a/docs/postgres-compatibility.md b/docs/postgres-compatibility.md index 58d6bb7b4..3628e7575 100644 --- a/docs/postgres-compatibility.md +++ b/docs/postgres-compatibility.md @@ -78,7 +78,7 @@ why. | ON CONFLICT DO NOTHING | 🟡 | `dml_test.go::TestDMLInsertOnConflict` (subtest skipped) | | | MERGE (user-facing) | ⛔ | — | Not a PostgreSQL compatibility target; Duckgres only uses MERGE internally for DuckLake ON CONFLICT emulation | | TRUNCATE | ✅ | `ddl_test.go::TestDDLTruncate` | | -| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON`; `conn_test.go::TestBuildDuckDBCopyFromSQLAllowsLargeRecord` | Escape sequences stored literally (documented DuckDB CSV-parser limitation); individual records are limited to 64 MiB | +| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON`; `conn_test.go::TestBuildDuckDBCopyFromSQLAllowsLargeRecord` | Escape sequences stored literally (documented DuckDB CSV-parser limitation); individual records are limited to 16 MiB | | COPY … TO STDOUT | 🟡 | `copy_test.go::TestCopyToStdout` (skipped under lib/pq); `conn_test.go::TestCopyToStdoutRegex`; client-compat `psycopg` COPY suite | Integration skip is a lib/pq driver limitation, not a Duckgres gap | | COPY binary format | 🟡 | `conn_test.go::TestShouldHandleCopyBeforeTranspile`; `types_test.go` encode/decode | Unit-level only | diff --git a/server/conn_copy.go b/server/conn_copy.go index c1582eb4a..c29406713 100644 --- a/server/conn_copy.go +++ b/server/conn_copy.go @@ -64,7 +64,9 @@ type CopyFromOptions struct { IsBinary bool // True if FORMAT binary } -const copyMaxLineSizeBytes = 64 * 1024 * 1024 +// DuckDB sizes CSV buffers from MAX_LINE_SIZE. Keep enough headroom for large +// records without exhausting the memory available to a standard worker. +const copyMaxLineSizeBytes = 16 * 1024 * 1024 // ParseCopyFromOptions extracts options from a COPY FROM STDIN command func ParseCopyFromOptions(query string) (*CopyFromOptions, error) { diff --git a/server/conn_test.go b/server/conn_test.go index 8af4dbfe1..2c8b4c5e7 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -2674,7 +2674,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { HasHeader: false, NullString: "\\N", }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL '\\N', DELIMITER '\t')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER '\t')", }, { name: "CSV with header", @@ -2687,7 +2687,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "with column list", @@ -2700,7 +2700,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "custom NULL string", @@ -2713,7 +2713,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "NA", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "empty NULL string", @@ -2726,7 +2726,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "schema qualified table", @@ -2739,7 +2739,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", + want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", }, { name: "CSV with custom escape character", @@ -2753,7 +2753,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { Quote: `"`, Escape: `\`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 67108864, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", }, } @@ -2776,6 +2776,9 @@ func TestBuildDuckDBCopyFromSQLAllowsLargeRecord(t *testing.T) { } defer func() { _ = db.Close() }() + if _, err = db.Exec("SET memory_limit = '878 MiB'"); err != nil { + t.Fatalf("set worker memory limit: %v", err) + } if _, err = db.Exec("CREATE TABLE copy_large_record_test (payload TEXT)"); err != nil { t.Fatalf("create table: %v", err) }