diff --git a/README.md b/README.md index 741c4040..0aa2c5a2 100644 --- a/README.md +++ b/README.md @@ -865,6 +865,7 @@ COPY tablename FROM STDIN WITH CSV HEADER; ``` This works with psql's `\copy` command and programmatic COPY operations from PostgreSQL drivers. +Individual text and CSV records may be up to 16 MiB; larger records are rejected by the DuckDB CSV reader. ## Graceful Shutdown diff --git a/docs/postgres-compatibility.md b/docs/postgres-compatibility.md index b1d134f8..3628e757 100644 --- a/docs/postgres-compatibility.md +++ b/docs/postgres-compatibility.md @@ -78,7 +78,7 @@ why. | ON CONFLICT DO NOTHING | 🟡 | `dml_test.go::TestDMLInsertOnConflict` (subtest skipped) | | | MERGE (user-facing) | ⛔ | — | Not a PostgreSQL compatibility target; Duckgres only uses MERGE internally for DuckLake ON CONFLICT emulation | | TRUNCATE | ✅ | `ddl_test.go::TestDDLTruncate` | | -| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON` | Escape sequences stored literally (documented DuckDB CSV-parser limitation) | +| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON`; `conn_test.go::TestBuildDuckDBCopyFromSQLAllowsLargeRecord` | Escape sequences stored literally (documented DuckDB CSV-parser limitation); individual records are limited to 16 MiB | | COPY … TO STDOUT | 🟡 | `copy_test.go::TestCopyToStdout` (skipped under lib/pq); `conn_test.go::TestCopyToStdoutRegex`; client-compat `psycopg` COPY suite | Integration skip is a lib/pq driver limitation, not a Duckgres gap | | COPY binary format | 🟡 | `conn_test.go::TestShouldHandleCopyBeforeTranspile`; `types_test.go` encode/decode | Unit-level only | diff --git a/server/conn_copy.go b/server/conn_copy.go index 2f17e278..c2940671 100644 --- a/server/conn_copy.go +++ b/server/conn_copy.go @@ -64,6 +64,10 @@ type CopyFromOptions struct { IsBinary bool // True if FORMAT binary } +// DuckDB sizes CSV buffers from MAX_LINE_SIZE. Keep enough headroom for large +// records without exhausting the memory available to a standard worker. +const copyMaxLineSizeBytes = 16 * 1024 * 1024 + // ParseCopyFromOptions extracts options from a COPY FROM STDIN command func ParseCopyFromOptions(query string) (*CopyFromOptions, error) { upperQuery := strings.ToUpper(query) @@ -164,7 +168,13 @@ func BuildDuckDBCopyFromSQL(tableName, columnList, filePath string, opts *CopyFr // STRICT_MODE FALSE allows reading rows that don't strictly comply with CSV standard // PARALLEL FALSE avoids "Parallel CSV Reader does not support full read" errors // on files streamed from COPY FROM STDIN (temp files with no seek support for sniffing) - copyOptions := []string{"FORMAT CSV", "AUTO_DETECT FALSE", "STRICT_MODE FALSE", "PARALLEL FALSE", "MAX_LINE_SIZE 10485760"} + copyOptions := []string{ + "FORMAT CSV", + "AUTO_DETECT FALSE", + "STRICT_MODE FALSE", + "PARALLEL FALSE", + fmt.Sprintf("MAX_LINE_SIZE %d", copyMaxLineSizeBytes), + } if opts.HasHeader { copyOptions = append(copyOptions, "HEADER") } diff --git a/server/conn_test.go b/server/conn_test.go index 460084a3..2c8b4c5e 100644 --- a/server/conn_test.go +++ b/server/conn_test.go @@ -12,6 +12,7 @@ import ( "io" "math" "net" + "os" "slices" "strings" "testing" @@ -2673,7 +2674,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { HasHeader: false, NullString: "\\N", }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER '\t')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER '\t')", }, { name: "CSV with header", @@ -2686,7 +2687,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "with column list", @@ -2699,7 +2700,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "custom NULL string", @@ -2712,7 +2713,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "NA", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "empty NULL string", @@ -2725,7 +2726,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "", Quote: `"`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')", }, { name: "schema qualified table", @@ -2738,7 +2739,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { NullString: "\\N", Quote: `"`, }, - want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", + want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')", }, { name: "CSV with custom escape character", @@ -2752,7 +2753,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { Quote: `"`, Escape: `\`, }, - want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", + want: "COPY users FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')", }, } @@ -2766,6 +2767,45 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) { } } +func TestBuildDuckDBCopyFromSQLAllowsLargeRecord(t *testing.T) { + const payloadSize = 11 << 20 + + db, err := sql.Open("duckdb", ":memory:") + if err != nil { + t.Fatalf("open duckdb: %v", err) + } + defer func() { _ = db.Close() }() + + if _, err = db.Exec("SET memory_limit = '878 MiB'"); err != nil { + t.Fatalf("set worker memory limit: %v", err) + } + if _, err = db.Exec("CREATE TABLE copy_large_record_test (payload TEXT)"); err != nil { + t.Fatalf("create table: %v", err) + } + + path := t.TempDir() + "/large-record.tsv" + payload := strings.Repeat("x", payloadSize) + if err = os.WriteFile(path, []byte(payload+"\n"), 0o600); err != nil { + t.Fatalf("write COPY input: %v", err) + } + + copySQL := BuildDuckDBCopyFromSQL("copy_large_record_test", "", path, &CopyFromOptions{ + Delimiter: "\t", + NullString: "\\N", + }) + if _, err = db.Exec(copySQL); err != nil { + t.Fatalf("COPY 11 MiB record: %v", err) + } + + var storedSize int64 + if err = db.QueryRow("SELECT length(payload) FROM copy_large_record_test").Scan(&storedSize); err != nil { + t.Fatalf("query copied payload size: %v", err) + } + if storedSize != payloadSize { + t.Fatalf("copied payload size = %d, want %d", storedSize, payloadSize) + } +} + func TestCopyCommandTypeDetection(t *testing.T) { c := &clientConn{}