fix(copy): raise CSV record limit to 64 MiB

README.md

@@ -865,6 +865,7 @@ COPY tablename FROM STDIN WITH CSV HEADER;
 ```
 
 This works with psql's `\copy` command and programmatic COPY operations from PostgreSQL drivers.
+Individual text and CSV records may be up to 16 MiB; larger records are rejected by the DuckDB CSV reader.
 
 ## Graceful Shutdown
 

docs/postgres-compatibility.md

@@ -78,7 +78,7 @@ why.
 | ON CONFLICT DO NOTHING | 🟡 | `dml_test.go::TestDMLInsertOnConflict` (subtest skipped) | |
 | MERGE (user-facing) | ⛔ | — | Not a PostgreSQL compatibility target; Duckgres only uses MERGE internally for DuckLake ON CONFLICT emulation |
 | TRUNCATE | ✅ | `ddl_test.go::TestDDLTruncate` | |
-| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON` | Escape sequences stored literally (documented DuckDB CSV-parser limitation) |
+| COPY … FROM STDIN (text/CSV) | ✅ | `copy_test.go::TestCopyFromStdin`, `::TestCopyFromStdinWithSpecialChars`, `::TestCopyFromStdinMultilineJSON`; `conn_test.go::TestBuildDuckDBCopyFromSQLAllowsLargeRecord` | Escape sequences stored literally (documented DuckDB CSV-parser limitation); individual records are limited to 16 MiB |
 | COPY … TO STDOUT | 🟡 | `copy_test.go::TestCopyToStdout` (skipped under lib/pq); `conn_test.go::TestCopyToStdoutRegex`; client-compat `psycopg` COPY suite | Integration skip is a lib/pq driver limitation, not a Duckgres gap |
 | COPY binary format | 🟡 | `conn_test.go::TestShouldHandleCopyBeforeTranspile`; `types_test.go` encode/decode | Unit-level only |
 

server/conn_copy.go

@@ -64,6 +64,10 @@ type CopyFromOptions struct {
 	IsBinary   bool   // True if FORMAT binary
 }
 
+// DuckDB sizes CSV buffers from MAX_LINE_SIZE. Keep enough headroom for large
+// records without exhausting the memory available to a standard worker.
+const copyMaxLineSizeBytes = 16 * 1024 * 1024
+
 // ParseCopyFromOptions extracts options from a COPY FROM STDIN command
 func ParseCopyFromOptions(query string) (*CopyFromOptions, error) {
 	upperQuery := strings.ToUpper(query)
@@ -164,7 +168,13 @@ func BuildDuckDBCopyFromSQL(tableName, columnList, filePath string, opts *CopyFr
 	// STRICT_MODE FALSE allows reading rows that don't strictly comply with CSV standard
 	// PARALLEL FALSE avoids "Parallel CSV Reader does not support full read" errors
 	// on files streamed from COPY FROM STDIN (temp files with no seek support for sniffing)
-	copyOptions := []string{"FORMAT CSV", "AUTO_DETECT FALSE", "STRICT_MODE FALSE", "PARALLEL FALSE", "MAX_LINE_SIZE 10485760"}
+	copyOptions := []string{
+		"FORMAT CSV",
+		"AUTO_DETECT FALSE",
+		"STRICT_MODE FALSE",
+		"PARALLEL FALSE",
+		fmt.Sprintf("MAX_LINE_SIZE %d", copyMaxLineSizeBytes),
+	}
 	if opts.HasHeader {
 		copyOptions = append(copyOptions, "HEADER")
 	}

server/conn_test.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"math"
 	"net"
+	"os"
 	"slices"
 	"strings"
 	"testing"
@@ -2673,7 +2674,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				HasHeader:  false,
 				NullString: "\\N",
 			},
-			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER '\t')",
+			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER '\t')",
 		},
 		{
 			name:       "CSV with header",
@@ -2686,7 +2687,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				NullString: "\\N",
 				Quote:      `"`,
 			},
-			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
+			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
 		},
 		{
 			name:       "with column list",
@@ -2699,7 +2700,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				NullString: "\\N",
 				Quote:      `"`,
 			},
-			want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
+			want: "COPY users (id, name) FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
 		},
 		{
 			name:       "custom NULL string",
@@ -2712,7 +2713,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				NullString: "NA",
 				Quote:      `"`,
 			},
-			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
+			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, NULL 'NA', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
 		},
 		{
 			name:       "empty NULL string",
@@ -2725,7 +2726,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				NullString: "",
 				Quote:      `"`,
 			},
-			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
+			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '', DELIMITER ',', QUOTE '\"', ESCAPE '\"')",
 		},
 		{
 			name:       "schema qualified table",
@@ -2738,7 +2739,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				NullString: "\\N",
 				Quote:      `"`,
 			},
-			want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')",
+			want: "COPY public.users (id, name, email) FROM '/var/tmp/copy-123.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER '\t', QUOTE '\"', ESCAPE '\"')",
 		},
 		{
 			name:       "CSV with custom escape character",
@@ -2752,7 +2753,7 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 				Quote:      `"`,
 				Escape:     `\`,
 			},
-			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 10485760, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')",
+			want: "COPY users  FROM '/tmp/data.csv' (FORMAT CSV, AUTO_DETECT FALSE, STRICT_MODE FALSE, PARALLEL FALSE, MAX_LINE_SIZE 16777216, HEADER, NULL '\\N', DELIMITER ',', QUOTE '\"', ESCAPE '\\')",
 		},
 	}
 
@@ -2766,6 +2767,45 @@ func TestBuildDuckDBCopyFromSQL(t *testing.T) {
 	}
 }
 
+func TestBuildDuckDBCopyFromSQLAllowsLargeRecord(t *testing.T) {
+	const payloadSize = 11 << 20
+
+	db, err := sql.Open("duckdb", ":memory:")
+	if err != nil {
+		t.Fatalf("open duckdb: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	if _, err = db.Exec("SET memory_limit = '878 MiB'"); err != nil {
+		t.Fatalf("set worker memory limit: %v", err)
+	}
+	if _, err = db.Exec("CREATE TABLE copy_large_record_test (payload TEXT)"); err != nil {
+		t.Fatalf("create table: %v", err)
+	}
+
+	path := t.TempDir() + "/large-record.tsv"
+	payload := strings.Repeat("x", payloadSize)
+	if err = os.WriteFile(path, []byte(payload+"\n"), 0o600); err != nil {
+		t.Fatalf("write COPY input: %v", err)
+	}
+
+	copySQL := BuildDuckDBCopyFromSQL("copy_large_record_test", "", path, &CopyFromOptions{
+		Delimiter:  "\t",
+		NullString: "\\N",
+	})
+	if _, err = db.Exec(copySQL); err != nil {
+		t.Fatalf("COPY 11 MiB record: %v", err)
+	}
+
+	var storedSize int64
+	if err = db.QueryRow("SELECT length(payload) FROM copy_large_record_test").Scan(&storedSize); err != nil {
+		t.Fatalf("query copied payload size: %v", err)
+	}
+	if storedSize != payloadSize {
+		t.Fatalf("copied payload size = %d, want %d", storedSize, payloadSize)
+	}
+}
+
 func TestCopyCommandTypeDetection(t *testing.T) {
 	c := &clientConn{}