From 35e4f88d9b9ccd0f4de7a8a55ca747268b498124 Mon Sep 17 00:00:00 2001 From: Miroslav Pokrovskii <7105861+mirpo@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:23:04 +0300 Subject: [PATCH 1/2] feat: unify image input into read: + an image: attach field (D.1 Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the imagePath prompt-step modifier (which self-globbed and row-indexed images in one field) with a cleaner two-part model: - `read:` enumerates image files into rows like any other files. - a new `image:` field on prompt steps takes a templatable path (e.g. `image: "{{.item.path}}"`) and attaches that file as a vision image. This composes with forEach and matches how everything else references rows. Removed: config.ImagePath + Step.HasImages; utils.setImagePath; the resolveIterations image-count branch; fs.PickImageFile / fs.CountFiles / fs.globFiles (fs/images.go is now just ImageToBase64). promptbuilder gains RenderString (BuildPrompt refactored to share buildValues) so the image path renders against the same row context. Migrated the vision example to read local images (ships 3 sample .jpg) → forEach → `image: {{.item.path}}`; dropped the HF download + magick shell steps, so it's self-contained. Live-verified on qwen2.5vl:3b (3 images → correct color/shape/chart descriptions, 0 retries). TDD: promptbuilder RenderString, prompt-step image attach + missing-image failure, preprocess read-rejects-image; full suite + golangci-lint clean. --- config/config.go | 6 +-- examples/v1/vision/README.md | 16 ++++--- examples/v1/vision/config.yaml | 55 +++++++++++++------------ examples/v1/vision/images/barchart.jpg | Bin 0 -> 5297 bytes examples/v1/vision/images/shapes.jpg | Bin 0 -> 5777 bytes examples/v1/vision/images/triangle.jpg | Bin 0 -> 4868 bytes fs/image_test.go | 33 --------------- fs/images.go | 46 +-------------------- promptbuilder/promptbuilder.go | 25 ++++++++++- promptbuilder/promptbuilder_test.go | 12 ++++++ runner/runner.go | 9 ---- step/prompt_step.go | 13 +++--- step/prompt_step_test.go | 33 +++++++++++++-- utils/preprocess.go | 22 +--------- utils/preprocess_test.go | 15 +++---- 15 files changed, 121 insertions(+), 164 deletions(-) create mode 100644 examples/v1/vision/images/barchart.jpg create mode 100644 examples/v1/vision/images/shapes.jpg create mode 100644 examples/v1/vision/images/triangle.jpg diff --git a/config/config.go b/config/config.go index 9cbedb6..eef60f8 100644 --- a/config/config.go +++ b/config/config.go @@ -82,7 +82,7 @@ type Step struct { ModelConfig ModelConfig `yaml:"modelConfig"` OutputFilename string `yaml:"outputFilename"` JSONSchemaRaw interface{} `yaml:"jsonSchema"` - ImagePath string `yaml:"imagePath"` + Image string `yaml:"image"` // prompt steps: file path (templatable) to attach as a vision image ResolvedCount int JSONSchema jsonschema.Schema // JQProgram holds the compiled jq program (set during preprocessing); @@ -118,7 +118,3 @@ func (c *Config) GetStepByName(name string) *Step { } return nil } - -func (s *Step) HasImages() bool { - return len(s.ImagePath) > 0 -} diff --git a/examples/v1/vision/README.md b/examples/v1/vision/README.md index fda80cd..0e146b6 100644 --- a/examples/v1/vision/README.md +++ b/examples/v1/vision/README.md @@ -1,20 +1,24 @@ # Vision -Transcribe handwritten-math images to LaTeX with a vision model, then explain each formula — a multimodal dataset pipeline. +Run a vision model over your **own local images**: `read` enumerates a folder of +images, and each prompt call attaches the current row's file as a vision image +via `image: {{.item.path}}`. Here each image gets a structured alt-text record +(description + colors + tags) — the kind of set used for cataloging or +accessibility. -**Features:** `imagePath` · `shell` · `forEach` +**Features:** `read` · `image:` (vision attach) · `forEach` · `jsonSchema` ## Steps -1. `download_images` — `hf download` + unzip + `magick` convert (BMP → JPEG) -2. `to_latex` — vision step over an `imagePath` glob → LaTeX transcription -3. `explain` — `forEach` → a step-by-step explanation of the formula +1. `images` — `read: ./images/*.jpg` → one row per image: `{path, name, content}` +2. `describe` — `forEach` image, `image: {{.item.path}}` attaches it → `{description, main_colors[], tags[]}` + +Point `read` at your own image folder to process your files. ## Requirements - `datamatic` - [Ollama](https://ollama.com/download) (or [LM Studio](https://lmstudio.ai/download)) + a vision model: `ollama pull qwen2.5vl:3b` (or `gemma3:4b`) -- [hf](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli), [magick](https://imagemagick.org/script/download.php) ## Run diff --git a/examples/v1/vision/config.yaml b/examples/v1/vision/config.yaml index f2f3671..2d4e4bb 100644 --- a/examples/v1/vision/config.yaml +++ b/examples/v1/vision/config.yaml @@ -1,33 +1,36 @@ version: 1.0 +# Vision over your OWN local images: read a folder of images and describe each. +# `read` enumerates the files; the prompt step attaches the current row's file +# as a vision image via `image: {{.item.path}}`. steps: - - name: download_dataset - type: shell - run: | - # download image dataset from huggingface - hf download --repo-type dataset Azu/Handwritten-Mathematical-Expression-Convert-LaTeX --local-dir ./ --include data.zip + - name: images + read: ./images/*.jpg - # unzip - unzip -q -o data.zip -d data - - # convert all BMP images to JPG - magick mogrify -format jpg -quality 90 -path ./data/2019 ./data/2019/*.bmp - outputFilename: data - - - name: analyze_math_image - type: prompt - model: ollama:qwen2.5vl:3b - # model: lmstudio:gemma-3-4b-it - prompt: | - Image contains mathematical formula. Convert to Latex. Don't add any extra information - imagePath: | - ./data/2019/*.jpg - - - name: explain_math - type: prompt + - name: describe model: ollama:qwen2.5vl:3b # model: lmstudio:gemma-3-4b-it - forEach: analyze_math_image + forEach: images + image: "{{.item.path}}" prompt: | - Given the following formula, break down the steps needed to understand or solve it. Start by identifying the type of mathematical expression it is (e.g., complex numbers, exponential form, etc.), then explain each term and operation, and proceed to derive or simplify as appropriate. Show your reasoning at each step. Format your response in clear Markdown, using headings, bullet points, and LaTeX for math where helpful. - Formula: {{.analyze_math_image}} + Describe this image for an alt-text catalog: what it shows, its main colors, + and a few searchable tags. Return as JSON. + jsonSchema: + type: object + properties: + description: + type: string + minLength: 10 + maxLength: 300 + main_colors: + type: array + items: { type: string } + minItems: 1 + maxItems: 5 + tags: + type: array + items: { type: string } + minItems: 1 + maxItems: 5 + required: [description, main_colors, tags] + additionalProperties: false diff --git a/examples/v1/vision/images/barchart.jpg b/examples/v1/vision/images/barchart.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ff82a1f8bc08c8d5197f789a61760928eddf1e9c GIT binary patch literal 5297 zcmeHKX;c)~wm#4xlZXf{AZkBEYyyd>I3|WNBBCZ&L<2a|K{TKkQ5-;~G6sX9*a1bv zVYNaG&aGjTsRU(EG%X^dLetC+*w8dh4^?&EDe`jf1$}FI?|Z-Adewbut?E8!*Z%hY z_ILJy^{@quU9oieQlO;;fEM}y_yV{BmbNzYMVAix>gwz2>gebi>gkQrA8j~#w2`5a zk@1*uV~xj{jxjPCJASOGnK_%y9&KV_Io{lIoH^T^F`~snYjkuCbaf5Pjg5@W|K$tb z1E%_*LTeFAYYNad)nb`y!3ThY{M6MNy1;*3TH46RQF{6YhDK;Y)>xpe#bRmeuo$n= z>?rg<&@t5=XEXDwQD&ZgdQ*0r&-yXplD_TYJH_nPRlQT4H-{WDFdT1TX*I!a8rS|4 zhuL%H&YQpBbJrzqOWl_(U$N#}FK?f<>(={k+4_Az;I{31_U_vs8g?N3rs-M@?*3~yON?P06 zJ370%U%l>=^~(nogG!Z}!KDRQf5JldKY@+FWr}cV>*%m_^cY-P+M#H$Om%c^W{w*7 zm8YKHZnG(~e$+Q#oN(z*v4O4g>R$HdkSfFRQ)f%;`WR?K$UY9(p??e6-+}!DS0fnB z(n85&nF0(@Yf|Iv!T-I{%WsUOw+b`|AZYYz31|>1rfu?>(caabot!1LoAg#A_OE%= zi#4vT@LO>Afbgd9_u!ygO`_1$BGXjD27)8kAy~yzt$`rZ1p;a*1Xl#)NeHrC-Z4}7 z$~a;`7lI^Avj_rD;x!k7sSxz=^REbFf+Fr=vM7GQJx4_H8*HKL;KW&A2SwKmSLIc_xT_!vHz>O0(XV+ z`s#oS!3yh?xjWtVsS;e@{I*;Z6VR4jK!*N3Lt2k)0f zmO1v{@G;%oz1d;)asR;V?5Iikf}*VjtkeE8X3w1C@a^jJw%h>gO}{o3Hu0-HlskAa z9Qi!FV^%)|$EzSPKGypa{j-$(%@^-GOID`dC2AuBWbW^9GT{~eVE!Y#tH+}VBY#wH z8!;D?+=(8GKW&2GRHcY^=B4cQq%+#EL=3sr{u6Cewgv)hdsH8KGN~bAA(%`W^m(j1 zx4gF~pGv>ty0gy1q`oWG^k=WnQfxhIp8GH8EYv12qD4T+*6kB$z~3OaJibQF;su>* zbMWtdl=bb&{Pp-ZWav@ZS*z)rd(Z5Dx!R*SBV28#(lz(=eJJxAp6bHp0$ar;2vP#* zW?YdJsW%yaxZo(F1txFzxF*3>)(CzMAy1iX>SpdEv(of>x|sv-!Gm~vcyAXKbCZy7 zuQd)1)1%#mWAOSFFCbWYGu#T-ba49I``xEIQO)T}UVGFfw0dwISz_+#20ms)Z>8%A z+3x1)j4PDwp@PkY0!1kVCr|W+Qg_XV4F{VT}j7+2L>A(&IG5K%V}(%jdXH#>f~FrjIePBOb>)hu!&|qcWFZ zZ(d^ZQz#+3CU>L6YRT(JvqAdn=@}4wXF$kP8;M0h{TkXp1A$#KQj6O00KY9 zU}{2E_f`m`r|?qFlZm;+V7NenCnydh15E+_`V2*fkLc+>&$py-6{1Y4i))wXJ4l>; zd%%*!pC5>7-WfEIT`P+cHGb~c7ZRuD4{+&JOy)t?%kUgTw7ND30F{R27LJ-IWLpTn^eKu{?ZbfZPl7bA}_nfB?PN1Cb0_ zoD@9Ui98>AjOYiZcVaH?zXg+hQi9jz z;;-ycrM7&Dc4d3kE8%<5?izf*qy}~(G1%pMgZ4+{QcbEPXOf5F?1kyre8XYo@@`;Q zMq1|g)5{-wFi}XK66IrrM5LKd6{2dd=#8^aC4Lc!k(rCyBRAsPBRY(yl@N&SA&A4y zF4&E~8^B7CaP=T65_zJ|u?WYo)`B5-8gp|!gm>C9gnYqIL^0~96Umu1t6Y|#0074i zgj+C2Z47}9s*=Q+H}U%DU}{>nFuql!8jtnaHa#So>(X2ZkLL&ep7~#J>?GrQ2oB?QZv(!l2HtVhD-auUP4+TmPzQZr z&rnf_3uq)(-hiN9JD9T17CxOoZHUEQT3pA@AePYXArOql{{vCtmFN%cVXa9-;qaYB zIc6a;wa1K!#j6T|7vivNUK(m(@mI4JPC?bw?mXDSaCyXTdt<_g?O2ZFckmI`%I`d} zk*6v2=p`yNi_!Xb8LAit%U?q#JRvZ3;VO+ZW;DCJ1XHdT1ohKm2-u(9;wy}ccu91j z$2nrJAoQIol9Kp(p~*rc!PTOIbe zXQdKTvSql8+m4aW_)v0fA#gJOE4kt!n&=~%-2am%eQuok1)-#x+)De= z`A8Gws3sC!(f*9h!2+bCqfs6Twnhl?`|}|9f%>iZZ8TDEgWBZaoV^J}Jq?i%TrZm` z7P93g_Af78Cm9crZr=Q)arPu|Z^E*9r9k4le5hvDz> zGSpTFL>)tMID7bVB@r!B_*gbbYl}Kl3&?1eaf;+}OlkSq^EdbH&)#-(jmO@+3t=0* zcCZ8ME;%mWA&!j+A83?j$kqkpO5+0d8J$%Tct z4`3==RKDw-aYmUE%bAGHBkthH@3|tfNKW*nX)K0-p{!I&Q9LG4HmUpplUS@9aglo! zwPaZ~KsH$+p>v&IXsPw&GhOT^0NK!}$?z}AD-GP@xT)ssex3=oCD&OeS zoy5NEq9=L1CVrXUjq1F?t~}J@91)m(Uwtg*dLTLtaOfo;8)0F-%Xc4|E2Rx1y_BVG z0`{i0k&Bg95Exdvop{(4>jObdp2)jD-DJXz$=^6wh1|3aUTESpX{}FI)uN$pj_K(B J7Y7G6{3n_`Y|j7y literal 0 HcmV?d00001 diff --git a/examples/v1/vision/images/shapes.jpg b/examples/v1/vision/images/shapes.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a21f3b3b9ca3ef9876f9b5626c95cff3ab6b6d4c GIT binary patch literal 5777 zcmeHKcT`i^x4xlw1Ow<;q5{%%=pcdwbm#*JQdPVISU`p@1Q3WQpdd(*B1I!jP{61d zf`CMN8$b~0C7~!y2|a`)_jj4{do$y#x88f}{rBF<-S@7u?!G7cJ7@3jdD#yv0Q+yS;QMdD{)S5g#>K|L!Op?GjSInsfsI{+gY%#=m#DrKx3llgLn;wG$dgGq zFKc-vRjsLFE`Hs7;!7{LGzasmu0lWD>LiTT9|AA`=5MW2Zi^nbkpaJHY+*n!Q z|E9GpUAe1^{kR|x5%Ei zP@8Rhqj@o{l=#Z5`10Fe#U{m}1j^@Hl*+hMpYCAE6ZC{hLBt7iXZ_0ala0$Q&vly2 z+k{3NTxv9!1!(Vmm?q)|Yd`HeM&lOc!IWO*$2ZA>C#gA7uU;Ie-?^{iG?f@apKXx3mGxE zo0;=v{QBbLaS6oa?r9)`QT2&LqD@QVvJSvxZ{Y)b0{LFsknVB>rSY> zV4QVJk928Gx|?g1iPWL=mif z(t-Pu0!~9e+3$%SeLQt5DGn9yR$!^x^Z^rZzk*?Wb-zpY+TB#2{G2Tl9F4uTMQMY8 z7N!@?Z0jOJl|~A$47m4HBdWMnCY|dc{?NHr9}@%c=q)v`UpkQo!lt4(eN-<{z)1B~ z%)I+m6mR=hCgo#l^2&1>k61XtC} zc8>UF->W_>Ual=-$$L9NZO+6{!2lz3@%cjP@Z7?W`MKGTT2|W>R*YCEbqKJ*LcsUl ztjWB;^zc2T2?<{;6M>yJ1sNI5Ya{5DaA8J5&)k3hT^{QoMgjufOR#JTAfWYjzYhd- z(9p|hvm$&4iWb_>w1NP}8`9Rjguiy8MnDMA12?VHA>c5|sU89}$=j>?eJ)82OIZQe zj38k769jxcj{a-!Phma?!w56RZ$W_MYGo{=8v<^3U_YWDpgju$DrUav-f?P#fK^Ti zNR`7bJO6JW6k;Zl-0`%lr0B}@yI9H|2#5q1LaHml0oLex$JW7UP!*fLU-h%A7y7o= zXZ8_~m6CcyOxu#Jl0(aK@2%IVAIjFu@oygbBxl>?mGQ~uRxj7Z%06a;jgKCkmb zejGesf8~KtWd`^EG)l7j&IX`?T}Tc%E-0ZR?#$ITfXWOEfgwP9D0 z$n(Y$!v2Tur@5F#j@~RXYAfl&6=2D^*u7rWGL#NY%KF{w>Q8b?oBGe@Cn_6?Yh84R z8E$K%tPmm^q#eZW<2oy8dm%vCFa4w3TZzW`Vfy?G_I!|0?fX1^UvU-9?5|G?d$hH_ znN~4UqPYs1MEt$kT}5jr(bMCu!eW!VTOGPjwHon5;$jY&`OM^7c2K9 zoRAO==?B>nZl*S zKkCoE14Xq~+*Db-gWC{-fDiGdVTzy{w$_+5)n0Q5JE9E%78-=L^D+=1E#w3N;8X0Z zuM(cIn~rW_Xy?W%Fy(ZQ$>zoae>Y|@%*H3-S|QNVSxv(@rsOMpMKNWZCZ)*80%NcX zo^K$aM_Ya?mekpahJfTe6iahca{dMBUBY841F_A`UYIBVq}7FUTh>8o{dL%~<_7-x ziciUM*D^|wdI)FfmMxu&Qwb?L&n!dPZ&-566U5~Giq=g?6^rkjr4t&vYwJUjYSR_@~sa>?GkYnn(iHy{ejTIPc z;rCQV%WDn&q2&AT(uAUP)WzKNWeWzSFV>KAW95Y&|5+&%)48g`C}9}$zH6wqnFU-evt*H6g>SC#_C;fM)&QCW1XLynLt9vHf!({rry#5Wp3lu#6U($&G~o zQ_FjB9pCu7p)X*`iKJ!EaA^p5S^gOUj2DW(%F{3`UVfPW`X|7xJzm(=du=rn_?5B* z2$(8rhJbjMF?~UiDmep(79JkJiL}z=3_^cthrb|9dT;r`c`_fe5xWcyHP?+(AB1;a z(0#FGKdAFiogl2*K8`C>8ChsD8e0zhT)se_b~l(h+c~tAG;wBIglVzBe$= zFvb9XQIjZ;PRz$5%kx65Ky@1Ed4@_8(c0)*w*yWA7E0)u(CZp z<|-^%rhBVcBx`sACY(W<>MAyghJg#1ZXN3LM*;uoiJ-4Q07n!A-1k^gWX2QGbfZh4 zuYEmGc%g3XP_jcO%)o#uB>E6w+2RhxT zEPGG|Pf)*B)$WGRdd0AO4@N)YCeYleTlHh&ua&CLny1*6s_ozr4;C}Zkkhl+J?{U= zq+Jx&kmd$rTSr`S#CE}1Vg2^%gUm9#?U=PEag-RyqQYbnx4jLr{logthHU1s_zd7G za9Y~QqKwmh3HWCe^ro!eBAp3KGP9T{EV-+jWkeZ8T7k1BOaGYD^IZ&436?p&fwZnM z0s*u8Qo|ZSnCWW|VyA`}@G!2PU96-*`w?4Kpf9XpQg=bX5~&sC6vg5QgG0l2b^gJ2 zY|*3~2U_0A-q)TM!mnp5<6ozxpZ31Eawy64_*ZM5wqz9^$(Q&R{18Y!y73NmXXCix za1~D2DIBJO-mg|hLoV-*SFT@uZ;!8& zwgr03socY;G?QVG`xXM$Q<#YiGpalb0_P<$Za zDsEwm`-#fOsCmoxz`ZiS^>V`3kz$S7D-I5mK4Qq(_GdF2W#^vYpM}5^ppd-=S1>GD z6#8n|SRMNnY0PK%bpzr+^$EeYTI}-?*Vgj3w1YN@mwX6jI9*`anJ_fGHLqc5++14Z}TJE zxRtV&HQpKs0je~QhW*EJ;uEf?_=70!RgEX6kG>gN?NP`j+4aChM`vH&vCUuG_D>vE zeS4ii(=TBJGjXA|WKeiyU~rC(AJmB<$n84-%75LR<$p+9I8B3{FIckhEdBdE4c2v6 zf{qvj`_+lLw-R5Ox)YZT)$gZjNx#jYaTd4ZfdJj37~UVRB27h1Kp!h6@Db zq`w=dpClFFQ9k*Fjlcl{!EDwOom0|FFu_vO}1eCwKmtELkTQ|yk(k{}>C8{QidkniQjhcr5e)u>Ls=L9`Upor{ zZC$tB@D-%LyHRJrIB->k1}kRvMf9dwSao1oLJamYDl&u=RedI`0GyM^oEpmZn^`fFIw?aW z#<5+zr&^^%+||VM`MK^r{DhEdlkNh5uCW4|cAx)9?e(>q zDn<6M5nYGc{7af8@oU-T)lb`7XBhd9k}sI6g@ZwpEZ)ycXX-H&wPV*7B^*tpaYGKh0Lg{;;SA(C1i{X$=q_9G*za!uq>w){}NtmoKYX^wED@F?2NHD7jPM z6hui$G&+z3mR+;&w59xV&8dAETb|vLNPHdh6~_0x6#BRLlzw~ zzG@MA4&JV{@hr^?1r$6D7ROCT16gL*zerk8r_6RII$aG(vD{OxH^BKOmALnQTSS!3 zXMj~?Wr`D|Y^W^r;1X?WD&pgr%1^F={V=-zwB z8V}PBR@_?XGe#I%zrXD&$yUOHk!_>N`exGaTYmU*EsNh{&mOm*7fSOSekv!%N%@LE`F-nK`<(Av`*-%<=j_cH z=1c&hR_2!G00aU65U>Lr8gLxof>D!MOQ(d3oT1{DOi4`~m_(!s4Pr z!eYV#0-{o)ViE{RNl8Hw=^avt9pVT{#J8V7xWI2<+!iN^YIIS3u;6GD1?g(3gh~wHMklB&I2$p zZgGYEraTh19=wWZgih4Ge7MrFC#{lplqF@|^ZwC%{8G|8cJ5NyqpGH^aX{~&{vm_I z$4{7n{}Qv`iTx)pDj>)O0gcBc1{ecu%>tYz zePxVatswIMHYzhYfUgKudt1^g2Gw>}wsVk_u$F+<7^8*fderQ*FA_H09KTR-yIuPw zGws%BwymQVkD-jUc6+ac^Cx8Is`abfA8p-tM6~5ryRWA|Yi`#T9d&iNtp2Rpx*Qu) z0!v2MC<_u>l6su(k<`%BtLkY4YnKrM$Y zn}atplV5(w1B5{bM}QrSPS@t=9N-R>^c*QWc9|3`PQ2r%e9v1-xKWBaN`jSLl)C^F z(^Qqu$yVfr7NcLgykrT)Rn=b{PO|db_?gz%h%NQ}{9?h6c)fA{Mz3MKueR2VbADY^ zkEKL;&(mLLL`~$AOeMEsnc5MQFuyJi0B`jUM9+rwo+6nIJocpSng-Mx64ijh-Qs}E z&f0s?iQ^eH1FQoKd?ibT9#|SjHVj_(`FPavPU|J2qViRoefQ{{sOKhJ05yHHVC&~bMx7TFp*~d%|b_2J|d;aO5Ri2!ei``E^PYrH7Qi6iA+Jde^A2= zx?4V)&yMdiffTSMo@3z)Pf!vXkGc44@gYR8q@C+&k=*RXgE=J?jhylxa1voV> zTC?X@s3b5p3=*biQ8@DUnoe>6o$)3!zXUK#}BRtFQ_xkIKahG6ibIL)cLsS0h4y30l#rY zHB?{5&ZOOM-$HSm;nPgtkA!CiH1mk`w$SPJSLgQYY^W?X^^po$$np()gsrRUf`=L6 zXUJcHn>Evt6~V$?j`p8*bcxwtDpR{`tvDOJDBIj&EceM+ z&`7!>q!1N)`p370BJ$BkmJKe#;|d3J(#^=&$blu?aE*|#{tq_=L&-~)%(&fh5Lzn`Do@ZlpDYA^HX7s7p$y$Au*VM3U?Pb zHy7*RP_)u1y^cjUOV=dl%w**hOXX$DGPQzwkye!N*lo~{K-=U&L5TE!9iCo~>Yj) zirnon5F>kB@V0^N;7`F0UW-QB5BFk_6+_r(I1bQ)$J$phnK^3+TNE+`VQU8bdvF)q z9yP9R-s+2W6B}&)P~@giSBo^!k}|=Lub3dmj*ZWYm)L=4ZhGe28GrJDCz27?1Pp$D zP8vp+=Ndeb)_KZB}YZYAQ*iQNq<3%{H9Kq2_-l9d{94=vbh z<)N;$qpX_~ZgCmE<~alIX%YI{=Kda}Vfckl^sbEw#|{|`DsZstW&ktC@u%8A~wF!A*s7fbD)N`^({= zd_UQ-YMOI+cg}Gok4Po?o6(Rp!r?(wWTNLrUtUW4c_u zsmwZet;Xp59#W5BBKE}mQ=K_S;{yNicNu;$_W4B_DXzJjN29>&;^D;iUlnK$Al0Pm zZ43|RD`(&xvQ$tZ!)Ixhd%DPV8b4lBfFwC3<;mvh$|3Gv@p5>i)@H9^Xr@V;2_5?4 z$~=JzmsoPC4Rc+!p2k@Ha9}=QM_SA6UpK_n4*@`{{x7;w%TEc@E5ch_AI>z@k;!!# zi_25ySw8Wj(?36p7f!~ zwWh}n@5bF~*{=zuC_`jbZ;E8()ngXYVeAQ=-g=x~SWb{?;C1YYNjidlrqC^c{0k9u zkzYZRy^aKrSm3__RA7GIcVExc^753XwP6U=o2Yv2UI$L+<;xHNuvCrYj&O|VAFtN> z`uwZ3>9yMNk!`uRM%}B8`t~cw>Rp$u+KmktY|>;a5RlaR>|x`p%~a&JxYke8lO=KJ zQtu(YJ%k&GO=;bV858J1esk2ENZS`PwVHU| z{gncSi4*d{@4@nqf%Erw*^O3Y;#pG5S6Lb(*(ye}W`DL%uR$FuJIM2*$*_vBt%b*% zvTl!bW5RnlJt>>c+jiI{ZZ$r9%+QWx9rv`yS;ycc)k3&bqeUXd&d_mM79+vj9q|SX zl`Ve(N5hYW2|#(UmTA1o~Nktf6H6Pg%@M)pAN zHk7$_BzJt2tqta?W$X^M0vzzKpgD)79bX4M>9+G^#!^NsI?#8DeSXx66f~|n>}#Fw z&2={-c=%)fwceGZvkLlQxzCYFhk08n;kA_n64kGCOfI*$W73nHCVZ1P;1d1Xlpj7( zr9qnI#f6_HgwQ(lsdM$1eZ>{ymUDdp`!^q~0+1E}fJQLs-4ffhV=i zW0v_s%Xi&zvnhwP;7Ku)?1@nPo?{jBmdPJM!BQR8Ie;MfI64qrRLPsLT%)wkWi4PD;q*=6@U(E_=N@S$_@j- zOo|1>L?s@8T1XOHM7p$ZzBAs%IwBlP@X30IFW&p`$~yG~2X^s|G!Cidm7<;RTeB|EPHW z{m?`x{>p=UkwFzvbLm^E96)%U;l}|?n-Lsf!j;4Ulqmm1Hl$msN-*A0**5Ze&m)3; zRay>h&e%|`l48sVc5laS2{B4J0Q5`qrd;wL1^B9aecPpT&k8v}6f!5x@nbH_uyV4Q z&BHnkN`l5&$tV9LsJe?S->kd@YIO30r{s5QNm-adeva^=8Z*Q;N6aGPIY4{EwhRY& z!fQ7?Z&A9k)g@eA*Idm%w3O@&D61HWBmd3YlMyP|y zRHNVa$-+MgMqcbXJ!$klanLHr?rcf%ldstCElPTi@^ANrooq=ab>{hOvFCDb#H9W- z@FB)JFzsAug_Md0L&lO Date: Sat, 18 Jul 2026 17:35:48 +0300 Subject: [PATCH 2/2] fix+refactor: windows test JSON, multi-source placeholder discovery, image guard - Fix TestPromptStepRun_AttachesImageFromRowPath on Windows: encode the row path with json.Marshal instead of string-concatenation (backslashes in Windows paths broke the hand-built JSON). This was the CI failure. Post-/simplify (altitude findings): - NewPromptBuilder takes variadic extra template sources for placeholder discovery; the image path is discovered/validated as its own template instead of being string-glued onto the prompt (`prompt+"\n"+image`). Parse errors are now attributed correctly, and image: placeholders are validated at config time (previously runtime-only). - Hoist the 'image only on prompt steps' guard to a top-level type guard next to collect/sourceFormat, so it also rejects image: on shell/transform (was only checked inside the read branch). --- promptbuilder/promptbuilder.go | 19 ++++++++++++++++++- step/prompt_step.go | 2 +- step/prompt_step_test.go | 7 +++++-- utils/preprocess.go | 16 ++++++++-------- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/promptbuilder/promptbuilder.go b/promptbuilder/promptbuilder.go index 8dd2c73..ac3376f 100644 --- a/promptbuilder/promptbuilder.go +++ b/promptbuilder/promptbuilder.go @@ -128,7 +128,11 @@ const ItemAliasName = "item" // source, and at render time .item shares the source step's values. The alias // is semantic — it works anywhere in the template, including {{len .item.x}} // and {{range .item.xs}}. Pass forEachSource="" for steps without forEach. -func NewPromptBuilder(prompt string, forEachSource string) (*PromptBuilder, error) { +// NewPromptBuilder parses the prompt into the render template. Any extra +// template sources (e.g. a step's `image:` path) are parsed independently and +// their placeholders merged into discovery, so referenced steps are loaded and +// validated without gluing the templates together. +func NewPromptBuilder(prompt string, forEachSource string, discoverAlso ...string) (*PromptBuilder, error) { tmpl, err := template.New("prompt").Option("missingkey=zero").Parse(prompt) if err != nil { return nil, fmt.Errorf("invalid prompt template: %w", err) @@ -136,6 +140,19 @@ func NewPromptBuilder(prompt string, forEachSource string) (*PromptBuilder, erro placeholders := collectPlaceholders(tmpl) + for _, src := range discoverAlso { + if src == "" { + continue + } + extra, err := template.New("extra").Option("missingkey=zero").Parse(src) + if err != nil { + return nil, fmt.Errorf("invalid template: %w", err) + } + for key, info := range collectPlaceholders(extra) { + placeholders[key] = info + } + } + for key, info := range placeholders { if info.Step != ItemAliasName { continue diff --git a/step/prompt_step.go b/step/prompt_step.go index 107858f..e340c6c 100644 --- a/step/prompt_step.go +++ b/step/prompt_step.go @@ -74,7 +74,7 @@ func (p *PromptStep) Run(ctx context.Context, cfg *config.Config, step config.St // parse the prompt (plus the image path, which may reference row fields // like {{.item.path}}) once to discover which steps it references, then read // each referenced file a single time up front (rows only differ by values) - base, err := promptbuilder.NewPromptBuilder(step.Prompt+"\n"+step.Image, step.ForEach) + base, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach, step.Image) if err != nil { return err } diff --git a/step/prompt_step_test.go b/step/prompt_step_test.go index b7c4730..b7fcc62 100644 --- a/step/prompt_step_test.go +++ b/step/prompt_step_test.go @@ -2,6 +2,7 @@ package step import ( "context" + "encoding/json" "os" "path/filepath" "testing" @@ -119,7 +120,9 @@ func TestPromptStepRun_AttachesImageFromRowPath(t *testing.T) { require.NoError(t, os.WriteFile(imgPath, []byte("fake-image-bytes"), 0o644)) srcPath := filepath.Join(dir, "src.jsonl") - require.NoError(t, os.WriteFile(srcPath, []byte(`{"path":"`+imgPath+`"}`+"\n"), 0o644)) + row, err := json.Marshal(map[string]string{"path": imgPath}) // encode: Windows paths have backslashes + require.NoError(t, err) + require.NoError(t, os.WriteFile(srcPath, append(row, '\n'), 0o644)) cfg.Steps = []config.Step{{Name: "imgs", Type: config.ReadStepType, OutputFilename: srcPath}} step.ForEach = "imgs" @@ -127,7 +130,7 @@ func TestPromptStepRun_AttachesImageFromRowPath(t *testing.T) { step.Prompt = "Describe the image." step.Image = "{{.item.path}}" - err := (&PromptStep{}).Run(context.Background(), cfg, step, dir) + err = (&PromptStep{}).Run(context.Background(), cfg, step, dir) require.NoError(t, err) // the request carried the base64 of the file diff --git a/utils/preprocess.go b/utils/preprocess.go index d7c8040..68937cc 100644 --- a/utils/preprocess.go +++ b/utils/preprocess.go @@ -153,6 +153,9 @@ func PreprocessConfig(cfg *config.Config) error { if step.SourceFormat != "" && step.Type != config.TransformStepType { return fmt.Errorf("step '%s': 'sourceFormat' is only valid on transform steps", step.Name) } + if step.Image != "" && step.Type != config.PromptStepType { + return fmt.Errorf("step '%s': 'image' is only valid on prompt steps", step.Name) + } if step.Type == config.TransformStepType { if step.From == "" { return fmt.Errorf("step '%s': 'from' is required for transform steps", step.Name) @@ -194,9 +197,6 @@ func PreprocessConfig(cfg *config.Config) error { // Read steps: local-file source (path resolves relative to CWD; rows // materialize to outputFolder like a transform) if step.Type == config.ReadStepType { - if step.Image != "" { - return fmt.Errorf("step '%s': 'image' is only valid on prompt steps", step.Name) - } format, err := resolveReadFormat(step) if err != nil { return fmt.Errorf("step '%s': %w", step.Name, err) @@ -225,12 +225,12 @@ func PreprocessConfig(cfg *config.Config) error { } // validatePromptPlaceholders checks every {{.step.field}} reference in the -// prompt against earlier steps: the step must exist ({{.item}} aliases the -// forEach source), field references into prompt steps must match their JSON -// schema, and a step may not be referenced both as a whole and by field in -// one prompt. +// prompt (and the image path) against earlier steps: the step must exist +// ({{.item}} aliases the forEach source), field references into prompt steps +// must match their JSON schema, and a step may not be referenced both as a +// whole and by field in one prompt. func validatePromptPlaceholders(step *config.Step, stepByName map[string]*config.Step) error { - builder, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach) + builder, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach, step.Image) if err != nil { return err }