feat: unify image input into read: + an image: attach field (D.1 Phase 2)

config/config.go

@@ -82,7 +82,7 @@ type Step struct {
 	ModelConfig    ModelConfig `yaml:"modelConfig"`
 	OutputFilename string      `yaml:"outputFilename"`
 	JSONSchemaRaw  interface{} `yaml:"jsonSchema"`
-	ImagePath      string      `yaml:"imagePath"`
+	Image          string      `yaml:"image"` // prompt steps: file path (templatable) to attach as a vision image
 	ResolvedCount  int
 	JSONSchema     jsonschema.Schema
 	// JQProgram holds the compiled jq program (set during preprocessing);
@@ -118,7 +118,3 @@ func (c *Config) GetStepByName(name string) *Step {
 	}
 	return nil
 }
-
-func (s *Step) HasImages() bool {
-	return len(s.ImagePath) > 0
-}

examples/v1/vision/README.md

@@ -1,20 +1,24 @@
 # Vision
 
-Transcribe handwritten-math images to LaTeX with a vision model, then explain each formula — a multimodal dataset pipeline.
+Run a vision model over your **own local images**: `read` enumerates a folder of
+images, and each prompt call attaches the current row's file as a vision image
+via `image: {{.item.path}}`. Here each image gets a structured alt-text record
+(description + colors + tags) — the kind of set used for cataloging or
+accessibility.
 
-**Features:** `imagePath` · `shell` · `forEach`
+**Features:** `read` · `image:` (vision attach) · `forEach` · `jsonSchema`
 
 ## Steps
 
-1. `download_images` — `hf download` + unzip + `magick` convert (BMP → JPEG)
-2. `to_latex` — vision step over an `imagePath` glob → LaTeX transcription
-3. `explain` — `forEach` → a step-by-step explanation of the formula
+1. `images` — `read: ./images/*.jpg` → one row per image: `{path, name, content}`
+2. `describe` — `forEach` image, `image: {{.item.path}}` attaches it → `{description, main_colors[], tags[]}`
+
+Point `read` at your own image folder to process your files.
 
 ## Requirements
 
 - `datamatic`
 - [Ollama](https://ollama.com/download) (or [LM Studio](https://lmstudio.ai/download)) + a vision model: `ollama pull qwen2.5vl:3b` (or `gemma3:4b`)
-- [hf](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli), [magick](https://imagemagick.org/script/download.php)
 
 ## Run
 

examples/v1/vision/config.yaml

@@ -1,33 +1,36 @@
 version: 1.0
 
+# Vision over your OWN local images: read a folder of images and describe each.
+# `read` enumerates the files; the prompt step attaches the current row's file
+# as a vision image via `image: {{.item.path}}`.
 steps:
-  - name: download_dataset
-    type: shell
-    run: |
-      # download image dataset from huggingface
-      hf download --repo-type dataset Azu/Handwritten-Mathematical-Expression-Convert-LaTeX --local-dir ./ --include data.zip
+  - name: images
+    read: ./images/*.jpg
 
-      # unzip
-      unzip -q -o data.zip -d data
-
-      # convert all BMP images to JPG
-      magick mogrify -format jpg -quality 90 -path ./data/2019 ./data/2019/*.bmp
-    outputFilename: data
-
-  - name: analyze_math_image
-    type: prompt
-    model: ollama:qwen2.5vl:3b
-    # model: lmstudio:gemma-3-4b-it
-    prompt: |
-      Image contains mathematical formula. Convert to Latex. Don't add any extra information
-    imagePath: |
-      ./data/2019/*.jpg
-
-  - name: explain_math
-    type: prompt
+  - name: describe
     model: ollama:qwen2.5vl:3b
     # model: lmstudio:gemma-3-4b-it
-    forEach: analyze_math_image
+    forEach: images
+    image: "{{.item.path}}"
     prompt: |
-      Given the following formula, break down the steps needed to understand or solve it. Start by identifying the type of mathematical expression it is (e.g., complex numbers, exponential form, etc.), then explain each term and operation, and proceed to derive or simplify as appropriate. Show your reasoning at each step. Format your response in clear Markdown, using headings, bullet points, and LaTeX for math where helpful.
-      Formula: {{.analyze_math_image}}
+      Describe this image for an alt-text catalog: what it shows, its main colors,
+      and a few searchable tags. Return as JSON.
+    jsonSchema:
+      type: object
+      properties:
+        description:
+          type: string
+          minLength: 10
+          maxLength: 300
+        main_colors:
+          type: array
+          items: { type: string }
+          minItems: 1
+          maxItems: 5
+        tags:
+          type: array
+          items: { type: string }
+          minItems: 1
+          maxItems: 5
+      required: [description, main_colors, tags]
+      additionalProperties: false

fs/image_test.go

@@ -9,39 +9,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestPickImageFile(t *testing.T) {
-	tmpDir := t.TempDir()
-	files := []string{"img1.jpg", "img2.jpg", "img3.jpg"}
-
-	for _, name := range files {
-		fullPath := filepath.Join(tmpDir, name)
-		err := os.WriteFile(fullPath, []byte("dummy image data"), 0o644)
-		assert.NoError(t, err)
-	}
-
-	pattern := filepath.Join(tmpDir, "*.jpg")
-
-	t.Run("valid index", func(t *testing.T) {
-		path, err := PickImageFile(pattern, 1)
-		assert.NoError(t, err)
-		assert.Contains(t, path, "img2.jpg")
-	})
-
-	t.Run("index wraps around", func(t *testing.T) {
-		path, err := PickImageFile(pattern, 7)
-		assert.NoError(t, err)
-		assert.Contains(t, path, "img2.jpg")
-	})
-
-	t.Run("no matches", func(t *testing.T) {
-		emptyPattern := filepath.Join(tmpDir, "*.png")
-		_, err := PickImageFile(emptyPattern, 0)
-		assert.Error(t, err)
-		_, err = PickImageFile(emptyPattern, 5)
-		assert.Error(t, err)
-	})
-}
-
 func TestImageToBase64(t *testing.T) {
 	t.Run("valid image file", func(t *testing.T) {
 		tmpFile := filepath.Join(t.TempDir(), "test.jpg")

fs/images.go

@@ -5,52 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
-
-	"github.com/rs/zerolog/log"
 )
 
-func globFiles(pattern string) ([]string, error) {
-	files, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-	return files, nil
-}
-
-func CountFiles(pattern string) (int, error) {
-	log.Debug().Msgf("searching for files matching pattern: %s", pattern)
-
-	files, err := globFiles(pattern)
-	if err != nil {
-		return 0, fmt.Errorf("failed to search for files with pattern %s: %w", pattern, err)
-	}
-
-	totalFiles := len(files)
-	log.Debug().Msgf("found %d files matching pattern: %s", totalFiles, pattern)
-
-	return totalFiles, nil
-}
-
-func PickImageFile(pattern string, index int) (string, error) {
-	log.Debug().Msgf("Searching for files matching pattern: %s", pattern)
-
-	files, err := globFiles(pattern)
-	if err != nil {
-		return "", fmt.Errorf("failed to search for files with pattern %s: %w", pattern, err)
-	}
-
-	if len(files) == 0 {
-		return "", fmt.Errorf("no files matched pattern: %s", pattern)
-	}
-
-	index = index % len(files)
-	selected := files[index]
-
-	log.Debug().Msgf("Selected file [%d]: %s", index, selected)
-	return selected, nil
-}
-
+// ImageToBase64 reads a file and returns its base64-encoded contents, used to
+// attach an image to a vision model request.
 func ImageToBase64(imagePath string) (string, error) {
 	if imagePath == "" {
 		return "", errors.New("image path is empty")

promptbuilder/promptbuilder.go

@@ -128,14 +128,31 @@ const ItemAliasName = "item"
 // source, and at render time .item shares the source step's values. The alias
 // is semantic — it works anywhere in the template, including {{len .item.x}}
 // and {{range .item.xs}}. Pass forEachSource="" for steps without forEach.
-func NewPromptBuilder(prompt string, forEachSource string) (*PromptBuilder, error) {
+// NewPromptBuilder parses the prompt into the render template. Any extra
+// template sources (e.g. a step's `image:` path) are parsed independently and
+// their placeholders merged into discovery, so referenced steps are loaded and
+// validated without gluing the templates together.
+func NewPromptBuilder(prompt string, forEachSource string, discoverAlso ...string) (*PromptBuilder, error) {
 	tmpl, err := template.New("prompt").Option("missingkey=zero").Parse(prompt)
 	if err != nil {
 		return nil, fmt.Errorf("invalid prompt template: %w", err)
 	}
 
 	placeholders := collectPlaceholders(tmpl)
 
+	for _, src := range discoverAlso {
+		if src == "" {
+			continue
+		}
+		extra, err := template.New("extra").Option("missingkey=zero").Parse(src)
+		if err != nil {
+			return nil, fmt.Errorf("invalid template: %w", err)
+		}
+		for key, info := range collectPlaceholders(extra) {
+			placeholders[key] = info
+		}
+	}
+
 	for key, info := range placeholders {
 		if info.Step != ItemAliasName {
 			continue
@@ -191,7 +208,9 @@ func setNestedValue(target Object, path string, value interface{}) {
 	current[parts[len(parts)-1]] = value
 }
 
-func (pb *PromptBuilder) BuildPrompt() (string, error) {
+// buildValues assembles the template context from the loaded step data,
+// exposing the forEach source under the {{.item}} alias.
+func (pb *PromptBuilder) buildValues() map[string]interface{} {
 	values := make(map[string]interface{})
 	for stepName, stepFields := range pb.stepData {
 		stepObj := make(Object)
@@ -213,6 +232,11 @@ func (pb *PromptBuilder) BuildPrompt() (string, error) {
 		}
 	}
 
+	return values
+}
+
+func (pb *PromptBuilder) BuildPrompt() (string, error) {
+	values := pb.buildValues()
 	log.Debug().Msgf("using values: %+v", values)
 
 	var output bytes.Buffer
@@ -223,6 +247,22 @@ func (pb *PromptBuilder) BuildPrompt() (string, error) {
 	return output.String(), nil
 }
 
+// RenderString renders an arbitrary template string against the same values as
+// the prompt — used for the per-row `image:` path (e.g. "{{.item.path}}").
+func (pb *PromptBuilder) RenderString(s string) (string, error) {
+	tmpl, err := template.New("render").Option("missingkey=zero").Parse(s)
+	if err != nil {
+		return "", fmt.Errorf("invalid template: %w", err)
+	}
+
+	var output bytes.Buffer
+	if err := tmpl.Execute(&output, pb.buildValues()); err != nil {
+		return "", fmt.Errorf("failed to execute template: %w", err)
+	}
+
+	return output.String(), nil
+}
+
 func (pb *PromptBuilder) GetPlaceholders() map[string]PlaceholderInfo {
 	return pb.placeholders
 }

promptbuilder/promptbuilder_test.go

@@ -391,3 +391,15 @@ func TestParseTemplatePlaceholders_ScopedDotIsNotAStepRef(t *testing.T) {
 		})
 	}
 }
+
+func TestPromptBuilder_RenderString(t *testing.T) {
+	pb, err := NewPromptBuilder("describe {{.item.name}}", "docs")
+	require.NoError(t, err)
+	pb.AddValue("1", "docs", "path", "images/a.jpg")
+	pb.AddValue("2", "docs", "name", "a.jpg")
+
+	got, err := pb.RenderString("{{.item.path}}")
+
+	require.NoError(t, err)
+	assert.Equal(t, "images/a.jpg", got)
+}

runner/runner.go

@@ -50,15 +50,6 @@ func (r *Runner) resolveIterations(step *config.Step) error {
 		step.ResolvedCount = lines
 		log.Debug().Msgf("Resolved iterations for step '%s' to %d from forEach: %s", step.Name, lines, step.ForEach)
 
-	case step.HasImages() && step.Count == 0:
-		images, err := fs.CountFiles(step.ImagePath)
-		if err != nil {
-			return fmt.Errorf("failed to count images matching '%s': %w", step.ImagePath, err)
-		}
-
-		step.ResolvedCount = images
-		log.Debug().Msgf("Resolved iterations for step '%s' to %d from imagePath: %s", step.Name, images, step.ImagePath)
-
 	case step.Count == 0:
 		step.ResolvedCount = config.DefaultStepCount
 

step/prompt_step.go

@@ -71,9 +71,10 @@ func (p *PromptStep) Run(ctx context.Context, cfg *config.Config, step config.St
 
 	hasSchema := step.JSONSchema.HasSchemaDefinition()
 
-	// parse the prompt once to discover which steps it references, then read
+	// parse the prompt (plus the image path, which may reference row fields
+	// like {{.item.path}}) once to discover which steps it references, then read
 	// each referenced file a single time up front (rows only differ by values)
-	base, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach)
+	base, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach, step.Image)
 	if err != nil {
 		return err
 	}
@@ -116,18 +117,16 @@ func (p *PromptStep) runRow(ctx context.Context, cfg *config.Config, step config
 	}
 
 	var base64Image string
-	if step.HasImages() {
-		imagePath, err := fs.PickImageFile(step.ImagePath, i)
+	if step.Image != "" {
+		imagePath, err := pb.RenderString(step.Image)
 		if err != nil {
-			return jsonl.LineEntity{}, fmt.Errorf("failed to find images by pattern '%s': %w", step.ImagePath, err)
+			return jsonl.LineEntity{}, fmt.Errorf("failed to resolve image path '%s': %w", step.Image, err)
 		}
 
 		base64Image, err = fs.ImageToBase64(imagePath)
 		if err != nil {
 			return jsonl.LineEntity{}, fmt.Errorf("failed to encode image '%s': %w", imagePath, err)
 		}
-
-		pb.AddValue(base64Image[:15], step.Name, "image", imagePath)
 	}
 
 	userPrompt, err := pb.BuildPrompt()