diff --git a/config/config.go b/config/config.go index 9cbedb6..eef60f8 100644 --- a/config/config.go +++ b/config/config.go @@ -82,7 +82,7 @@ type Step struct { ModelConfig ModelConfig `yaml:"modelConfig"` OutputFilename string `yaml:"outputFilename"` JSONSchemaRaw interface{} `yaml:"jsonSchema"` - ImagePath string `yaml:"imagePath"` + Image string `yaml:"image"` // prompt steps: file path (templatable) to attach as a vision image ResolvedCount int JSONSchema jsonschema.Schema // JQProgram holds the compiled jq program (set during preprocessing); @@ -118,7 +118,3 @@ func (c *Config) GetStepByName(name string) *Step { } return nil } - -func (s *Step) HasImages() bool { - return len(s.ImagePath) > 0 -} diff --git a/examples/v1/vision/README.md b/examples/v1/vision/README.md index fda80cd..0e146b6 100644 --- a/examples/v1/vision/README.md +++ b/examples/v1/vision/README.md @@ -1,20 +1,24 @@ # Vision -Transcribe handwritten-math images to LaTeX with a vision model, then explain each formula — a multimodal dataset pipeline. +Run a vision model over your **own local images**: `read` enumerates a folder of +images, and each prompt call attaches the current row's file as a vision image +via `image: {{.item.path}}`. Here each image gets a structured alt-text record +(description + colors + tags) — the kind of set used for cataloging or +accessibility. -**Features:** `imagePath` · `shell` · `forEach` +**Features:** `read` · `image:` (vision attach) · `forEach` · `jsonSchema` ## Steps -1. `download_images` — `hf download` + unzip + `magick` convert (BMP → JPEG) -2. `to_latex` — vision step over an `imagePath` glob → LaTeX transcription -3. `explain` — `forEach` → a step-by-step explanation of the formula +1. `images` — `read: ./images/*.jpg` → one row per image: `{path, name, content}` +2. `describe` — `forEach` image, `image: {{.item.path}}` attaches it → `{description, main_colors[], tags[]}` + +Point `read` at your own image folder to process your files. ## Requirements - `datamatic` - [Ollama](https://ollama.com/download) (or [LM Studio](https://lmstudio.ai/download)) + a vision model: `ollama pull qwen2.5vl:3b` (or `gemma3:4b`) -- [hf](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli), [magick](https://imagemagick.org/script/download.php) ## Run diff --git a/examples/v1/vision/config.yaml b/examples/v1/vision/config.yaml index f2f3671..2d4e4bb 100644 --- a/examples/v1/vision/config.yaml +++ b/examples/v1/vision/config.yaml @@ -1,33 +1,36 @@ version: 1.0 +# Vision over your OWN local images: read a folder of images and describe each. +# `read` enumerates the files; the prompt step attaches the current row's file +# as a vision image via `image: {{.item.path}}`. steps: - - name: download_dataset - type: shell - run: | - # download image dataset from huggingface - hf download --repo-type dataset Azu/Handwritten-Mathematical-Expression-Convert-LaTeX --local-dir ./ --include data.zip + - name: images + read: ./images/*.jpg - # unzip - unzip -q -o data.zip -d data - - # convert all BMP images to JPG - magick mogrify -format jpg -quality 90 -path ./data/2019 ./data/2019/*.bmp - outputFilename: data - - - name: analyze_math_image - type: prompt - model: ollama:qwen2.5vl:3b - # model: lmstudio:gemma-3-4b-it - prompt: | - Image contains mathematical formula. Convert to Latex. Don't add any extra information - imagePath: | - ./data/2019/*.jpg - - - name: explain_math - type: prompt + - name: describe model: ollama:qwen2.5vl:3b # model: lmstudio:gemma-3-4b-it - forEach: analyze_math_image + forEach: images + image: "{{.item.path}}" prompt: | - Given the following formula, break down the steps needed to understand or solve it. Start by identifying the type of mathematical expression it is (e.g., complex numbers, exponential form, etc.), then explain each term and operation, and proceed to derive or simplify as appropriate. Show your reasoning at each step. Format your response in clear Markdown, using headings, bullet points, and LaTeX for math where helpful. - Formula: {{.analyze_math_image}} + Describe this image for an alt-text catalog: what it shows, its main colors, + and a few searchable tags. Return as JSON. + jsonSchema: + type: object + properties: + description: + type: string + minLength: 10 + maxLength: 300 + main_colors: + type: array + items: { type: string } + minItems: 1 + maxItems: 5 + tags: + type: array + items: { type: string } + minItems: 1 + maxItems: 5 + required: [description, main_colors, tags] + additionalProperties: false diff --git a/examples/v1/vision/images/barchart.jpg b/examples/v1/vision/images/barchart.jpg new file mode 100644 index 0000000..ff82a1f Binary files /dev/null and b/examples/v1/vision/images/barchart.jpg differ diff --git a/examples/v1/vision/images/shapes.jpg b/examples/v1/vision/images/shapes.jpg new file mode 100644 index 0000000..a21f3b3 Binary files /dev/null and b/examples/v1/vision/images/shapes.jpg differ diff --git a/examples/v1/vision/images/triangle.jpg b/examples/v1/vision/images/triangle.jpg new file mode 100644 index 0000000..7353762 Binary files /dev/null and b/examples/v1/vision/images/triangle.jpg differ diff --git a/fs/image_test.go b/fs/image_test.go index d6a7c33..31e5a19 100644 --- a/fs/image_test.go +++ b/fs/image_test.go @@ -9,39 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPickImageFile(t *testing.T) { - tmpDir := t.TempDir() - files := []string{"img1.jpg", "img2.jpg", "img3.jpg"} - - for _, name := range files { - fullPath := filepath.Join(tmpDir, name) - err := os.WriteFile(fullPath, []byte("dummy image data"), 0o644) - assert.NoError(t, err) - } - - pattern := filepath.Join(tmpDir, "*.jpg") - - t.Run("valid index", func(t *testing.T) { - path, err := PickImageFile(pattern, 1) - assert.NoError(t, err) - assert.Contains(t, path, "img2.jpg") - }) - - t.Run("index wraps around", func(t *testing.T) { - path, err := PickImageFile(pattern, 7) - assert.NoError(t, err) - assert.Contains(t, path, "img2.jpg") - }) - - t.Run("no matches", func(t *testing.T) { - emptyPattern := filepath.Join(tmpDir, "*.png") - _, err := PickImageFile(emptyPattern, 0) - assert.Error(t, err) - _, err = PickImageFile(emptyPattern, 5) - assert.Error(t, err) - }) -} - func TestImageToBase64(t *testing.T) { t.Run("valid image file", func(t *testing.T) { tmpFile := filepath.Join(t.TempDir(), "test.jpg") diff --git a/fs/images.go b/fs/images.go index 1ce89ab..5cffaf6 100644 --- a/fs/images.go +++ b/fs/images.go @@ -5,52 +5,10 @@ import ( "errors" "fmt" "os" - "path/filepath" - - "github.com/rs/zerolog/log" ) -func globFiles(pattern string) ([]string, error) { - files, err := filepath.Glob(pattern) - if err != nil { - return nil, err - } - return files, nil -} - -func CountFiles(pattern string) (int, error) { - log.Debug().Msgf("searching for files matching pattern: %s", pattern) - - files, err := globFiles(pattern) - if err != nil { - return 0, fmt.Errorf("failed to search for files with pattern %s: %w", pattern, err) - } - - totalFiles := len(files) - log.Debug().Msgf("found %d files matching pattern: %s", totalFiles, pattern) - - return totalFiles, nil -} - -func PickImageFile(pattern string, index int) (string, error) { - log.Debug().Msgf("Searching for files matching pattern: %s", pattern) - - files, err := globFiles(pattern) - if err != nil { - return "", fmt.Errorf("failed to search for files with pattern %s: %w", pattern, err) - } - - if len(files) == 0 { - return "", fmt.Errorf("no files matched pattern: %s", pattern) - } - - index = index % len(files) - selected := files[index] - - log.Debug().Msgf("Selected file [%d]: %s", index, selected) - return selected, nil -} - +// ImageToBase64 reads a file and returns its base64-encoded contents, used to +// attach an image to a vision model request. func ImageToBase64(imagePath string) (string, error) { if imagePath == "" { return "", errors.New("image path is empty") diff --git a/promptbuilder/promptbuilder.go b/promptbuilder/promptbuilder.go index 5a98c04..ac3376f 100644 --- a/promptbuilder/promptbuilder.go +++ b/promptbuilder/promptbuilder.go @@ -128,7 +128,11 @@ const ItemAliasName = "item" // source, and at render time .item shares the source step's values. The alias // is semantic — it works anywhere in the template, including {{len .item.x}} // and {{range .item.xs}}. Pass forEachSource="" for steps without forEach. -func NewPromptBuilder(prompt string, forEachSource string) (*PromptBuilder, error) { +// NewPromptBuilder parses the prompt into the render template. Any extra +// template sources (e.g. a step's `image:` path) are parsed independently and +// their placeholders merged into discovery, so referenced steps are loaded and +// validated without gluing the templates together. +func NewPromptBuilder(prompt string, forEachSource string, discoverAlso ...string) (*PromptBuilder, error) { tmpl, err := template.New("prompt").Option("missingkey=zero").Parse(prompt) if err != nil { return nil, fmt.Errorf("invalid prompt template: %w", err) @@ -136,6 +140,19 @@ func NewPromptBuilder(prompt string, forEachSource string) (*PromptBuilder, erro placeholders := collectPlaceholders(tmpl) + for _, src := range discoverAlso { + if src == "" { + continue + } + extra, err := template.New("extra").Option("missingkey=zero").Parse(src) + if err != nil { + return nil, fmt.Errorf("invalid template: %w", err) + } + for key, info := range collectPlaceholders(extra) { + placeholders[key] = info + } + } + for key, info := range placeholders { if info.Step != ItemAliasName { continue @@ -191,7 +208,9 @@ func setNestedValue(target Object, path string, value interface{}) { current[parts[len(parts)-1]] = value } -func (pb *PromptBuilder) BuildPrompt() (string, error) { +// buildValues assembles the template context from the loaded step data, +// exposing the forEach source under the {{.item}} alias. +func (pb *PromptBuilder) buildValues() map[string]interface{} { values := make(map[string]interface{}) for stepName, stepFields := range pb.stepData { stepObj := make(Object) @@ -213,6 +232,11 @@ func (pb *PromptBuilder) BuildPrompt() (string, error) { } } + return values +} + +func (pb *PromptBuilder) BuildPrompt() (string, error) { + values := pb.buildValues() log.Debug().Msgf("using values: %+v", values) var output bytes.Buffer @@ -223,6 +247,22 @@ func (pb *PromptBuilder) BuildPrompt() (string, error) { return output.String(), nil } +// RenderString renders an arbitrary template string against the same values as +// the prompt — used for the per-row `image:` path (e.g. "{{.item.path}}"). +func (pb *PromptBuilder) RenderString(s string) (string, error) { + tmpl, err := template.New("render").Option("missingkey=zero").Parse(s) + if err != nil { + return "", fmt.Errorf("invalid template: %w", err) + } + + var output bytes.Buffer + if err := tmpl.Execute(&output, pb.buildValues()); err != nil { + return "", fmt.Errorf("failed to execute template: %w", err) + } + + return output.String(), nil +} + func (pb *PromptBuilder) GetPlaceholders() map[string]PlaceholderInfo { return pb.placeholders } diff --git a/promptbuilder/promptbuilder_test.go b/promptbuilder/promptbuilder_test.go index 41beeee..5501003 100644 --- a/promptbuilder/promptbuilder_test.go +++ b/promptbuilder/promptbuilder_test.go @@ -391,3 +391,15 @@ func TestParseTemplatePlaceholders_ScopedDotIsNotAStepRef(t *testing.T) { }) } } + +func TestPromptBuilder_RenderString(t *testing.T) { + pb, err := NewPromptBuilder("describe {{.item.name}}", "docs") + require.NoError(t, err) + pb.AddValue("1", "docs", "path", "images/a.jpg") + pb.AddValue("2", "docs", "name", "a.jpg") + + got, err := pb.RenderString("{{.item.path}}") + + require.NoError(t, err) + assert.Equal(t, "images/a.jpg", got) +} diff --git a/runner/runner.go b/runner/runner.go index 24604d9..f577dd2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -50,15 +50,6 @@ func (r *Runner) resolveIterations(step *config.Step) error { step.ResolvedCount = lines log.Debug().Msgf("Resolved iterations for step '%s' to %d from forEach: %s", step.Name, lines, step.ForEach) - case step.HasImages() && step.Count == 0: - images, err := fs.CountFiles(step.ImagePath) - if err != nil { - return fmt.Errorf("failed to count images matching '%s': %w", step.ImagePath, err) - } - - step.ResolvedCount = images - log.Debug().Msgf("Resolved iterations for step '%s' to %d from imagePath: %s", step.Name, images, step.ImagePath) - case step.Count == 0: step.ResolvedCount = config.DefaultStepCount diff --git a/step/prompt_step.go b/step/prompt_step.go index 8b8693e..e340c6c 100644 --- a/step/prompt_step.go +++ b/step/prompt_step.go @@ -71,9 +71,10 @@ func (p *PromptStep) Run(ctx context.Context, cfg *config.Config, step config.St hasSchema := step.JSONSchema.HasSchemaDefinition() - // parse the prompt once to discover which steps it references, then read + // parse the prompt (plus the image path, which may reference row fields + // like {{.item.path}}) once to discover which steps it references, then read // each referenced file a single time up front (rows only differ by values) - base, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach) + base, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach, step.Image) if err != nil { return err } @@ -116,18 +117,16 @@ func (p *PromptStep) runRow(ctx context.Context, cfg *config.Config, step config } var base64Image string - if step.HasImages() { - imagePath, err := fs.PickImageFile(step.ImagePath, i) + if step.Image != "" { + imagePath, err := pb.RenderString(step.Image) if err != nil { - return jsonl.LineEntity{}, fmt.Errorf("failed to find images by pattern '%s': %w", step.ImagePath, err) + return jsonl.LineEntity{}, fmt.Errorf("failed to resolve image path '%s': %w", step.Image, err) } base64Image, err = fs.ImageToBase64(imagePath) if err != nil { return jsonl.LineEntity{}, fmt.Errorf("failed to encode image '%s': %w", imagePath, err) } - - pb.AddValue(base64Image[:15], step.Name, "image", imagePath) } userPrompt, err := pb.BuildPrompt() diff --git a/step/prompt_step_test.go b/step/prompt_step_test.go index dd631e7..b7fcc62 100644 --- a/step/prompt_step_test.go +++ b/step/prompt_step_test.go @@ -2,6 +2,7 @@ package step import ( "context" + "encoding/json" "os" "path/filepath" "testing" @@ -99,15 +100,44 @@ func TestPromptStepRun_RefValuesReadFailureFailsStep(t *testing.T) { assert.Equal(t, 0, srv.CallCount(), "no LLM call with a broken prompt") } -func TestPromptStepRun_MissingImagesFailsStep(t *testing.T) { +func TestPromptStepRun_MissingImageFailsStep(t *testing.T) { srv := llmtest.NewServer(t, "never reached") cfg, step, dir := promptStepConfig(t, srv.URL) - step.ImagePath = filepath.Join(dir, "no-such-dir", "*.jpg") + step.Image = filepath.Join(dir, "no-such.jpg") // file does not exist err := (&PromptStep{}).Run(context.Background(), cfg, step, dir) require.Error(t, err) - assert.Equal(t, 0, srv.CallCount()) + assert.Equal(t, 0, srv.CallCount(), "no LLM call when the image can't be read") +} + +func TestPromptStepRun_AttachesImageFromRowPath(t *testing.T) { + // image: renders a per-row path template and attaches the file as base64 + srv := llmtest.NewServer(t, "described") + cfg, step, dir := promptStepConfig(t, srv.URL) + + imgPath := filepath.Join(dir, "pic.jpg") + require.NoError(t, os.WriteFile(imgPath, []byte("fake-image-bytes"), 0o644)) + + srcPath := filepath.Join(dir, "src.jsonl") + row, err := json.Marshal(map[string]string{"path": imgPath}) // encode: Windows paths have backslashes + require.NoError(t, err) + require.NoError(t, os.WriteFile(srcPath, append(row, '\n'), 0o644)) + cfg.Steps = []config.Step{{Name: "imgs", Type: config.ReadStepType, OutputFilename: srcPath}} + + step.ForEach = "imgs" + step.ResolvedCount = 1 + step.Prompt = "Describe the image." + step.Image = "{{.item.path}}" + + err = (&PromptStep{}).Run(context.Background(), cfg, step, dir) + require.NoError(t, err) + + // the request carried the base64 of the file + messages := srv.Requests()[0]["messages"].([]interface{}) + last := messages[len(messages)-1].(map[string]interface{}) + assert.NotNil(t, last["content"], "vision content attached") + assert.Equal(t, 1, srv.CallCount()) } func TestPromptStepRun_FailsAfterRepeatedInvalidResponses(t *testing.T) { diff --git a/utils/preprocess.go b/utils/preprocess.go index c5023c2..68937cc 100644 --- a/utils/preprocess.go +++ b/utils/preprocess.go @@ -153,6 +153,9 @@ func PreprocessConfig(cfg *config.Config) error { if step.SourceFormat != "" && step.Type != config.TransformStepType { return fmt.Errorf("step '%s': 'sourceFormat' is only valid on transform steps", step.Name) } + if step.Image != "" && step.Type != config.PromptStepType { + return fmt.Errorf("step '%s': 'image' is only valid on prompt steps", step.Name) + } if step.Type == config.TransformStepType { if step.From == "" { return fmt.Errorf("step '%s': 'from' is required for transform steps", step.Name) @@ -194,9 +197,6 @@ func PreprocessConfig(cfg *config.Config) error { // Read steps: local-file source (path resolves relative to CWD; rows // materialize to outputFolder like a transform) if step.Type == config.ReadStepType { - if step.ImagePath != "" { - return fmt.Errorf("step '%s': 'imagePath' is not valid on read steps", step.Name) - } format, err := resolveReadFormat(step) if err != nil { return fmt.Errorf("step '%s': %w", step.Name, err) @@ -207,13 +207,6 @@ func PreprocessConfig(cfg *config.Config) error { } } - // Normalize image path if needed - if step.HasImages() { - if err := setImagePath(step, cfg.OutputFolder); err != nil { - return fmt.Errorf("step '%s': %w", step.Name, err) - } - } - if err := validateIterationSettings(step, stepNames); err != nil { return fmt.Errorf("step '%s': %w", step.Name, err) } @@ -232,12 +225,12 @@ func PreprocessConfig(cfg *config.Config) error { } // validatePromptPlaceholders checks every {{.step.field}} reference in the -// prompt against earlier steps: the step must exist ({{.item}} aliases the -// forEach source), field references into prompt steps must match their JSON -// schema, and a step may not be referenced both as a whole and by field in -// one prompt. +// prompt (and the image path) against earlier steps: the step must exist +// ({{.item}} aliases the forEach source), field references into prompt steps +// must match their JSON schema, and a step may not be referenced both as a +// whole and by field in one prompt. func validatePromptPlaceholders(step *config.Step, stepByName map[string]*config.Step) error { - builder, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach) + builder, err := promptbuilder.NewPromptBuilder(step.Prompt, step.ForEach, step.Image) if err != nil { return err } @@ -366,17 +359,6 @@ func setOutputFilename(step *config.Step, outputFolder string) error { return nil } -// setImagePath processes and sets the image path for a step -func setImagePath(step *config.Step, outputFolder string) error { - step.ImagePath = strings.TrimSpace(step.ImagePath) - - if !filepath.IsAbs(step.ImagePath) { - step.ImagePath = filepath.Join(outputFolder, step.ImagePath) - } - - return nil -} - // requireEarlierStep checks that a cross-step reference points at an already // defined step. stepNames must hold earlier steps only. func requireEarlierStep(stepNames map[string]bool, field, name string) error { diff --git a/utils/preprocess_test.go b/utils/preprocess_test.go index c81a805..ebf0523 100644 --- a/utils/preprocess_test.go +++ b/utils/preprocess_test.go @@ -56,8 +56,7 @@ func TestPreprocessConfig_Success(t *testing.T) { Model: "ollama:llama3.2", Prompt: "Generate something", OutputFilename: "custom", - ImagePath: "images/photo.jpg", - // no count/forEach: image step, iterations resolved at runtime + // no count/forEach: iterations resolved at runtime }, { Name: "cli1", @@ -99,12 +98,8 @@ func TestPreprocessConfig_Success(t *testing.T) { assert.Equal(t, expectedCustom, cfg.Steps[0].OutputFilename) assert.Equal(t, expectedCli, cfg.Steps[1].OutputFilename) // CLI steps get absolute path but no extension change - // Image path - use absolute paths that work cross-platform - expectedImage, _ := filepath.Abs(filepath.Join(outputFolder, "images", "photo.jpg")) - assert.Equal(t, expectedImage, cfg.Steps[0].ImagePath) - // Iteration settings - assert.Equal(t, 0, cfg.Steps[0].Count, "image step: iterations resolved at runtime, no default count") + assert.Equal(t, 0, cfg.Steps[0].Count, "no count/forEach: iterations resolved at runtime") assert.Equal(t, 0, cfg.Steps[1].Count, "shell steps have no count") assert.Equal(t, 5, cfg.Steps[2].Count) assert.Equal(t, "prompt1", cfg.Steps[3].ForEach) @@ -631,9 +626,9 @@ func TestPreprocessConfig_ReadStep(t *testing.T) { cfg.Steps[0].Count = 3 assert.ErrorContains(t, PreprocessConfig(cfg), "only valid on prompt") }) - t.Run("read plus imagePath fails", func(t *testing.T) { + t.Run("read plus image fails", func(t *testing.T) { cfg := base("./x/*.md", "") - cfg.Steps[0].ImagePath = "*.jpg" - assert.ErrorContains(t, PreprocessConfig(cfg), "imagePath") + cfg.Steps[0].Image = "*.jpg" + assert.ErrorContains(t, PreprocessConfig(cfg), "image") }) }