Skip to content

Commit 272b350

Browse files
netdurnetdur
committed
v0.1.0
vision support
1 parent 6992a04 commit 272b350

File tree

6 files changed

+798
-264
lines changed

6 files changed

+798
-264
lines changed

‎android/llamalib/build.gradle‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,14 @@ android {
2121
cppFlags "-std=c++17", "-fexceptions", "-frtti"
2222
arguments "-DBUILD_SHARED_LIBS=ON",
2323
"-DLLAMA_CURL=OFF",
24-
"-DGGML_VULKAN=ON",
25-
"-DGGML_VULKAN_COOPMAT_GLSLC_SUPPORT=OFF",
26-
"-DGGML_VULKAN_COOPMAT2_GLSLC_SUPPORT=OFF",
27-
"-DGGML_VULKAN_BFLOAT16_GLSLC_SUPPORT=OFF",
24+
25+
// "-DGGML_OPENCL=ON",
26+
// "-DOpenCL_INCLUDE_DIR=/Users/adel/Workspace/OpenCL-Headers/include",
27+
28+
// "-DGGML_VULKAN=ON",
29+
// "-DGGML_VULKAN_COOPMAT_GLSLC_SUPPORT=OFF",
30+
// "-DGGML_VULKAN_COOPMAT2_GLSLC_SUPPORT=OFF",
31+
// "-DGGML_VULKAN_BFLOAT16_GLSLC_SUPPORT=OFF",
2832
// "-DVK_USE_PLATFORM_ANDROID_KHR=ON",
2933
// "-DGGML_VULKAN_VALIDATE=OFF",
3034
"-DLLAMA_BUILD_TESTS=OFF",

‎example/test.dart‎

Lines changed: 25 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -1,183 +1,46 @@
11
// ignore_for_file: avoid_print
22

33
import 'dart:io';
4-
import 'dart:convert';
5-
import 'package:http/http.dart' as http;
6-
import 'package:intl/intl.dart';
74
import 'package:llama_cpp_dart/llama_cpp_dart.dart';
85

9-
// ==========================================================
10-
// TOOLS (No changes here, they are correct)
11-
// ==========================================================
6+
String prompt = """what is 2 * 4?""";
127

13-
/// Tool to get the current time.
14-
String getCurrentTime() {
15-
final now = DateTime.now();
16-
final formattedTime = DateFormat('HH:mm').format(now);
17-
return "The current time is $formattedTime.";
18-
}
19-
20-
/// Tool to get the weather for a city.
21-
Future<String> getWeather(String city) async {
22-
try {
23-
final url = Uri.parse('https://wttr.in/$city?format=%C,+%t');
24-
final response = await http.get(url);
25-
if (response.statusCode == 200) {
26-
final body = response.body.trim();
27-
if (body.contains(',')) {
28-
return "The weather in $city is ${body.split(',')[0]} with a temperature of ${body.split(',')[1]}.";
29-
}
30-
return "Could not parse weather for $city. Response: $body";
31-
} else {
32-
return "Sorry, I couldn't get the weather for $city.";
33-
}
34-
} catch (e) {
35-
return "An error occurred while fetching weather: $e";
36-
}
37-
}
38-
39-
final Map<String, Function> toolBox = {
40-
'getCurrentTime': getCurrentTime,
41-
'getWeather': getWeather,
42-
};
43-
44-
// ==========================================================
45-
// PROMPT ENGINEERING
46-
// ==========================================================
47-
48-
String buildToolDescriptions() {
49-
return """
50-
- `getCurrentTime()`: Use this tool to get the current time. It takes no arguments.
51-
- `getWeather(city: string)`: Use this tool to get the weather for a specific city.
52-
""";
53-
}
54-
55-
String buildSystemPrompt() {
56-
return """
57-
You are an expert at choosing the right function to call to answer a user's question. You have access to the following tools:
58-
${buildToolDescriptions()}
59-
60-
Your goal is to respond with ONLY a JSON object that represents the function call(s) needed.
61-
- The JSON should be an array of objects.
62-
- Each object must have a "tool_name" and an "arguments" map.
63-
- If no tool is needed, or if you don't have enough information (e.g., the user asks for weather but doesn't name a city), respond with an empty JSON array: `[]`.
64-
- Do NOT add any other text, explanation, or conversation.
65-
66-
Example:
67-
User: What's the weather like in Paris?
68-
Your response:
69-
[
70-
{"tool_name": "getWeather", "arguments": {"city": "Paris"}}
71-
]
72-
73-
User: what time is it?
74-
Your response:
75-
[
76-
{"tool_name": "getCurrentTime", "arguments": {}}
77-
]
78-
79-
User: Tell me a joke.
80-
Your response:
81-
[]
82-
""";
83-
}
84-
85-
// ==========================================================
86-
// MAIN AGENT LOGIC (This is where the fix is)
87-
// ==========================================================
88-
89-
Future<void> main() async {
8+
void main() async {
909
try {
91-
// --- Standard LLM Setup (using your working example's structure) ---
9210
Llama.libraryPath = "bin/MAC_ARM64/libllama.dylib";
9311
String modelPath = "/Users/adel/Workspace/gguf/gemma-3-4b-it-q4_0.gguf";
9412

95-
final modelParams = ModelParams()..nGpuLayers = -1;
96-
final contextParams = ContextParams()..nCtx = 2048;
97-
final samplerParams = SamplerParams();
98-
99-
final llama =
100-
Llama(modelPath, modelParams, contextParams, samplerParams, false);
101-
102-
// --- The Agent's Main Loop ---
103-
while (true) {
104-
stdout.write("\nAsk me something (or type 'exit'): ");
105-
final userInput = stdin.readLineSync();
106-
107-
if (userInput == null || userInput.toLowerCase() == 'exit') {
108-
break;
109-
}
13+
ChatHistory history = ChatHistory()
14+
..addMessage(role: Role.user, content: prompt)
15+
..addMessage(role: Role.assistant, content: "");
11016

111-
print("🧠 Thinking...");
17+
final modelParams = ModelParams()..nGpuLayers = 99;
11218

113-
// 1. Build the prompt using ChatHistory (THE CORRECT WAY)
114-
final history = ChatHistory()
115-
..addMessage(role: Role.user, content: buildSystemPrompt())
116-
..addMessage(role: Role.user, content: userInput)
117-
..addMessage(role: Role.assistant, content: '');
19+
final contextParams = ContextParams()
20+
..nPredict = -1
21+
..nCtx = 8192
22+
..nBatch = 8192;
11823

119-
// 2. Set the prompt and get the response by streaming tokens (THE CORRECT WAY)
120-
llama.setPrompt(history.exportFormat(ChatFormat.gemini,
121-
leaveLastAssistantOpen: true));
24+
final samplerParams = SamplerParams()
25+
..temp = 0.7
26+
..topK = 64
27+
..topP = 0.95
28+
..penaltyRepeat = 1.1;
12229

123-
final responseBuffer = StringBuffer();
124-
while (true) {
125-
final (token, done) = llama.getNext();
126-
responseBuffer.write(token);
127-
if (done) break;
128-
}
129-
final llmResponse = responseBuffer.toString().trim();
130-
131-
print("✅ LLM's Plan (raw response): $llmResponse");
132-
133-
// 3. The "Coordinator" (our code) reads the plan and executes it.
134-
try {
135-
final jsonRegex = RegExp(r'\[.*\]', dotAll: true);
136-
final match = jsonRegex.firstMatch(llmResponse);
137-
138-
if (match == null) {
139-
print(
140-
"🤖 I couldn't decide on a tool to use. Here's my raw thought: $llmResponse");
141-
continue;
142-
}
143-
144-
final jsonString = match.group(0)!;
145-
final List<dynamic> toolCalls = jsonDecode(jsonString);
146-
147-
if (toolCalls.isEmpty) {
148-
print(
149-
"🤖 I don't have a tool for that. Please ask me about the time or weather.");
150-
continue;
151-
}
152-
153-
// 4. Execute each tool in the plan
154-
for (var call in toolCalls) {
155-
final toolName = call['tool_name'];
156-
final arguments = call['arguments'] as Map<String, dynamic>;
157-
final tool = toolBox[toolName];
30+
Llama llama =
31+
Llama(modelPath, modelParams, contextParams, samplerParams, false);
15832

159-
if (tool != null) {
160-
print("🛠️ Executing tool: $toolName with args: $arguments");
161-
if (toolName == 'getWeather') {
162-
final result = await getWeather(arguments['city']);
163-
print("✔️ Result: $result");
164-
} else if (toolName == 'getCurrentTime') {
165-
final result = getCurrentTime();
166-
print("✔️ Result: $result");
167-
}
168-
} else {
169-
print("❌ Error: Tool '$toolName' not found in my toolbox.");
170-
}
171-
}
172-
} catch (e) {
173-
print("❌ Error processing the LLM's plan: $e");
174-
print(" My plan was: $llmResponse");
175-
}
33+
llama.setPrompt(
34+
history.exportFormat(ChatFormat.gemini, leaveLastAssistantOpen: true));
35+
while (true) {
36+
var (token, done) = llama.getNext();
37+
stdout.write(token);
38+
if (done) break;
17639
}
40+
stdout.write("\n");
17741

17842
llama.dispose();
179-
print("Goodbye!");
18043
} catch (e) {
181-
print("\nFatal Error: ${e.toString()}");
44+
print("\nError: ${e.toString()}");
18245
}
18346
}

‎example/test_vision.finetuned.dart‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ Future<void> main() async {
88

99
final contextParams = ContextParams()
1010
..nPredict = -1
11-
..nCtx = 8192
12-
..nBatch = 8192;
11+
..nCtx = 2048
12+
..nUbatch = 2048
13+
..nBatch = 2048;
1314

1415
final samplerParams = SamplerParams()
1516
..temp = 0.7
@@ -18,12 +19,13 @@ Future<void> main() async {
1819
..penaltyRepeat = 1.1;
1920

2021
final llama = Llama(
21-
"/Users/adel/Downloads/gemma-3n-E2B-it-plantvillage-model-Q4_K_M.gguf",
22+
// "/Users/adel/Downloads/gemma-3n-E2B-it-plantvillage-model.gguf",
23+
"/Users/adel/Downloads/gemma-3-4B-it-plantvillage-model-Q4_K_M.gguf",
2224
modelParams,
2325
contextParams,
2426
samplerParams,
2527
true,
26-
"/Users/adel/Downloads/gemma-3n-E2B-it-plantvillage -mmproj.gguf");
28+
"/Users/adel/Downloads/gemma-3-4B-it-plantvillage-mmproj.gguf");
2729

2830
final image =
2931
LlamaImage.fromFile(File("/Users/adel/Downloads/plantvillage-test.png"));

0 commit comments

Comments
 (0)