Skip to content

Commit 38477a5

Browse files
johnoliverjohnoliver
committed
Add support for HSQLDB store
1 parent 14a5b3d commit 38477a5

File tree

17 files changed

+622
-109
lines changed

17 files changed

+622
-109
lines changed

‎aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,22 @@
88
import com.microsoft.semantickernel.aiservices.openai.OpenAiService;
99
import com.microsoft.semantickernel.exceptions.AIException;
1010
import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder;
11-
import com.microsoft.semantickernel.services.textcompletion.TextGenerationService;
1211
import com.microsoft.semantickernel.services.textembedding.Embedding;
1312
import com.microsoft.semantickernel.services.textembedding.TextEmbeddingGenerationService;
13+
import java.util.ArrayList;
14+
import java.util.Arrays;
15+
import java.util.List;
16+
import javax.annotation.Nullable;
1417
import org.slf4j.Logger;
1518
import org.slf4j.LoggerFactory;
1619
import reactor.core.publisher.Mono;
1720

18-
import javax.annotation.Nullable;
19-
import java.util.ArrayList;
20-
import java.util.List;
21-
2221
/**
2322
* An OpenAI implementation of a {@link TextEmbeddingGenerationService}.
24-
*
2523
*/
2624
public class OpenAITextEmbeddingGenerationService extends OpenAiService<OpenAIAsyncClient>
2725
implements TextEmbeddingGenerationService {
26+
2827
private static final Logger LOGGER = LoggerFactory
2928
.getLogger(OpenAITextEmbeddingGenerationService.class);
3029
private static final int DEFAULT_DIMENSIONS = 1536;
@@ -33,10 +32,10 @@ public class OpenAITextEmbeddingGenerationService extends OpenAiService<OpenAIAs
3332
/**
3433
* Creates a new {@link OpenAITextEmbeddingGenerationService}.
3534
*
36-
* @param client OpenAI client
35+
* @param client OpenAI client
3736
* @param deploymentName deployment name
38-
* @param modelId OpenAI model id
39-
* @param serviceId Service id
37+
* @param modelId OpenAI model id
38+
* @param serviceId Service id
4039
*/
4140
public OpenAITextEmbeddingGenerationService(
4241
OpenAIAsyncClient client,
@@ -57,6 +56,24 @@ public static Builder builder() {
5756
return new Builder();
5857
}
5958

59+
/**
60+
* Generates embeddings for the given data.
61+
*
62+
* @param data The data to generate embeddings for.
63+
* @return A Mono that completes with the embeddings.
64+
*/
65+
@Override
66+
public Mono<Embedding> generateEmbeddingAsync(String data) {
67+
return this.internalGenerateTextEmbeddingsAsync(Arrays.asList(data))
68+
.flatMap(embeddings -> {
69+
if (embeddings.isEmpty()) {
70+
return Mono.empty();
71+
}
72+
73+
return Mono.just(embeddings.get(0));
74+
});
75+
}
76+
6077
/**
6178
* Generates embeddings for the given data.
6279
*
@@ -88,6 +105,7 @@ protected Mono<List<Embedding>> internalGenerateTextEmbeddingsAsync(List<String>
88105
*/
89106
public static class Builder extends
90107
OpenAiServiceBuilder<OpenAIAsyncClient, OpenAITextEmbeddingGenerationService, OpenAITextEmbeddingGenerationService.Builder> {
108+
91109
private int dimensions = DEFAULT_DIMENSIONS;
92110

93111
/**

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute;
66
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute;
77
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute;
8+
import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction;
89

910
import java.util.List;
1011

@@ -23,19 +24,19 @@ public class Hotel {
2324
private final String description;
2425

2526
@JsonProperty("summaryEmbedding1")
26-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean")
27+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE)
2728
private final List<Float> euclidean;
2829

2930
@JsonProperty("summaryEmbedding2")
30-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance")
31+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE)
3132
private final List<Float> cosineDistance;
3233

3334
@JsonProperty("summaryEmbedding3")
34-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct")
35+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT)
3536
private final List<Float> dotProduct;
3637

3738
@JsonProperty("indexedSummaryEmbedding")
38-
@VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = "euclidean")
39+
@VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE)
3940
private final List<Float> indexedEuclidean;
4041
@VectorStoreRecordDataAttribute
4142
private double rating;

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ private JDBCVectorStoreRecordCollection<Hotel> buildRecordCollection(QueryProvid
7979
throw new IllegalArgumentException("Unknown query provider: " + provider);
8080
}
8181

82-
8382
JDBCVectorStoreRecordCollection<Hotel> recordCollection = new JDBCVectorStoreRecordCollection<>(
8483
dataSource,
8584
collectionName,

‎api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute;
66
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute;
77
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute;
8-
8+
import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction;
99
import java.util.List;
1010

1111
public class Hotel {
12+
1213
@VectorStoreRecordKeyAttribute
1314
private final String id;
1415

@@ -23,15 +24,15 @@ public class Hotel {
2324
private final String description;
2425

2526
@JsonProperty("summaryEmbedding1")
26-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean")
27+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE)
2728
private final List<Float> euclidean;
2829

2930
@JsonProperty("summaryEmbedding2")
30-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance")
31+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE)
3132
private final List<Float> cosineDistance;
3233

3334
@JsonProperty("summaryEmbedding3")
34-
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct")
35+
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT)
3536
private final List<Float> dotProduct;
3637
@VectorStoreRecordDataAttribute
3738
private double rating;
@@ -42,14 +43,14 @@ public Hotel() {
4243

4344
@JsonCreator
4445
public Hotel(
45-
@JsonProperty("id") String id,
46-
@JsonProperty("name") String name,
47-
@JsonProperty("code") int code,
48-
@JsonProperty("summary") String description,
49-
@JsonProperty("summaryEmbedding1") List<Float> euclidean,
50-
@JsonProperty("summaryEmbedding2") List<Float> cosineDistance,
51-
@JsonProperty("summaryEmbedding3") List<Float> dotProduct,
52-
@JsonProperty("rating") double rating) {
46+
@JsonProperty("id") String id,
47+
@JsonProperty("name") String name,
48+
@JsonProperty("code") int code,
49+
@JsonProperty("summary") String description,
50+
@JsonProperty("summaryEmbedding1") List<Float> euclidean,
51+
@JsonProperty("summaryEmbedding2") List<Float> cosineDistance,
52+
@JsonProperty("summaryEmbedding3") List<Float> dotProduct,
53+
@JsonProperty("rating") double rating) {
5354
this.id = id;
5455
this.name = name;
5556
this.code = code;

‎samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/Example63_ChatCompletionPrompts.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class Example63_ChatCompletionPrompts {
1919
// Only required if AZURE_CLIENT_KEY is set
2020
private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT");
2121
private static final String MODEL_ID = System.getenv()
22-
.getOrDefault("MODEL_ID", "gpt-35-turbo");
22+
.getOrDefault("MODEL_ID", "gpt-35-turbo-2");
2323

2424
public static void main(String[] args) throws InterruptedException {
2525

‎semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,29 @@
11
// Copyright (c) Microsoft. All rights reserved.
22
package com.microsoft.semantickernel.services.textembedding;
33

4-
import com.microsoft.semantickernel.builders.SemanticKernelBuilder;
54
import com.microsoft.semantickernel.services.AIService;
6-
import reactor.core.publisher.Mono;
7-
85
import java.util.List;
6+
import reactor.core.publisher.Mono;
97

10-
/** Interface for text embedding generation services */
8+
/**
9+
* Interface for text embedding generation services
10+
*/
1111
public interface EmbeddingGenerationService<TValue> extends AIService {
12+
1213
/**
1314
* Generates a list of embeddings associated to the data
1415
*
1516
* @param data List of texts to generate embeddings for
1617
* @return List of embeddings of each data point
1718
*/
1819
Mono<List<Embedding>> generateEmbeddingsAsync(List<TValue> data);
20+
21+
/**
22+
* Generates an embedding associated to the data
23+
*
24+
* @param data Text to generate embedding for
25+
* @return Embedding of the data
26+
*/
27+
28+
Mono<Embedding> generateEmbeddingAsync(TValue data);
1929
}

‎semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java‎

Lines changed: 42 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@
2929
import java.util.ArrayList;
3030
import java.util.HashMap;
3131
import java.util.HashSet;
32-
import java.util.Iterator;
3332
import java.util.Locale;
3433
import java.util.Map.Entry;
3534
import java.util.Optional;
3635
import java.util.Set;
3736
import java.util.stream.Collectors;
3837
import javax.annotation.Nonnull;
3938
import javax.annotation.Nullable;
39+
import org.apache.commons.text.StringEscapeUtils;
4040
import reactor.core.publisher.Mono;
4141

4242
/**
@@ -89,8 +89,7 @@ public Object resolve(Object context, String name) {
8989
if ("role".equalsIgnoreCase(name)) {
9090
return ((ChatMessageContent) context).getAuthorRole().name();
9191
} else if ("content".equalsIgnoreCase(name)) {
92-
return ContextVariableTypeConverter
93-
.escapeXmlString(((ChatMessageContent) context).getContent());
92+
return ((ChatMessageContent) context).getContent();
9493
}
9594
}
9695
return UNRESOLVED;
@@ -131,29 +130,48 @@ private static class ContextVariableResolver implements ValueResolver {
131130
@Override
132131
public Object resolve(Object context, String name) {
133132
Object value = null;
133+
ContextVariable<?> variable = null;
134134
if (context instanceof KernelFunctionArguments) {
135-
ContextVariable<?> variable = ((KernelFunctionArguments) context).get(name);
136-
value = variable != null ? variable.getValue() : UNRESOLVED;
137-
}
138-
if (context instanceof ContextVariable) {
139-
value = ((ContextVariable<?>) context).getValue();
135+
variable = ((KernelFunctionArguments) context).get(name);
136+
} else if (context instanceof ContextVariable) {
137+
variable = ((ContextVariable<?>) context);
140138
}
141-
if (value == null) {
139+
140+
if (variable == null || variable.getValue() == null) {
142141
return UNRESOLVED;
143-
} else {
142+
}
143+
144+
value = variable.getValue();
145+
146+
if (value instanceof Iterable) {
144147
return value;
148+
} else {
149+
// It is likely this will come escaped, but will be re escaped by the handlebars engine
150+
return promptString(variable);
145151
}
146152
}
147153

148154
@Override
149155
public Object resolve(Object context) {
150156
if (context instanceof ContextVariable) {
151157
Object result = ((ContextVariable<?>) context).getValue();
152-
return result != null ? result : UNRESOLVED;
158+
159+
if (result == null) {
160+
return UNRESOLVED;
161+
} else if (result instanceof Iterable) {
162+
return result;
163+
} else {
164+
return promptString(((ContextVariable<?>) context));
165+
}
153166
}
154167
return UNRESOLVED;
155168
}
156169

170+
private String promptString(ContextVariable<?> context) {
171+
// This will come escaped, but will be re escaped by the handlebars engine
172+
return StringEscapeUtils.unescapeXml(context.toPromptString());
173+
}
174+
157175
@Override
158176
public Set<Entry<String, Object>> propertySet(Object context) {
159177
if (context instanceof KernelFunctionArguments) {
@@ -192,22 +210,19 @@ public HandleBarsPromptTemplateHandler(
192210
}
193211

194212
private Helper<Object> handleEach(InvocationContext invocationContext) {
195-
return (context, options) -> {
196-
if (context instanceof ContextVariable) {
197-
return ((ContextVariable<?>) context)
213+
return (variable, options) -> {
214+
if (variable instanceof ContextVariable) {
215+
return ((ContextVariable<?>) variable)
198216
.toPromptString(invocationContext.getContextVariableTypes());
199217
}
200218

201-
if (context instanceof Iterable) {
219+
if (variable instanceof Iterable) {
202220
StringBuilder sb = new StringBuilder();
203-
Iterator<?> iterator = ((Iterable<?>) context).iterator();
204-
while (iterator.hasNext()) {
205-
Object element = iterator.next();
221+
222+
for (Object element : (Iterable<?>) variable) {
206223
if (element instanceof KernelPlugin) {
207224
KernelPlugin plugin = (KernelPlugin) element;
208-
Iterator<KernelFunction<?>> functions = plugin.iterator();
209-
while (functions.hasNext()) {
210-
KernelFunction<?> function = functions.next();
225+
for (KernelFunction<?> function : plugin) {
211226
sb.append(options.fn(function));
212227
}
213228
} else {
@@ -218,10 +233,11 @@ private Helper<Object> handleEach(InvocationContext invocationContext) {
218233
}
219234

220235
ContextVariableType type = invocationContext.getContextVariableTypes()
221-
.getVariableTypeForClass(context.getClass());
236+
.getVariableTypeForClass(variable.getClass());
222237
if (type != null) {
223-
return type.getConverter()
224-
.toPromptString(invocationContext.getContextVariableTypes(), context);
238+
return type
239+
.getConverter()
240+
.toPromptString(invocationContext.getContextVariableTypes(), variable);
225241
}
226242
return null;
227243
};
@@ -248,7 +264,8 @@ private CharSequence handleMessage(Object context, Options options)
248264
return new Handlebars.SafeString(
249265
String.format(
250266
"<message role=\"%s\">%s</message>",
251-
role.toLowerCase(Locale.ROOT), content));
267+
role.toLowerCase(Locale.ROOT),
268+
content));
252269
}
253270
return null;
254271
}

0 commit comments

Comments
 (0)