diff --git a/README.md b/README.md index 6b802ac..968e275 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ CI worker for processing RiverBench datasets and other repositories. Please file issues, bugs, and other requests in the **[main issue tracker](https://github.com/RiverBench/RiverBench/issues)**. +This project uses Scala Next (3.8.4) and JDK 25. + ## Usage In the container, use `ci-worker ` to run the worker. diff --git a/src/main/scala/commands/DatasetDocGenCommand.scala b/src/main/scala/commands/DatasetDocGenCommand.scala index 70ed1df..805be8c 100644 --- a/src/main/scala/commands/DatasetDocGenCommand.scala +++ b/src/main/scala/commands/DatasetDocGenCommand.scala @@ -170,7 +170,8 @@ object DatasetDocGenCommand extends Command: |### Download links | |The dataset is published in a few size variants, each containing a specific number of stream elements. - |For each size, there are three distribution types available: flat (just an N-Triples/N-Quads file), + |For each size, there are three distribution types available: flat + |(an N-Triples/N-Quads file in the [RDF Message Log format](https://w3c-cg.github.io/rsp/spec/messages)), |streaming (a .tar.gz archive with Turtle/TriG files, one file per stream element), |and [Jelly](https://w3id.org/jelly) (a native binary format for streaming RDF). |See the [documentation](../../documentation/dataset-release-format.md) for more details. diff --git a/src/main/scala/commands/PackageCommand.scala b/src/main/scala/commands/PackageCommand.scala index fcb1251..0c46253 100644 --- a/src/main/scala/commands/PackageCommand.scala +++ b/src/main/scala/commands/PackageCommand.scala @@ -328,6 +328,12 @@ object PackageCommand extends Command: } StreamUtil.broadcastSink(sinks) + /** + * RDF Messages delimiter. + * https://w3c-cg.github.io/rsp/spec/messages#message-delimiter + */ + private val messageDelimiter = "MESSAGE\n".getBytes + private def packageFlatSerializeFlow(metadata: MetadataInfo): Flow[(DatasetGraph, Long), ByteString, NotUsed] = Flow[(DatasetGraph, Long)] .map((ds, _) => ds.find().asScala.toSeq.sorted) @@ -341,9 +347,13 @@ object PackageCommand extends Command: val writer = StreamRDFWriter.getWriterStream(os, Lang.NQUADS) quads.foreach(q => writer.quad(q)) writer.finish() + os.write(messageDelimiter) os.toByteArray }) .map(ByteString.fromArrayUnsafe) + // RDF Messages version announcement + // https://w3c-cg.github.io/rsp/spec/messages#version-announcement + .prepend(Source.single(ByteString("VERSION \"1.1-messages\"\n"))) /** * Creates a sink that writes the data to flat files diff --git a/src/main/scala/util/io/SaveResult.scala b/src/main/scala/util/io/SaveResult.scala index 769e4b7..601509d 100644 --- a/src/main/scala/util/io/SaveResult.scala +++ b/src/main/scala/util/io/SaveResult.scala @@ -19,6 +19,8 @@ case class SaveResult(io: IOResult, name: String, size: Long, md5: String, sha1: dType match case DistType.Flat => + // Flat files using the RDF Message Log format: + // https://w3c-cg.github.io/rsp/spec/messages#turtle if mi.streamTypes.exists(_.elementType == ElementType.Triple) then distRes.addProperty(RdfUtil.dcatMediaType, "application/n-triples") else