Skip to content

Commit 5b19e2e

Browse files
Peppo Lorussoagilelab-tmnd1991
authored andcommitted
[#521] set meaningful defaults in models
# New features and improvements - Set empty list as default machine learning models to be applied - Set `timed=false` as default in RawModel # Related issue Closes #521
1 parent ddefbf5 commit 5b19e2e

File tree

5 files changed

+258
-177
lines changed

5 files changed

+258
-177
lines changed

model/src/main/scala/it/agilelab/bigdata/wasp/models/BatchJobModel.scala

Lines changed: 81 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,31 @@ import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
44
import it.agilelab.bigdata.wasp.models.JobStatus.JobStatus
55
import spray.json._
66

7-
87
object JobStatus extends Enumeration {
98
type JobStatus = Value
109

1110
val PENDING, PROCESSING, SUCCESSFUL, FAILED, STOPPED = Value
1211
}
1312

14-
15-
case class BatchJobModel(override val name: String,
16-
description: String,
17-
owner: String,
18-
system: Boolean,
19-
creationTime: Long,
20-
etl: BatchETL,
21-
exclusivityConfig: BatchJobExclusionConfig = BatchJobExclusionConfig(
22-
isFullyExclusive = true,
23-
Seq.empty[String])
24-
)
25-
extends Model
26-
27-
28-
case class BatchJobInstanceModel(override val name: String,
29-
instanceOf: String,
30-
startTimestamp: Long,
31-
currentStatusTimestamp: Long,
32-
status: JobStatus,
33-
restConfig: Config = ConfigFactory.empty,
34-
error: Option[String] = None
35-
) extends Model
13+
case class BatchJobModel(
14+
override val name: String,
15+
description: String,
16+
owner: String,
17+
system: Boolean,
18+
creationTime: Long,
19+
etl: BatchETL,
20+
exclusivityConfig: BatchJobExclusionConfig = BatchJobExclusionConfig(isFullyExclusive = true, Seq.empty[String])
21+
) extends Model
22+
23+
case class BatchJobInstanceModel(
24+
override val name: String,
25+
instanceOf: String,
26+
startTimestamp: Long,
27+
currentStatusTimestamp: Long,
28+
status: JobStatus,
29+
restConfig: Config = ConfigFactory.empty,
30+
error: Option[String] = None
31+
) extends Model
3632

3733
sealed trait BatchETL {
3834
val name: String
@@ -42,25 +38,51 @@ sealed trait BatchETL {
4238
var isActive: Boolean
4339
}
4440

45-
case class BatchETLModel(name: String,
46-
inputs: List[ReaderModel],
47-
output: WriterModel,
48-
mlModels: List[MlModelOnlyInfo],
49-
strategy: Option[StrategyModel],
50-
kafkaAccessType: String,
51-
group: String = "default",
52-
var isActive: Boolean = false) extends BatchETL
41+
case class BatchETLModel(
42+
name: String,
43+
inputs: List[ReaderModel],
44+
output: WriterModel,
45+
mlModels: List[MlModelOnlyInfo],
46+
strategy: Option[StrategyModel],
47+
kafkaAccessType: String,
48+
group: String = "default",
49+
var isActive: Boolean = false
50+
) extends BatchETL
51+
5352
object BatchETLModel {
5453
val TYPE = "BatchETL"
54+
55+
// Alternative constructor to avoid passing (often) empty machine learning models
56+
def create(
57+
name: String,
58+
inputs: List[ReaderModel],
59+
output: WriterModel,
60+
strategy: Option[StrategyModel],
61+
kafkaAccessType: String,
62+
group: String,
63+
isActive: Boolean
64+
): BatchETLModel =
65+
new BatchETLModel(
66+
name = name,
67+
inputs = inputs,
68+
output = output,
69+
mlModels = List.empty[MlModelOnlyInfo],
70+
strategy = strategy,
71+
kafkaAccessType = kafkaAccessType,
72+
group = group,
73+
isActive = isActive
74+
)
5575
}
5676

57-
case class BatchGdprETLModel(name: String,
58-
dataStores: List[DataStoreConf],
59-
strategyConfig: String,
60-
inputs: List[ReaderModel],
61-
output: WriterModel,
62-
group: String = "default",
63-
var isActive: Boolean = false) extends BatchETL {
77+
case class BatchGdprETLModel(
78+
name: String,
79+
dataStores: List[DataStoreConf],
80+
strategyConfig: String,
81+
inputs: List[ReaderModel],
82+
output: WriterModel,
83+
group: String = "default",
84+
var isActive: Boolean = false
85+
) extends BatchETL {
6486
val strategy: GdprStrategyModel = GdprStrategyModel(
6587
"it.agilelab.bigdata.wasp.consumers.spark.strategies.gdpr.GdprStrategy",
6688
dataStores,
@@ -70,13 +92,15 @@ case class BatchGdprETLModel(name: String,
7092
object BatchGdprETLModel {
7193
val TYPE = "BatchGdprETL"
7294

73-
def create(name: String,
74-
dataStores: List[DataStoreConf],
75-
strategyConfig: Config,
76-
inputs: List[ReaderModel],
77-
output: WriterModel,
78-
group: String = "default",
79-
isActive: Boolean = false): BatchGdprETLModel = {
95+
def create(
96+
name: String,
97+
dataStores: List[DataStoreConf],
98+
strategyConfig: Config,
99+
inputs: List[ReaderModel],
100+
output: WriterModel,
101+
group: String = "default",
102+
isActive: Boolean = false
103+
): BatchGdprETLModel = {
80104
BatchGdprETLModel(
81105
name,
82106
dataStores,
@@ -89,7 +113,6 @@ object BatchGdprETLModel {
89113
}
90114
}
91115

92-
93116
case class BatchJobExclusionConfig(isFullyExclusive: Boolean, restConfigExclusiveParams: Seq[String])
94117

95118
trait BatchJobJsonSupport extends DefaultJsonProtocol {
@@ -100,24 +123,27 @@ trait BatchJobJsonSupport extends DefaultJsonProtocol {
100123
strategyModelFormat: RootJsonFormat[StrategyModel]) = {
101124
implicit val
102125
}
103-
*/
126+
*/
104127

105-
def createBatchETLFormat(implicit batchETLModelFormat: RootJsonFormat[BatchETLModel],
106-
batchGdprETLModelFormat: RootJsonFormat[BatchGdprETLModel]): RootJsonFormat[BatchETL] = {
128+
def createBatchETLFormat(
129+
implicit batchETLModelFormat: RootJsonFormat[BatchETLModel],
130+
batchGdprETLModelFormat: RootJsonFormat[BatchGdprETLModel]
131+
): RootJsonFormat[BatchETL] = {
107132

108133
new RootJsonFormat[BatchETL] {
109134
override def read(json: JsValue): BatchETL = {
110135
json.asJsObject.getFields("type") match {
111-
case Seq(JsString("BatchETLModel")) => json.convertTo[BatchETLModel]
136+
case Seq(JsString("BatchETLModel")) => json.convertTo[BatchETLModel]
112137
case Seq(JsString("BatchGdprETLModel")) => json.convertTo[BatchGdprETLModel]
113-
case _ => throw DeserializationException("Unknown json")
138+
case _ => throw DeserializationException("Unknown json")
114139
}
115140
}
116141

117-
override def write(obj: BatchETL): JsValue = JsObject(obj match {
118-
case etl: BatchETLModel => etl.toJson.asJsObject.fields + ("type" -> JsString("BatchETLModel"))
119-
case gdpr: BatchGdprETLModel => gdpr.toJson.asJsObject.fields + ("type" -> JsString("BatchGdprETLModel"))
120-
})
142+
override def write(obj: BatchETL): JsValue =
143+
JsObject(obj match {
144+
case etl: BatchETLModel => etl.toJson.asJsObject.fields + ("type" -> JsString("BatchETLModel"))
145+
case gdpr: BatchGdprETLModel => gdpr.toJson.asJsObject.fields + ("type" -> JsString("BatchGdprETLModel"))
146+
})
121147
}
122148
}
123149
}

0 commit comments

Comments
 (0)