@@ -4,35 +4,31 @@ import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
44import it .agilelab .bigdata .wasp .models .JobStatus .JobStatus
55import spray .json ._
66
7-
87object JobStatus extends Enumeration {
98 type JobStatus = Value
109
1110 val PENDING, PROCESSING, SUCCESSFUL, FAILED, STOPPED = Value
1211}
1312
14-
15- case class BatchJobModel (override val name : String ,
16- description : String ,
17- owner : String ,
18- system : Boolean ,
19- creationTime : Long ,
20- etl : BatchETL ,
21- exclusivityConfig : BatchJobExclusionConfig = BatchJobExclusionConfig (
22- isFullyExclusive = true ,
23- Seq .empty[String ])
24- )
25- extends Model
26-
27-
28- case class BatchJobInstanceModel (override val name : String ,
29- instanceOf : String ,
30- startTimestamp : Long ,
31- currentStatusTimestamp : Long ,
32- status : JobStatus ,
33- restConfig : Config = ConfigFactory .empty,
34- error : Option [String ] = None
35- ) extends Model
13+ case class BatchJobModel (
14+ override val name : String ,
15+ description : String ,
16+ owner : String ,
17+ system : Boolean ,
18+ creationTime : Long ,
19+ etl : BatchETL ,
20+ exclusivityConfig : BatchJobExclusionConfig = BatchJobExclusionConfig (isFullyExclusive = true , Seq .empty[String ])
21+ ) extends Model
22+
23+ case class BatchJobInstanceModel (
24+ override val name : String ,
25+ instanceOf : String ,
26+ startTimestamp : Long ,
27+ currentStatusTimestamp : Long ,
28+ status : JobStatus ,
29+ restConfig : Config = ConfigFactory .empty,
30+ error : Option [String ] = None
31+ ) extends Model
3632
3733sealed trait BatchETL {
3834 val name : String
@@ -42,25 +38,51 @@ sealed trait BatchETL {
4238 var isActive : Boolean
4339}
4440
45- case class BatchETLModel (name : String ,
46- inputs : List [ReaderModel ],
47- output : WriterModel ,
48- mlModels : List [MlModelOnlyInfo ],
49- strategy : Option [StrategyModel ],
50- kafkaAccessType : String ,
51- group : String = " default" ,
52- var isActive : Boolean = false ) extends BatchETL
41+ case class BatchETLModel (
42+ name : String ,
43+ inputs : List [ReaderModel ],
44+ output : WriterModel ,
45+ mlModels : List [MlModelOnlyInfo ],
46+ strategy : Option [StrategyModel ],
47+ kafkaAccessType : String ,
48+ group : String = " default" ,
49+ var isActive : Boolean = false
50+ ) extends BatchETL
51+
5352object BatchETLModel {
5453 val TYPE = " BatchETL"
54+
55+ // Alternative constructor to avoid passing (often) empty machine learning models
56+ def create (
57+ name : String ,
58+ inputs : List [ReaderModel ],
59+ output : WriterModel ,
60+ strategy : Option [StrategyModel ],
61+ kafkaAccessType : String ,
62+ group : String ,
63+ isActive : Boolean
64+ ): BatchETLModel =
65+ new BatchETLModel (
66+ name = name,
67+ inputs = inputs,
68+ output = output,
69+ mlModels = List .empty[MlModelOnlyInfo ],
70+ strategy = strategy,
71+ kafkaAccessType = kafkaAccessType,
72+ group = group,
73+ isActive = isActive
74+ )
5575}
5676
57- case class BatchGdprETLModel (name : String ,
58- dataStores : List [DataStoreConf ],
59- strategyConfig : String ,
60- inputs : List [ReaderModel ],
61- output : WriterModel ,
62- group : String = " default" ,
63- var isActive : Boolean = false ) extends BatchETL {
77+ case class BatchGdprETLModel (
78+ name : String ,
79+ dataStores : List [DataStoreConf ],
80+ strategyConfig : String ,
81+ inputs : List [ReaderModel ],
82+ output : WriterModel ,
83+ group : String = " default" ,
84+ var isActive : Boolean = false
85+ ) extends BatchETL {
6486 val strategy : GdprStrategyModel = GdprStrategyModel (
6587 " it.agilelab.bigdata.wasp.consumers.spark.strategies.gdpr.GdprStrategy" ,
6688 dataStores,
@@ -70,13 +92,15 @@ case class BatchGdprETLModel(name: String,
7092object BatchGdprETLModel {
7193 val TYPE = " BatchGdprETL"
7294
73- def create (name : String ,
74- dataStores : List [DataStoreConf ],
75- strategyConfig : Config ,
76- inputs : List [ReaderModel ],
77- output : WriterModel ,
78- group : String = " default" ,
79- isActive : Boolean = false ): BatchGdprETLModel = {
95+ def create (
96+ name : String ,
97+ dataStores : List [DataStoreConf ],
98+ strategyConfig : Config ,
99+ inputs : List [ReaderModel ],
100+ output : WriterModel ,
101+ group : String = " default" ,
102+ isActive : Boolean = false
103+ ): BatchGdprETLModel = {
80104 BatchGdprETLModel (
81105 name,
82106 dataStores,
@@ -89,7 +113,6 @@ object BatchGdprETLModel {
89113 }
90114}
91115
92-
93116case class BatchJobExclusionConfig (isFullyExclusive : Boolean , restConfigExclusiveParams : Seq [String ])
94117
95118trait BatchJobJsonSupport extends DefaultJsonProtocol {
@@ -100,24 +123,27 @@ trait BatchJobJsonSupport extends DefaultJsonProtocol {
100123 strategyModelFormat: RootJsonFormat[StrategyModel]) = {
101124 implicit val
102125 }
103- */
126+ */
104127
105- def createBatchETLFormat (implicit batchETLModelFormat : RootJsonFormat [BatchETLModel ],
106- batchGdprETLModelFormat : RootJsonFormat [BatchGdprETLModel ]): RootJsonFormat [BatchETL ] = {
128+ def createBatchETLFormat (
129+ implicit batchETLModelFormat : RootJsonFormat [BatchETLModel ],
130+ batchGdprETLModelFormat : RootJsonFormat [BatchGdprETLModel ]
131+ ): RootJsonFormat [BatchETL ] = {
107132
108133 new RootJsonFormat [BatchETL ] {
109134 override def read (json : JsValue ): BatchETL = {
110135 json.asJsObject.getFields(" type" ) match {
111- case Seq (JsString (" BatchETLModel" )) => json.convertTo[BatchETLModel ]
136+ case Seq (JsString (" BatchETLModel" )) => json.convertTo[BatchETLModel ]
112137 case Seq (JsString (" BatchGdprETLModel" )) => json.convertTo[BatchGdprETLModel ]
113- case _ => throw DeserializationException (" Unknown json" )
138+ case _ => throw DeserializationException (" Unknown json" )
114139 }
115140 }
116141
117- override def write (obj : BatchETL ): JsValue = JsObject (obj match {
118- case etl : BatchETLModel => etl.toJson.asJsObject.fields + (" type" -> JsString (" BatchETLModel" ))
119- case gdpr : BatchGdprETLModel => gdpr.toJson.asJsObject.fields + (" type" -> JsString (" BatchGdprETLModel" ))
120- })
142+ override def write (obj : BatchETL ): JsValue =
143+ JsObject (obj match {
144+ case etl : BatchETLModel => etl.toJson.asJsObject.fields + (" type" -> JsString (" BatchETLModel" ))
145+ case gdpr : BatchGdprETLModel => gdpr.toJson.asJsObject.fields + (" type" -> JsString (" BatchGdprETLModel" ))
146+ })
121147 }
122148 }
123149}
0 commit comments