Skip to content

Commit 0ed7572

Browse files
frittenthekefrittentheke
committed
Increase readiness and liveness probe timeouts, introduce an initial delay
Currently the default timeout of 1 second and no initial delay is applied to the probes of the runner pods. Depending on the startup time this can cause random Pod errors causing a whole TestRun to fail. At some point it might also make sense to introduce a startupProbe to cover the longer initial startup time a K6 instance / pod might need instead of ever increasing the runtime liveness and readiness checks. Since having the Liveness and Readiness checks be just the same makes not much sense, as the liveness check fail will cause the container to be restarted, this change also splits up those two tests, to allow for more individual configuration, be it timers or what is actually checked. Fixes #306 Signed-off-by: Christian Rohmann <[email protected]>
1 parent 40dce3f commit 0ed7572

File tree

2 files changed

+54
-3
lines changed

2 files changed

+54
-3
lines changed

‎pkg/resources/jobs/runner.go‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,8 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job,
185185
VolumeMounts: volumeMounts,
186186
Ports: ports,
187187
EnvFrom: k6.GetSpec().Runner.EnvFrom,
188-
LivenessProbe: generateProbe(k6.GetSpec().Runner.LivenessProbe),
189-
ReadinessProbe: generateProbe(k6.GetSpec().Runner.ReadinessProbe),
188+
LivenessProbe: generateLivenessProbe(k6.GetSpec().Runner.LivenessProbe),
189+
ReadinessProbe: generateReadinessProbe(k6.GetSpec().Runner.ReadinessProbe),
190190
SecurityContext: &k6.GetSpec().Runner.ContainerSecurityContext,
191191
}},
192192
TerminationGracePeriodSeconds: &zero,
@@ -276,11 +276,29 @@ func newAntiAffinity() *corev1.Affinity {
276276
}
277277
}
278278

279-
func generateProbe(configuredProbe *corev1.Probe) *corev1.Probe {
279+
func generateLivenessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
280280
if configuredProbe != nil {
281281
return configuredProbe
282282
}
283283
return &corev1.Probe{
284+
InitialDelaySeconds: 10,
285+
TimeoutSeconds: 3,
286+
ProbeHandler: corev1.ProbeHandler{
287+
HTTPGet: &corev1.HTTPGetAction{
288+
Path: "/v1/status",
289+
Port: intstr.IntOrString{IntVal: 6565},
290+
Scheme: "HTTP",
291+
},
292+
},
293+
}
294+
}
295+
296+
func generateReadinessProbe(configuredProbe *corev1.Probe) *corev1.Probe {
297+
if configuredProbe != nil {
298+
return configuredProbe
299+
}
300+
return &corev1.Probe{
301+
TimeoutSeconds: 3,
284302
ProbeHandler: corev1.ProbeHandler{
285303
HTTPGet: &corev1.HTTPGetAction{
286304
Path: "/v1/status",

‎pkg/resources/jobs/runner_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,8 @@ func TestNewRunnerJob(t *testing.T) {
339339
},
340340
},
341341
LivenessProbe: &corev1.Probe{
342+
InitialDelaySeconds: 10,
343+
TimeoutSeconds: 3,
342344
ProbeHandler: corev1.ProbeHandler{
343345
HTTPGet: &corev1.HTTPGetAction{
344346
Path: "/v1/status",
@@ -348,6 +350,7 @@ func TestNewRunnerJob(t *testing.T) {
348350
},
349351
},
350352
ReadinessProbe: &corev1.Probe{
353+
TimeoutSeconds: 3,
351354
ProbeHandler: corev1.ProbeHandler{
352355
HTTPGet: &corev1.HTTPGetAction{
353356
Path: "/v1/status",
@@ -463,6 +466,8 @@ func TestNewRunnerJobNoisy(t *testing.T) {
463466
VolumeMounts: script.VolumeMount(),
464467
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
465468
LivenessProbe: &corev1.Probe{
469+
InitialDelaySeconds: 10,
470+
TimeoutSeconds: 3,
466471
ProbeHandler: corev1.ProbeHandler{
467472
HTTPGet: &corev1.HTTPGetAction{
468473
Path: "/v1/status",
@@ -472,6 +477,7 @@ func TestNewRunnerJobNoisy(t *testing.T) {
472477
},
473478
},
474479
ReadinessProbe: &corev1.Probe{
480+
TimeoutSeconds: 3,
475481
ProbeHandler: corev1.ProbeHandler{
476482
HTTPGet: &corev1.HTTPGetAction{
477483
Path: "/v1/status",
@@ -578,6 +584,8 @@ func TestNewRunnerJobUnpaused(t *testing.T) {
578584
VolumeMounts: script.VolumeMount(),
579585
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
580586
LivenessProbe: &corev1.Probe{
587+
InitialDelaySeconds: 10,
588+
TimeoutSeconds: 3,
581589
ProbeHandler: corev1.ProbeHandler{
582590
HTTPGet: &corev1.HTTPGetAction{
583591
Path: "/v1/status",
@@ -587,6 +595,7 @@ func TestNewRunnerJobUnpaused(t *testing.T) {
587595
},
588596
},
589597
ReadinessProbe: &corev1.Probe{
598+
TimeoutSeconds: 3,
590599
ProbeHandler: corev1.ProbeHandler{
591600
HTTPGet: &corev1.HTTPGetAction{
592601
Path: "/v1/status",
@@ -693,6 +702,8 @@ func TestNewRunnerJobArguments(t *testing.T) {
693702
VolumeMounts: script.VolumeMount(),
694703
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
695704
LivenessProbe: &corev1.Probe{
705+
InitialDelaySeconds: 10,
706+
TimeoutSeconds: 3,
696707
ProbeHandler: corev1.ProbeHandler{
697708
HTTPGet: &corev1.HTTPGetAction{
698709
Path: "/v1/status",
@@ -702,6 +713,7 @@ func TestNewRunnerJobArguments(t *testing.T) {
702713
},
703714
},
704715
ReadinessProbe: &corev1.Probe{
716+
TimeoutSeconds: 3,
705717
ProbeHandler: corev1.ProbeHandler{
706718
HTTPGet: &corev1.HTTPGetAction{
707719
Path: "/v1/status",
@@ -809,6 +821,8 @@ func TestNewRunnerJobServiceAccount(t *testing.T) {
809821
VolumeMounts: script.VolumeMount(),
810822
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
811823
LivenessProbe: &corev1.Probe{
824+
InitialDelaySeconds: 10,
825+
TimeoutSeconds: 3,
812826
ProbeHandler: corev1.ProbeHandler{
813827
HTTPGet: &corev1.HTTPGetAction{
814828
Path: "/v1/status",
@@ -818,6 +832,7 @@ func TestNewRunnerJobServiceAccount(t *testing.T) {
818832
},
819833
},
820834
ReadinessProbe: &corev1.Probe{
835+
TimeoutSeconds: 3,
821836
ProbeHandler: corev1.ProbeHandler{
822837
HTTPGet: &corev1.HTTPGetAction{
823838
Path: "/v1/status",
@@ -939,6 +954,8 @@ func TestNewRunnerJobIstio(t *testing.T) {
939954
VolumeMounts: script.VolumeMount(),
940955
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
941956
LivenessProbe: &corev1.Probe{
957+
InitialDelaySeconds: 10,
958+
TimeoutSeconds: 3,
942959
ProbeHandler: corev1.ProbeHandler{
943960
HTTPGet: &corev1.HTTPGetAction{
944961
Path: "/v1/status",
@@ -948,6 +965,7 @@ func TestNewRunnerJobIstio(t *testing.T) {
948965
},
949966
},
950967
ReadinessProbe: &corev1.Probe{
968+
TimeoutSeconds: 3,
951969
ProbeHandler: corev1.ProbeHandler{
952970
HTTPGet: &corev1.HTTPGetAction{
953971
Path: "/v1/status",
@@ -1064,6 +1082,8 @@ func TestNewRunnerJobCloud(t *testing.T) {
10641082
VolumeMounts: script.VolumeMount(),
10651083
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
10661084
LivenessProbe: &corev1.Probe{
1085+
InitialDelaySeconds: 10,
1086+
TimeoutSeconds: 3,
10671087
ProbeHandler: corev1.ProbeHandler{
10681088
HTTPGet: &corev1.HTTPGetAction{
10691089
Path: "/v1/status",
@@ -1073,6 +1093,7 @@ func TestNewRunnerJobCloud(t *testing.T) {
10731093
},
10741094
},
10751095
ReadinessProbe: &corev1.Probe{
1096+
TimeoutSeconds: 3,
10761097
ProbeHandler: corev1.ProbeHandler{
10771098
HTTPGet: &corev1.HTTPGetAction{
10781099
Path: "/v1/status",
@@ -1181,6 +1202,8 @@ func TestNewRunnerJobLocalFile(t *testing.T) {
11811202
VolumeMounts: script.VolumeMount(),
11821203
Ports: []corev1.ContainerPort{{ContainerPort: 6565}},
11831204
LivenessProbe: &corev1.Probe{
1205+
InitialDelaySeconds: 10,
1206+
TimeoutSeconds: 3,
11841207
ProbeHandler: corev1.ProbeHandler{
11851208
HTTPGet: &corev1.HTTPGetAction{
11861209
Path: "/v1/status",
@@ -1190,6 +1213,7 @@ func TestNewRunnerJobLocalFile(t *testing.T) {
11901213
},
11911214
},
11921215
ReadinessProbe: &corev1.Probe{
1216+
TimeoutSeconds: 3,
11931217
ProbeHandler: corev1.ProbeHandler{
11941218
HTTPGet: &corev1.HTTPGetAction{
11951219
Path: "/v1/status",
@@ -1323,6 +1347,8 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) {
13231347
},
13241348
},
13251349
LivenessProbe: &corev1.Probe{
1350+
InitialDelaySeconds: 10,
1351+
TimeoutSeconds: 3,
13261352
ProbeHandler: corev1.ProbeHandler{
13271353
HTTPGet: &corev1.HTTPGetAction{
13281354
Path: "/v1/status",
@@ -1332,6 +1358,7 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) {
13321358
},
13331359
},
13341360
ReadinessProbe: &corev1.Probe{
1361+
TimeoutSeconds: 3,
13351362
ProbeHandler: corev1.ProbeHandler{
13361363
HTTPGet: &corev1.HTTPGetAction{
13371364
Path: "/v1/status",
@@ -1502,6 +1529,8 @@ func TestNewRunnerJobWithVolume(t *testing.T) {
15021529
},
15031530
},
15041531
LivenessProbe: &corev1.Probe{
1532+
InitialDelaySeconds: 10,
1533+
TimeoutSeconds: 3,
15051534
ProbeHandler: corev1.ProbeHandler{
15061535
HTTPGet: &corev1.HTTPGetAction{
15071536
Path: "/v1/status",
@@ -1511,6 +1540,7 @@ func TestNewRunnerJobWithVolume(t *testing.T) {
15111540
},
15121541
},
15131542
ReadinessProbe: &corev1.Probe{
1543+
TimeoutSeconds: 3,
15141544
ProbeHandler: corev1.ProbeHandler{
15151545
HTTPGet: &corev1.HTTPGetAction{
15161546
Path: "/v1/status",
@@ -1672,6 +1702,8 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) {
16721702
},
16731703
},
16741704
LivenessProbe: &corev1.Probe{
1705+
InitialDelaySeconds: 10,
1706+
TimeoutSeconds: 3,
16751707
ProbeHandler: corev1.ProbeHandler{
16761708
HTTPGet: &corev1.HTTPGetAction{
16771709
Path: "/v1/status",
@@ -1681,6 +1713,7 @@ func TestNewRunnerJobPLZTestRun(t *testing.T) {
16811713
},
16821714
},
16831715
ReadinessProbe: &corev1.Probe{
1716+
TimeoutSeconds: 3,
16841717
ProbeHandler: corev1.ProbeHandler{
16851718
HTTPGet: &corev1.HTTPGetAction{
16861719
Path: "/v1/status",

0 commit comments

Comments
 (0)