From 9400ace18e6722cf58890e6ea4f4334d8cab48e0 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 24 Jan 2026 23:28:39 -0800 Subject: [PATCH 01/39] annotation support for system task workers --- .../config/ConductorCoreConfiguration.java | 12 +- .../core/execution/DeciderService.java | 4 + .../execution/tasks/SystemTaskRegistry.java | 2 + .../sdk/workflow/task/InputParam.java | 26 ++ .../sdk/workflow/task/OutputParam.java | 24 ++ .../sdk/workflow/task/WorkerTask.java | 40 +++ .../mapper/AnnotatedSystemTaskMapper.java | 99 ++++++++ .../AnnotatedMethodParameterMapper.java | 166 +++++++++++++ .../AnnotatedMethodResultMapper.java | 90 +++++++ .../AnnotatedWorkflowSystemTask.java | 150 ++++++++++++ .../annotated/NonRetryableException.java | 32 +++ .../tasks/annotated/SampleWorkers.java | 26 ++ .../WorkerTaskAnnotationScanner.java | 156 ++++++++++++ .../sdk/workflow/task/InputParam.java | 26 ++ .../sdk/workflow/task/OutputParam.java | 24 ++ .../sdk/workflow/task/WorkerTask.java | 40 +++ .../core/execution/TestDeciderOutcomes.java | 1 + .../core/execution/TestDeciderService.java | 7 +- .../core/execution/TestWorkflowExecutor.java | 19 +- .../TestAnnotatedMethodParameterMapper.java | 145 +++++++++++ .../TestAnnotatedMethodResultMapper.java | 176 ++++++++++++++ .../TestAnnotatedSystemTaskIntegration.java | 150 ++++++++++++ .../TestAnnotatedWorkflowSystemTask.java | 228 ++++++++++++++++++ .../conductor/tasks/http/HttpTaskTest.java | 1 + 24 files changed, 1631 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/com/netflix/conductor/sdk/workflow/task/InputParam.java create mode 100644 core/src/main/java/com/netflix/conductor/sdk/workflow/task/OutputParam.java create mode 100644 core/src/main/java/com/netflix/conductor/sdk/workflow/task/WorkerTask.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/mapper/AnnotatedSystemTaskMapper.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/NonRetryableException.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java create mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java create mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java create mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java create mode 100644 core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java create mode 100644 core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java create mode 100644 core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java create mode 100644 core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java diff --git a/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java b/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java index c0e8de9ee2..2b1b302342 100644 --- a/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java +++ b/core/src/main/java/com/netflix/conductor/core/config/ConductorCoreConfiguration.java @@ -105,15 +105,23 @@ public ExecutorService executorService(ConductorProperties conductorProperties) @Bean @Qualifier("taskMappersByTaskType") public Map getTaskMappers(List taskMappers) { - return taskMappers.stream().collect(Collectors.toMap(TaskMapper::getTaskType, identity())); + // Return mutable map so annotated task mappers can be added + return taskMappers.stream() + .collect( + Collectors.toMap( + TaskMapper::getTaskType, + identity(), + (a, b) -> a, + java.util.HashMap::new)); } @Bean @Qualifier(ASYNC_SYSTEM_TASKS_QUALIFIER) public Set asyncSystemTasks(Set allSystemTasks) { + // Return mutable set so annotated tasks can be added return allSystemTasks.stream() .filter(WorkflowSystemTask::isAsync) - .collect(Collectors.toUnmodifiableSet()); + .collect(Collectors.toCollection(java.util.HashSet::new)); } @Bean diff --git a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java index e1bacb34d4..598f6f4b1f 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java @@ -75,6 +75,7 @@ public DeciderService( ExternalPayloadStorageUtils externalPayloadStorageUtils, SystemTaskRegistry systemTaskRegistry, @Qualifier("taskMappersByTaskType") Map taskMappers, + @Qualifier("annotatedTaskSystems") Map annotatedTaskSystems, @Value("${conductor.app.taskPendingTimeThreshold:60m}") Duration taskPendingTimeThreshold) { this.idGenerator = idGenerator; @@ -84,6 +85,9 @@ public DeciderService( this.externalPayloadStorageUtils = externalPayloadStorageUtils; this.taskPendingTimeThresholdMins = taskPendingTimeThreshold.toMinutes(); this.systemTaskRegistry = systemTaskRegistry; + LOGGER.info("taskMappers: {}", taskMappers.keySet()); + LOGGER.info("annotatedTaskMappers: {}", annotatedTaskSystems.keySet()); + this.taskMappers.putAll(annotatedTaskSystems); } public DeciderOutcome decide(WorkflowModel workflow) throws TerminateWorkflowException { diff --git a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskRegistry.java b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskRegistry.java index 7b9cf743ba..947eef9e4f 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskRegistry.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/tasks/SystemTaskRegistry.java @@ -18,6 +18,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import org.springframework.context.annotation.DependsOn; import org.springframework.stereotype.Component; /** @@ -25,6 +26,7 @@ * com.netflix.conductor.common.metadata.tasks.TaskType} to {@link WorkflowSystemTask} instances. */ @Component +@DependsOn("workerTaskAnnotationScanner") public class SystemTaskRegistry { public static final String ASYNC_SYSTEM_TASKS_QUALIFIER = "asyncSystemTasks"; diff --git a/core/src/main/java/com/netflix/conductor/sdk/workflow/task/InputParam.java b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/InputParam.java new file mode 100644 index 0000000000..2afa8bc42b --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/InputParam.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.PARAMETER) +public @interface InputParam { + String value(); + + boolean required() default false; +} diff --git a/core/src/main/java/com/netflix/conductor/sdk/workflow/task/OutputParam.java b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/OutputParam.java new file mode 100644 index 0000000000..3c1251317e --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/OutputParam.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE_USE) +public @interface OutputParam { + String value(); +} diff --git a/core/src/main/java/com/netflix/conductor/sdk/workflow/task/WorkerTask.java b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/WorkerTask.java new file mode 100644 index 0000000000..15df5ebbec --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/sdk/workflow/task/WorkerTask.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Identifies a simple worker task. */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface WorkerTask { + String value(); + + // No. of threads to use for executing the task + int threadCount() default 1; + + int pollingInterval() default 100; + + String domain() default ""; + + // In millis + int pollTimeout() default 100; + + // number of task pollers + // default is 1 which is good enough for most use cases + // a number higher than 1 will have concurrent pollers doing poll and execute + int pollerCount() default 1; +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/mapper/AnnotatedSystemTaskMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/mapper/AnnotatedSystemTaskMapper.java new file mode 100644 index 0000000000..a36f96a4a6 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/mapper/AnnotatedSystemTaskMapper.java @@ -0,0 +1,99 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.mapper; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; +import com.netflix.conductor.core.execution.mapper.TaskMapper; +import com.netflix.conductor.core.execution.mapper.TaskMapperContext; +import com.netflix.conductor.core.utils.ParametersUtils; +import com.netflix.conductor.dao.MetadataDAO; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +/** + * TaskMapper for @WorkerTask annotated system tasks. Unlike user-defined tasks, annotated system + * tasks do not require a task definition and can be scheduled directly from the workflow + * definition. If a task definition exists, it will be used for rate limiting and other settings. + */ +public class AnnotatedSystemTaskMapper implements TaskMapper { + + private static final Logger LOGGER = LoggerFactory.getLogger(AnnotatedSystemTaskMapper.class); + + private final String taskType; + private final ParametersUtils parametersUtils; + private final MetadataDAO metadataDAO; + + public AnnotatedSystemTaskMapper( + String taskType, ParametersUtils parametersUtils, MetadataDAO metadataDAO) { + this.taskType = taskType; + this.parametersUtils = parametersUtils; + this.metadataDAO = metadataDAO; + } + + @Override + public String getTaskType() { + return taskType; + } + + @Override + public List getMappedTasks(TaskMapperContext taskMapperContext) { + LOGGER.debug("TaskMapperContext {} in AnnotatedSystemTaskMapper", taskMapperContext); + + WorkflowTask workflowTask = taskMapperContext.getWorkflowTask(); + WorkflowModel workflowModel = taskMapperContext.getWorkflowModel(); + int retryCount = taskMapperContext.getRetryCount(); + String retriedTaskId = taskMapperContext.getRetryTaskId(); + + // Try to get task definition - don't fail if not found (unlike + // SimpleTaskMapper) + TaskDef taskDefinition = + Optional.ofNullable(workflowTask.getTaskDefinition()) + .orElseGet(() -> metadataDAO.getTaskDef(workflowTask.getName())); + + // Get input - pass taskDefinition (may be null) + Map input = + parametersUtils.getTaskInput( + workflowTask.getInputParameters(), + workflowModel, + taskDefinition, + taskMapperContext.getTaskId()); + + // Create task model + TaskModel task = taskMapperContext.createTaskModel(); + task.setTaskType(taskType); + task.setStartDelayInSeconds(workflowTask.getStartDelay()); + task.setInputData(input); + task.setStatus(TaskModel.Status.SCHEDULED); + task.setRetryCount(retryCount); + task.setCallbackAfterSeconds(workflowTask.getStartDelay()); + task.setRetriedTaskId(retriedTaskId); + + // If task definition exists, use its settings for rate limiting etc. + if (Objects.nonNull(taskDefinition)) { + task.setResponseTimeoutSeconds(taskDefinition.getResponseTimeoutSeconds()); + task.setRateLimitPerFrequency(taskDefinition.getRateLimitPerFrequency()); + task.setRateLimitFrequencyInSeconds(taskDefinition.getRateLimitFrequencyInSeconds()); + } + + return List.of(task); + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java new file mode 100644 index 0000000000..24d4491dde --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java @@ -0,0 +1,166 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.conductoross.conductor.sdk.workflow.task.InputParam; + +import com.netflix.conductor.common.config.ObjectMapperProvider; +import com.netflix.conductor.model.TaskModel; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +// Annotations copied from java-sdk to avoid external dependency + +/** + * Utility class for mapping TaskModel parameters to method invocation parameters for @WorkerTask + * annotated methods. + */ +public class AnnotatedMethodParameterMapper { + + private final ObjectMapper objectMapper; + + public AnnotatedMethodParameterMapper() { + this.objectMapper = new ObjectMapperProvider().getObjectMapper(); + this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + /** + * Maps a TaskModel to the parameters required by the annotated method. + * + * @param task The task model to map from + * @param method The method to map parameters for + * @return Array of parameter values ready for method invocation + */ + public Object[] mapParameters(TaskModel task, Method method) { + Class[] parameterTypes = method.getParameterTypes(); + Parameter[] parameters = method.getParameters(); + + // Special case: single TaskModel parameter + if (parameterTypes.length == 1 && parameterTypes[0].equals(TaskModel.class)) { + return new Object[] {task}; + } + + // Special case: single Map parameter (task input data) + if (parameterTypes.length == 1 && parameterTypes[0].equals(Map.class)) { + return new Object[] {task.getInputData()}; + } + + // General case: may have @InputParam, @WorkflowInstanceIdInputParam, or plain + // types + return mapAnnotatedParameters(task, parameterTypes, parameters, method); + } + + private Object[] mapAnnotatedParameters( + TaskModel task, Class[] parameterTypes, Parameter[] parameters, Method method) { + Annotation[][] parameterAnnotations = method.getParameterAnnotations(); + Object[] values = new Object[parameterTypes.length]; + + for (int i = 0; i < parameterTypes.length; i++) { + Annotation[] paramAnnotation = parameterAnnotations[i]; + + // WorkflowInstanceIdInputParam not available in SDK v3.x - uncomment when + // upgrading + // if (containsWorkflowInstanceIdInputParamAnnotation(paramAnnotation)) { + // validateParameterForWorkflowInstanceId(parameters[i]); + // values[i] = task.getWorkflowInstanceId(); + // } else + if (paramAnnotation.length > 0) { + Type type = parameters[i].getParameterizedType(); + Class parameterType = parameterTypes[i]; + values[i] = getInputValue(task, parameterType, type, paramAnnotation); + } else { + // No annotation - convert entire input data to parameter type + values[i] = objectMapper.convertValue(task.getInputData(), parameterTypes[i]); + } + } + + return values; + } + + // WorkflowInstanceIdInputParam not available in SDK v3.x - uncomment when + // upgrading + // private boolean containsWorkflowInstanceIdInputParamAnnotation(Annotation[] + // annotations) { + // return Arrays.stream(annotations) + // .map(Annotation::annotationType) + // .anyMatch(WorkflowInstanceIdInputParam.class::equals); + // } + // + // private void validateParameterForWorkflow InstanceId(Parameter parameter) { + // if (!parameter.getType().equals(String.class)) { + // throw new IllegalArgumentException( + // "Parameter " + // + parameter + // + " is annotated with " + // + WorkflowInstanceIdInputParam.class.getSimpleName() + // + " but is not of type " + // + String.class.getSimpleName() + // + "."); + // } + // } + + private Object getInputValue( + TaskModel task, Class parameterType, Type type, Annotation[] paramAnnotation) { + InputParam ip = findInputParamAnnotation(paramAnnotation); + + if (ip == null) { + return objectMapper.convertValue(task.getInputData(), parameterType); + } + + final String name = ip.value(); + final Object value = task.getInputData().get(name); + if (value == null) { + return null; + } + + if (List.class.isAssignableFrom(parameterType)) { + return convertToParameterizedList(value, type); + } else { + return objectMapper.convertValue(value, parameterType); + } + } + + private Object convertToParameterizedList(Object value, Type type) { + List list = objectMapper.convertValue(value, List.class); + if (type instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType) type; + Class typeOfParameter = (Class) parameterizedType.getActualTypeArguments()[0]; + List parameterizedList = new ArrayList<>(); + for (Object item : list) { + parameterizedList.add(objectMapper.convertValue(item, typeOfParameter)); + } + return parameterizedList; + } else { + return list; + } + } + + private static InputParam findInputParamAnnotation(Annotation[] paramAnnotation) { + return (InputParam) + Arrays.stream(paramAnnotation) + .filter(ann -> ann.annotationType().equals(InputParam.class)) + .findFirst() + .orElse(null); + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java new file mode 100644 index 0000000000..d6bc90ef3f --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +import org.conductoross.conductor.sdk.workflow.task.OutputParam; + +import com.netflix.conductor.common.config.ObjectMapperProvider; +import com.netflix.conductor.model.TaskModel; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Utility class for mapping method return values to TaskModel status and output data + * for @WorkerTask annotated methods. + */ +public class AnnotatedMethodResultMapper { + + private final ObjectMapper objectMapper; + + public AnnotatedMethodResultMapper() { + this.objectMapper = new ObjectMapperProvider().getObjectMapper(); + } + + /** + * Applies the method invocation result to the task model. + * + * @param invocationResult The result returned from the method invocation + * @param task The task model to update + * @param method The method that was invoked + */ + public void applyResult(Object invocationResult, TaskModel task, Method method) { + if (invocationResult == null) { + task.setStatus(TaskModel.Status.COMPLETED); + return; + } + + OutputParam opAnnotation = method.getAnnotatedReturnType().getAnnotation(OutputParam.class); + + if (opAnnotation != null) { + // Return value should be placed in a named output parameter + String name = opAnnotation.value(); + task.getOutputData().put(name, invocationResult); + task.setStatus(TaskModel.Status.COMPLETED); + + } else if (invocationResult instanceof Map) { + // Return Map becomes output data + @SuppressWarnings("unchecked") + Map resultAsMap = (Map) invocationResult; + task.getOutputData().putAll(resultAsMap); + task.setStatus(TaskModel.Status.COMPLETED); + + } else if (isPrimitive(invocationResult)) { + // Primitives (String, Number, Boolean) go into "result" key + task.getOutputData().put("result", invocationResult); + task.setStatus(TaskModel.Status.COMPLETED); + + } else if (invocationResult instanceof List) { + // Lists are converted and placed in "result" key + List resultAsList = objectMapper.convertValue(invocationResult, List.class); + task.getOutputData().put("result", resultAsList); + task.setStatus(TaskModel.Status.COMPLETED); + + } else { + // POJOs are converted to Map and merged into output data + @SuppressWarnings("unchecked") + Map resultAsMap = + objectMapper.convertValue(invocationResult, Map.class); + task.getOutputData().putAll(resultAsMap); + task.setStatus(TaskModel.Status.COMPLETED); + } + } + + private boolean isPrimitive(Object value) { + return value instanceof String || value instanceof Number || value instanceof Boolean; + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java new file mode 100644 index 0000000000..a4338148cc --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java @@ -0,0 +1,150 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import org.conductoross.conductor.sdk.workflow.task.WorkerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +/** + * Adapter that wraps a @WorkerTask annotated method as a WorkflowSystemTask. This enables + * annotation-based system task development while maintaining compatibility with the existing + * SystemTaskWorkerCoordinator infrastructure. + */ +public class AnnotatedWorkflowSystemTask extends WorkflowSystemTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(AnnotatedWorkflowSystemTask.class); + + private final Method method; + private final Object bean; + private final WorkerTask annotation; + private final AnnotatedMethodParameterMapper parameterMapper; + private final AnnotatedMethodResultMapper resultMapper; + + /** + * Creates a new AnnotatedWorkflowSystemTask. + * + * @param taskType The task type name + * @param method The annotated method to invoke + * @param bean The Spring bean instance containing the method + * @param annotation The @WorkerTask annotation metadata + */ + public AnnotatedWorkflowSystemTask( + String taskType, Method method, Object bean, WorkerTask annotation) { + super(taskType); + this.method = method; + this.bean = bean; + this.annotation = annotation; + this.parameterMapper = new AnnotatedMethodParameterMapper(); + this.resultMapper = new AnnotatedMethodResultMapper(); + } + + @Override + public boolean isAsync() { + // Always use async polling for annotated tasks + return true; + } + + @Override + public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { + execute(workflow, task, workflowExecutor); + } + + @Override + public boolean execute( + WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { + try { + LOGGER.debug( + "Executing annotated task {} for workflow {}", + getTaskType(), + workflow.getWorkflowId()); + + // Map task parameters to method parameters + Object[] parameters = parameterMapper.mapParameters(task, method); + + // Invoke the annotated method + Object result = method.invoke(bean, parameters); + + // Apply the result to the task + resultMapper.applyResult(result, task, method); + + LOGGER.debug( + "Completed annotated task {} with status {}", getTaskType(), task.getStatus()); + + return true; + + } catch (InvocationTargetException e) { + handleInvocationException(task, e); + return true; + } catch (Exception e) { + LOGGER.error("Unexpected error executing annotated task " + getTaskType(), e); + task.setStatus(TaskModel.Status.FAILED); + task.setReasonForIncompletion("Unexpected error: " + e.getMessage()); + return true; + } + } + + private void handleInvocationException(TaskModel task, InvocationTargetException e) { + Throwable cause = e.getCause(); + + LOGGER.error("Error executing annotated task " + getTaskType(), cause); + + if (cause instanceof NonRetryableException) { + task.setStatus(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR); + task.setReasonForIncompletion("Non-retryable error: " + cause.getMessage()); + } else { + task.setStatus(TaskModel.Status.FAILED); + task.setReasonForIncompletion("Task execution failed: " + cause.getMessage()); + } + } + + @Override + public void cancel(WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { + // Default implementation - annotated tasks typically don't need custom cancel + // logic + LOGGER.debug( + "Cancelling annotated task {} for workflow {}", + getTaskType(), + workflow.getWorkflowId()); + task.setStatus(TaskModel.Status.CANCELED); + } + + /** + * @return The annotation metadata for this task + */ + public WorkerTask getAnnotation() { + return annotation; + } + + /** + * @return The method that implements this task + */ + public Method getMethod() { + return method; + } + + /** + * @return The Spring bean instance + */ + public Object getBean() { + return bean; + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/NonRetryableException.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/NonRetryableException.java new file mode 100644 index 0000000000..f2037d7f2d --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/NonRetryableException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +/** + * Exception thrown when a worker method execution should not be retried. This maps to + * FAILED_WITH_TERMINAL_ERROR status. + */ +public class NonRetryableException extends RuntimeException { + + public NonRetryableException(String message) { + super(message); + } + + public NonRetryableException(String message, Throwable cause) { + super(message, cause); + } + + public NonRetryableException(Throwable cause) { + super(cause); + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java new file mode 100644 index 0000000000..9aad45de56 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import org.conductoross.conductor.sdk.workflow.task.InputParam; +import org.conductoross.conductor.sdk.workflow.task.WorkerTask; +import org.springframework.stereotype.Component; + +@Component +public class SampleWorkers { + + @WorkerTask("HELLO") + public String hello(@InputParam("name") String name) { + return "Hello %s, from the sample worker".formatted(name); + } +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java new file mode 100644 index 0000000000..7815afcd2e --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java @@ -0,0 +1,156 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.conductoross.conductor.core.execution.mapper.AnnotatedSystemTaskMapper; +import org.conductoross.conductor.sdk.workflow.task.WorkerTask; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import com.netflix.conductor.core.execution.mapper.TaskMapper; +import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; +import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; +import com.netflix.conductor.core.utils.ParametersUtils; +import com.netflix.conductor.dao.MetadataDAO; + +/** + * Spring component that scans for @WorkerTask annotated methods in Spring beans and adds them to + * the existing asyncSystemTasks collection and taskMappersByTaskType map. + */ +@Component +@ConditionalOnProperty( + name = "conductor.annotated-system-tasks.enabled", + havingValue = "true", + matchIfMissing = true) +public class WorkerTaskAnnotationScanner implements InitializingBean { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorkerTaskAnnotationScanner.class); + + private final ApplicationContext applicationContext; + private final Set asyncSystemTasks; + private final Map taskMappers = new HashMap<>(); + private final ParametersUtils parametersUtils; + private final MetadataDAO metadataDAO; + + public WorkerTaskAnnotationScanner( + ApplicationContext applicationContext, + @Qualifier(SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER) + Set asyncSystemTasks, + @Lazy ParametersUtils parametersUtils, + @Lazy MetadataDAO metadataDAO) { + this.applicationContext = applicationContext; + this.asyncSystemTasks = asyncSystemTasks; + this.parametersUtils = parametersUtils; + this.metadataDAO = metadataDAO; + } + + /** + * Scans all Spring beans for @WorkerTask annotated methods and adds them to the + * asyncSystemTasks collection. Also registers a TaskMapper for each annotated task type. + */ + @Override + public void afterPropertiesSet() { + long startTime = System.currentTimeMillis(); + + LOGGER.info("Scanning Spring beans for @WorkerTask annotated methods..."); + + String[] beanNames = applicationContext.getBeanDefinitionNames(); + int scannedBeans = 0; + int foundMethods = 0; + + for (String beanName : beanNames) { + try { + Object bean = applicationContext.getBean(beanName); + Class beanClass = bean.getClass(); + + // Skip framework classes and proxies + if (isFrameworkClass(beanClass)) { + continue; + } + + scannedBeans++; + + // Scan all public methods for @WorkerTask annotation + for (Method method : beanClass.getMethods()) { + WorkerTask annotation = method.getAnnotation(WorkerTask.class); + if (annotation != null) { + String taskType = annotation.value(); + + LOGGER.info( + "Found @WorkerTask method: {} in bean {} with taskType={}", + method.getName(), + beanName, + taskType); + + AnnotatedWorkflowSystemTask task = + new AnnotatedWorkflowSystemTask(taskType, method, bean, annotation); + + // Add to existing asyncSystemTasks collection + asyncSystemTasks.add(task); + + // Register a TaskMapper for this task type so DeciderService can find it + AnnotatedSystemTaskMapper mapper = + new AnnotatedSystemTaskMapper( + taskType, parametersUtils, metadataDAO); + LOGGER.info("Adding task mapper {} for task {}", mapper, taskType); + taskMappers.put(taskType, mapper); + + LOGGER.debug("Registered TaskMapper for annotated task type: {}", taskType); + foundMethods++; + } + } + } catch (Exception e) { + // Skip beans that can't be instantiated or scanned + LOGGER.debug( + "Skipping bean {} during @WorkerTask scanning: {}", + beanName, + e.getMessage()); + } + } + + long durationMs = System.currentTimeMillis() - startTime; + LOGGER.info( + "Completed @WorkerTask scanning in {}ms. Scanned {} beans, found {} annotated methods", + durationMs, + scannedBeans, + foundMethods); + } + + @Qualifier("annotatedTaskSystems") + @Bean + public Map annotatedTaskSystems() { + return taskMappers; + } + + private boolean isFrameworkClass(Class clazz) { + String className = clazz.getName(); + // Skip Spring framework classes, CGLIB proxies, and JDK classes + return className.startsWith("org.springframework.") + || className.contains("$$EnhancerBy") + || className.contains("$$FastClassBy") + || className.startsWith("java.") + || className.startsWith("jdk."); + } +} diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java new file mode 100644 index 0000000000..15fe99535f --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.PARAMETER) +public @interface InputParam { + String value(); + + boolean required() default false; +} diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java new file mode 100644 index 0000000000..2df17faab1 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE_USE) +public @interface OutputParam { + String value(); +} diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java new file mode 100644 index 0000000000..4de239bcf2 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.sdk.workflow.task; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Identifies a simple worker task. */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD}) +public @interface WorkerTask { + String value(); + + // No. of threads to use for executing the task + int threadCount() default 1; + + int pollingInterval() default 100; + + String domain() default ""; + + // In millis + int pollTimeout() default 100; + + // number of task pollers + // default is 1 which is good enough for most use cases + // a number higher than 1 will have concurrent pollers doing poll and execute + int pollerCount() default 1; +} diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderOutcomes.java b/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderOutcomes.java index c929bcd86d..18c9f508b5 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderOutcomes.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderOutcomes.java @@ -186,6 +186,7 @@ public void init() { externalPayloadStorageUtils, systemTaskRegistry, taskMappers, + new HashMap<>(), Duration.ofMinutes(60)); } diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderService.java b/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderService.java index fd6349e2e6..0e5d00d249 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderService.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestDeciderService.java @@ -163,6 +163,7 @@ public void setup() { externalPayloadStorageUtils, systemTaskRegistry, taskMappers, + new HashMap<>(), Duration.ofMinutes(60)); } @@ -747,7 +748,8 @@ public void testWorkflowTaskRetry() { task2.get().getTaskId(), ((Map) task2.get().getInputData().get("env")).get("env_task_id")); - // Set the retried task to FAILED, retry it again and assert that the workflow failed + // Set the retried task to FAILED, retry it again and assert that the workflow + // failed task2.get().setStatus(TaskModel.Status.FAILED); exception.expect(TerminateWorkflowException.class); final Optional task3 = @@ -1257,7 +1259,8 @@ public void testWorkflowCompleted_WhenAllOptionalTasksInTerminalState() { workflow.getTasks().addAll(List.of(task1, task2)); - // Workflow should be COMPLETED. All optional tasks have reached a terminal state. + // Workflow should be COMPLETED. All optional tasks have reached a terminal + // state. assertTrue(deciderService.checkForWorkflowCompletion(workflow)); } diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java b/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java index 1ed26ba4ad..475c0519a0 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestWorkflowExecutor.java @@ -196,6 +196,7 @@ public void init() { externalPayloadStorageUtils, systemTaskRegistry, taskMappers, + new HashMap<>(), Duration.ofMinutes(60)); MetadataMapperService metadataMapperService = new MetadataMapperService(metadataDAO); @@ -709,7 +710,7 @@ public void testRetryWorkflowNoFailedTasks() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); @@ -755,7 +756,7 @@ public void testRetryWorkflow() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); @@ -868,7 +869,7 @@ public void testRetryWorkflowReturnsNoDuplicates() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); @@ -947,7 +948,7 @@ public void testRetryWorkflowMultipleRetries() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); @@ -1037,7 +1038,7 @@ public void testRetryWorkflowWithJoinTask() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); @@ -1194,7 +1195,7 @@ public void testRetryTimedOutWorkflowWithoutFailedTasks() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.TIMED_OUT); @@ -1279,7 +1280,7 @@ public void testRerunWorkflow() { workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setReasonForIncompletion("task1 failed"); @@ -1435,7 +1436,7 @@ public void testRerunWorkflowWithTaskId() { workflow.setOwnerApp("junit_testRerunWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); workflow.setReasonForIncompletion("task1 failed"); @@ -2539,7 +2540,7 @@ private WorkflowModel generateSampleWorkflow() { workflow.setOwnerApp("junit_testRetryWorkflowId"); workflow.setCreateTime(10L); workflow.setEndTime(100L); - //noinspection unchecked + // noinspection unchecked workflow.setOutput(Collections.EMPTY_MAP); workflow.setStatus(WorkflowModel.Status.FAILED); diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java new file mode 100644 index 0000000000..07cc306aec --- /dev/null +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java @@ -0,0 +1,145 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.conductoross.conductor.sdk.workflow.task.InputParam; +import org.junit.Test; + +import com.netflix.conductor.model.TaskModel; + +import static org.junit.Assert.*; + +public class TestAnnotatedMethodParameterMapper { + + private final AnnotatedMethodParameterMapper mapper = new AnnotatedMethodParameterMapper(); + + // Test class with various method signatures + static class TestWorker { + public void taskModelParameter(TaskModel task) {} + + public void mapParameter(Map input) {} + + public void singleInputParam(@InputParam("name") String name) {} + + public void multipleInputParams( + @InputParam("firstName") String firstName, @InputParam("age") Integer age) {} + + public void listInputParam(@InputParam("items") List items) {} + + public void pojoParameter(TestPojo pojo) {} + } + + public static class TestPojo { + public String field1; + public int field2; + } + + @Test + public void testTaskModelParameter() throws Exception { + Method method = TestWorker.class.getMethod("taskModelParameter", TaskModel.class); + TaskModel task = createTaskModel("workflow-id", Map.of("key", "value")); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertSame(task, params[0]); + } + + @Test + public void testMapParameter() throws Exception { + Method method = TestWorker.class.getMethod("mapParameter", Map.class); + TaskModel task = createTaskModel("workflow-id", Map.of("key", "value")); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertSame(task.getInputData(), params[0]); + } + + @Test + public void testSingleInputParam() throws Exception { + Method method = TestWorker.class.getMethod("singleInputParam", String.class); + TaskModel task = createTaskModel("workflow-id", Map.of("name", "John")); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertEquals("John", params[0]); + } + + @Test + public void testMultipleInputParams() throws Exception { + Method method = + TestWorker.class.getMethod("multipleInputParams", String.class, Integer.class); + TaskModel task = createTaskModel("workflow-id", Map.of("firstName", "Jane", "age", 25)); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(2, params.length); + assertEquals("Jane", params[0]); + assertEquals(25, params[1]); + } + + @Test + public void testListInputParam() throws Exception { + Method method = TestWorker.class.getMethod("listInputParam", List.class); + TaskModel task = + createTaskModel("workflow-id", Map.of("items", List.of("item1", "item2", "item3"))); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertTrue(params[0] instanceof List); + @SuppressWarnings("unchecked") + List items = (List) params[0]; + assertEquals(3, items.size()); + assertEquals("item1", items.get(0)); + } + + @Test + public void testPojoParameter() throws Exception { + Method method = TestWorker.class.getMethod("pojoParameter", TestPojo.class); + TaskModel task = createTaskModel("workflow-id", Map.of("field1", "value1", "field2", 42)); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertTrue(params[0] instanceof TestPojo); + TestPojo pojo = (TestPojo) params[0]; + assertEquals("value1", pojo.field1); + assertEquals(42, pojo.field2); + } + + @Test + public void testInputParamWithNullValue() throws Exception { + Method method = TestWorker.class.getMethod("singleInputParam", String.class); + TaskModel task = createTaskModel("workflow-id", Map.of("other", "value")); + + Object[] params = mapper.mapParameters(task, method); + + assertEquals(1, params.length); + assertNull(params[0]); + } + + private TaskModel createTaskModel(String workflowId, Map inputData) { + TaskModel task = new TaskModel(); + task.setWorkflowInstanceId(workflowId); + task.setInputData(new HashMap<>(inputData)); + return task; + } +} diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java new file mode 100644 index 0000000000..c57f62c701 --- /dev/null +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java @@ -0,0 +1,176 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.conductoross.conductor.sdk.workflow.task.OutputParam; +import org.junit.Test; + +import com.netflix.conductor.model.TaskModel; + +import static org.junit.Assert.*; + +public class TestAnnotatedMethodResultMapper { + + private final AnnotatedMethodResultMapper mapper = new AnnotatedMethodResultMapper(); + + static class TestWorker { + public void voidReturn() {} + + public Map mapReturn() { + return Map.of("result", "success"); + } + + public String stringReturn() { + return "hello"; + } + + public Integer numberReturn() { + return 42; + } + + public Boolean booleanReturn() { + return true; + } + + public List listReturn() { + return List.of("a", "b", "c"); + } + + @OutputParam("customKey") + public String annotatedReturn() { + return "custom value"; + } + + public TestPojo pojoReturn() { + TestPojo pojo = new TestPojo(); + pojo.field1 = "value1"; + pojo.field2 = 123; + return pojo; + } + } + + static class TestPojo { + public String field1; + public int field2; + } + + @Test + public void testVoidReturn() throws Exception { + Method method = TestWorker.class.getMethod("voidReturn"); + TaskModel task = createTaskModel(); + + mapper.applyResult(null, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertTrue(task.getOutputData().isEmpty()); + } + + @Test + public void testMapReturn() throws Exception { + Method method = TestWorker.class.getMethod("mapReturn"); + TaskModel task = createTaskModel(); + Map returnValue = Map.of("result", "success", "count", 5); + + mapper.applyResult(returnValue, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("success", task.getOutputData().get("result")); + assertEquals(5, task.getOutputData().get("count")); + } + + @Test + public void testStringReturn() throws Exception { + Method method = TestWorker.class.getMethod("stringReturn"); + TaskModel task = createTaskModel(); + + mapper.applyResult("hello", task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("hello", task.getOutputData().get("result")); + } + + @Test + public void testNumberReturn() throws Exception { + Method method = TestWorker.class.getMethod("numberReturn"); + TaskModel task = createTaskModel(); + + mapper.applyResult(42, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals(42, task.getOutputData().get("result")); + } + + @Test + public void testBooleanReturn() throws Exception { + Method method = TestWorker.class.getMethod("booleanReturn"); + TaskModel task = createTaskModel(); + + mapper.applyResult(true, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals(true, task.getOutputData().get("result")); + } + + @Test + public void testListReturn() throws Exception { + Method method = TestWorker.class.getMethod("listReturn"); + TaskModel task = createTaskModel(); + List returnValue = List.of("a", "b", "c"); + + mapper.applyResult(returnValue, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + @SuppressWarnings("unchecked") + List result = (List) task.getOutputData().get("result"); + assertEquals(3, result.size()); + assertEquals("a", result.get(0)); + } + + @Test + public void testAnnotatedReturn() throws Exception { + Method method = TestWorker.class.getMethod("annotatedReturn"); + TaskModel task = createTaskModel(); + + mapper.applyResult("custom value", task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("custom value", task.getOutputData().get("customKey")); + assertFalse(task.getOutputData().containsKey("result")); + } + + @Test + public void testPojoReturn() throws Exception { + Method method = TestWorker.class.getMethod("pojoReturn"); + TaskModel task = createTaskModel(); + TestPojo pojo = new TestPojo(); + pojo.field1 = "test"; + pojo.field2 = 99; + + mapper.applyResult(pojo, task, method); + + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("test", task.getOutputData().get("field1")); + assertEquals(99, task.getOutputData().get("field2")); + } + + private TaskModel createTaskModel() { + TaskModel task = new TaskModel(); + task.setOutputData(new HashMap<>()); + return task; + } +} diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java new file mode 100644 index 0000000000..eb3c1384b4 --- /dev/null +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java @@ -0,0 +1,150 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.util.Map; +import java.util.Set; + +import org.conductoross.conductor.sdk.workflow.task.InputParam; +import org.conductoross.conductor.sdk.workflow.task.WorkerTask; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.junit4.SpringRunner; + +import com.netflix.conductor.core.execution.mapper.TaskMapper; +import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; +import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; +import com.netflix.conductor.core.utils.ParametersUtils; + +import static org.junit.Assert.*; + +/** + * Integration test to verify that WorkerTaskAnnotationScanner correctly discovers and + * registers @WorkerTask annotated methods with the Spring application context. + */ +@RunWith(SpringRunner.class) +@Import({TestAnnotatedSystemTaskIntegration.TestConfig.class, WorkerTaskAnnotationScanner.class}) +public class TestAnnotatedSystemTaskIntegration { + + @Autowired + @Qualifier(SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER) + private Set asyncSystemTasks; + + @TestConfiguration + static class TestConfig { + @Bean + public SampleAnnotatedTasks sampleAnnotatedTasks() { + return new SampleAnnotatedTasks(); + } + + @Bean + @Qualifier(SystemTaskRegistry.ASYNC_SYSTEM_TASKS_QUALIFIER) + public Set asyncSystemTasks() { + // Start with empty set - scanner will add to it + return new java.util.HashSet<>(); + } + + @Bean + @Qualifier("taskMappersByTaskType") + public Map taskMappersByTaskType() { + // Start with empty map - scanner will populate it + return new java.util.HashMap<>(); + } + + @Bean + public ParametersUtils parametersUtils() { + // Mock ParametersUtils for test + return new ParametersUtils(null); + } + } + + /** Sample bean with @WorkerTask annotated methods for testing */ + static class SampleAnnotatedTasks { + + @WorkerTask("integration_test_task_1") + public Map task1(@InputParam("input") String input) { + return Map.of("result", "task1: " + input); + } + + @WorkerTask(value = "integration_test_task_2", threadCount = 5, pollingInterval = 200) + public Map task2(@InputParam("value") Integer value) { + return Map.of("doubled", value * 2); + } + + // Method without @WorkerTask should be ignored + public String notATask() { + return "not registered"; + } + } + + @Test + public void testAnnotatedTasksAreDiscovered() { + assertNotNull(asyncSystemTasks); + assertFalse("Should have discovered annotated tasks", asyncSystemTasks.isEmpty()); + + // Find our test tasks + AnnotatedWorkflowSystemTask task1 = findTask("integration_test_task_1"); + AnnotatedWorkflowSystemTask task2 = findTask("integration_test_task_2"); + + assertNotNull("integration_test_task_1 should be discovered", task1); + assertNotNull("integration_test_task_2 should be discovered", task2); + } + + @Test + public void testAnnotatedTasksAreAsync() { + AnnotatedWorkflowSystemTask task1 = findTask("integration_test_task_1"); + assertNotNull(task1); + assertTrue("Annotated tasks should be async", task1.isAsync()); + } + + @Test + public void testAnnotatedTaskHasCorrectMetadata() { + AnnotatedWorkflowSystemTask task2 = findTask("integration_test_task_2"); + assertNotNull(task2); + + assertEquals("integration_test_task_2", task2.getTaskType()); + assertEquals("task2", task2.getMethod().getName()); + assertTrue(task2.getBean() instanceof SampleAnnotatedTasks); + + // Verify annotation metadata + assertEquals(5, task2.getAnnotation().threadCount()); + assertEquals(200, task2.getAnnotation().pollingInterval()); + } + + @Test + public void testNonAnnotatedMethodsNotRegistered() { + // Ensure that methods without @WorkerTask are not registered + long notATaskCount = + asyncSystemTasks.stream() + .filter(task -> task instanceof AnnotatedWorkflowSystemTask) + .map(task -> (AnnotatedWorkflowSystemTask) task) + .filter(task -> task.getMethod().getName().equals("notATask")) + .count(); + + assertEquals("Non-annotated methods should not be registered", 0, notATaskCount); + } + + private AnnotatedWorkflowSystemTask findTask(String taskType) { + return asyncSystemTasks.stream() + .filter(task -> task instanceof AnnotatedWorkflowSystemTask) + .map(task -> (AnnotatedWorkflowSystemTask) task) + .filter(task -> task.getTaskType().equals(taskType)) + .findFirst() + .orElse(null); + } +} diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java new file mode 100644 index 0000000000..536400a952 --- /dev/null +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java @@ -0,0 +1,228 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; + +import org.conductoross.conductor.sdk.workflow.task.InputParam; +import org.conductoross.conductor.sdk.workflow.task.WorkerTask; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +public class TestAnnotatedWorkflowSystemTask { + + private WorkflowModel workflow; + private WorkflowExecutor workflowExecutor; + + @Before + public void setUp() { + workflow = new WorkflowModel(); + workflow.setWorkflowId("test-workflow-123"); + workflowExecutor = mock(WorkflowExecutor.class); + } + + static class TestWorkerBean { + public Map successTask(@InputParam("input") String input) { + return Map.of("output", "processed: " + input); + } + + public void throwsException(@InputParam("input") String input) { + throw new RuntimeException("Task failed"); + } + + public void throwsNonRetryable(@InputParam("input") String input) { + throw new NonRetryableException("Terminal failure"); + } + + public Map returnsNull(@InputParam("input") String input) { + return null; + } + } + + @Test + public void testSuccessfulExecution() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("test_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("test_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "hello")); + + boolean result = systemTask.execute(workflow, task, workflowExecutor); + + assertTrue(result); + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("processed: hello", task.getOutputData().get("output")); + } + + @Test + public void testTaskWithException() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("throwsException", String.class); + WorkerTask annotation = createAnnotation("failing_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("failing_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "test")); + + boolean result = systemTask.execute(workflow, task, workflowExecutor); + + assertTrue(result); + assertEquals(TaskModel.Status.FAILED, task.getStatus()); + assertTrue(task.getReasonForIncompletion().contains("Task failed")); + } + + @Test + public void testTaskWithNonRetryableException() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("throwsNonRetryable", String.class); + WorkerTask annotation = createAnnotation("terminal_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("terminal_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "test")); + + boolean result = systemTask.execute(workflow, task, workflowExecutor); + + assertTrue(result); + assertEquals(TaskModel.Status.FAILED_WITH_TERMINAL_ERROR, task.getStatus()); + assertTrue(task.getReasonForIncompletion().contains("Terminal failure")); + } + + @Test + public void testTaskReturnsNull() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("returnsNull", String.class); + WorkerTask annotation = createAnnotation("null_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("null_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "test")); + + boolean result = systemTask.execute(workflow, task, workflowExecutor); + + assertTrue(result); + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertTrue(task.getOutputData().isEmpty()); + } + + @Test + public void testIsAsync() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("async_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("async_task", method, bean, annotation); + + assertTrue(systemTask.isAsync()); + } + + @Test + public void testCancel() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("cancel_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("cancel_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "test")); + + systemTask.cancel(workflow, task, workflowExecutor); + + assertEquals(TaskModel.Status.CANCELED, task.getStatus()); + } + + @Test + public void testGetTaskType() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("my_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("my_task", method, bean, annotation); + + assertEquals("my_task", systemTask.getTaskType()); + } + + @Test + public void testGetAnnotation() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("test"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("test", method, bean, annotation); + + assertSame(annotation, systemTask.getAnnotation()); + } + + @Test + public void testGetMethod() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("test"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("test", method, bean, annotation); + + assertSame(method, systemTask.getMethod()); + } + + @Test + public void testGetBean() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = TestWorkerBean.class.getMethod("successTask", String.class); + WorkerTask annotation = createAnnotation("test"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("test", method, bean, annotation); + + assertSame(bean, systemTask.getBean()); + } + + private TaskModel createTask(Map inputData) { + TaskModel task = new TaskModel(); + task.setTaskId("task-123"); + task.setWorkflowInstanceId("workflow-123"); + task.setInputData(new HashMap<>(inputData)); + task.setOutputData(new HashMap<>()); + return task; + } + + private WorkerTask createAnnotation(String taskName) { + WorkerTask annotation = Mockito.mock(WorkerTask.class); + Mockito.when(annotation.value()).thenReturn(taskName); + Mockito.when(annotation.threadCount()).thenReturn(1); + Mockito.when(annotation.pollingInterval()).thenReturn(100); + Mockito.when(annotation.domain()).thenReturn(""); + // pollTimeout and pollerCount not available in SDK v3.x + return annotation; + } +} diff --git a/http-task/src/test/java/com/netflix/conductor/tasks/http/HttpTaskTest.java b/http-task/src/test/java/com/netflix/conductor/tasks/http/HttpTaskTest.java index 5a26fd59af..5be0e5c26c 100644 --- a/http-task/src/test/java/com/netflix/conductor/tasks/http/HttpTaskTest.java +++ b/http-task/src/test/java/com/netflix/conductor/tasks/http/HttpTaskTest.java @@ -367,6 +367,7 @@ public void testOptional() { externalPayloadStorageUtils, systemTaskRegistry, Collections.emptyMap(), + Collections.emptyMap(), Duration.ofMinutes(60)) .decide(workflow); } From 0a7580b75cb0a8c3e270c3933b085b7987e1fc96 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Tue, 27 Jan 2026 23:12:29 -0800 Subject: [PATCH 02/39] support TaskContext --- .../AnnotatedMethodParameterMapper.java | 8 +- .../AnnotatedWorkflowSystemTask.java | 3 + .../tasks/annotated/TaskContext.java | 93 +++++++++++++++++++ .../TestAnnotatedWorkflowSystemTask.java | 29 ++++++ .../tasks/annotated/TestTaskContext.java | 69 ++++++++++++++ 5 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/TaskContext.java create mode 100644 core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestTaskContext.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java index 24d4491dde..f920659abf 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java @@ -77,6 +77,13 @@ private Object[] mapAnnotatedParameters( Object[] values = new Object[parameterTypes.length]; for (int i = 0; i < parameterTypes.length; i++) { + Class parameterType = parameterTypes[i]; + + if (parameterType.equals(TaskContext.class)) { + values[i] = TaskContext.get(); + continue; + } + Annotation[] paramAnnotation = parameterAnnotations[i]; // WorkflowInstanceIdInputParam not available in SDK v3.x - uncomment when @@ -87,7 +94,6 @@ private Object[] mapAnnotatedParameters( // } else if (paramAnnotation.length > 0) { Type type = parameters[i].getParameterizedType(); - Class parameterType = parameterTypes[i]; values[i] = getInputValue(task, parameterType, type, paramAnnotation); } else { // No annotation - convert entire input data to parameter type diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java index a4338148cc..65673c4024 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java @@ -71,6 +71,7 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor workf @Override public boolean execute( WorkflowModel workflow, TaskModel task, WorkflowExecutor workflowExecutor) { + TaskContext.set(task); try { LOGGER.debug( "Executing annotated task {} for workflow {}", @@ -99,6 +100,8 @@ public boolean execute( task.setStatus(TaskModel.Status.FAILED); task.setReasonForIncompletion("Unexpected error: " + e.getMessage()); return true; + } finally { + TaskContext.clear(); } } diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/TaskContext.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/TaskContext.java new file mode 100644 index 0000000000..f50ece80dc --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/TaskContext.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import com.netflix.conductor.model.TaskModel; + +/** + * Context that holds the current Task being executed. This allows annotated methods to access the + * Task object and its properties. + */ +public class TaskContext { + + private static final ThreadLocal THREAD_LOCAL = new ThreadLocal<>(); + + private final TaskModel taskModel; + + private TaskContext(TaskModel taskModel) { + this.taskModel = taskModel; + } + + /** + * Sets the current TaskContext for the thread. + * + * @param taskModel The task model to set + * @return The created TaskContext + */ + public static TaskContext set(TaskModel taskModel) { + TaskContext context = new TaskContext(taskModel); + THREAD_LOCAL.set(context); + return context; + } + + /** + * @return The current TaskContext or null if not set + */ + public static TaskContext get() { + return THREAD_LOCAL.get(); + } + + /** Clears the TaskContext for the current thread. */ + public static void clear() { + THREAD_LOCAL.remove(); + } + + public TaskModel getTaskModel() { + return taskModel; + } + + public String getWorkflowInstanceId() { + return taskModel.getWorkflowInstanceId(); + } + + public String getTaskId() { + return taskModel.getTaskId(); + } + + public String getWorkerId() { + return taskModel.getWorkerId(); + } + + public int getRetryCount() { + return taskModel.getRetryCount(); + } + + public int getPollCount() { + return taskModel.getPollCount(); + } + + public long getCallbackAfterSeconds() { + return taskModel.getCallbackAfterSeconds(); + } + + public void setCallbackAfter(long seconds) { + taskModel.setCallbackAfterSeconds(seconds); + } + + public void addLog(String log) { + // TaskModel does not support logs directly. + // For system tasks, logging is typically handled via SLF4J or side effects. + // We might want to support attaching logs to execution metadata or similar in + // future. + } +} diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java index 536400a952..5f2093c91f 100644 --- a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java @@ -57,6 +57,14 @@ public void throwsNonRetryable(@InputParam("input") String input) { public Map returnsNull(@InputParam("input") String input) { return null; } + + public Map taskWithContext( + TaskContext context, @InputParam("input") String input) { + if (context == null) { + throw new RuntimeException("TaskContext is null"); + } + return Map.of("taskId", context.getTaskId(), "input", input); + } } @Test @@ -131,6 +139,27 @@ public void testTaskReturnsNull() throws Exception { assertTrue(task.getOutputData().isEmpty()); } + @Test + public void testTaskWithContext() throws Exception { + TestWorkerBean bean = new TestWorkerBean(); + Method method = + TestWorkerBean.class.getMethod("taskWithContext", TaskContext.class, String.class); + WorkerTask annotation = createAnnotation("context_task"); + + AnnotatedWorkflowSystemTask systemTask = + new AnnotatedWorkflowSystemTask("context_task", method, bean, annotation); + + TaskModel task = createTask(Map.of("input", "context_test")); + task.setTaskId("ctx-task-id"); + + boolean result = systemTask.execute(workflow, task, workflowExecutor); + + assertTrue(result); + assertEquals(TaskModel.Status.COMPLETED, task.getStatus()); + assertEquals("ctx-task-id", task.getOutputData().get("taskId")); + assertEquals("context_test", task.getOutputData().get("input")); + } + @Test public void testIsAsync() throws Exception { TestWorkerBean bean = new TestWorkerBean(); diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestTaskContext.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestTaskContext.java new file mode 100644 index 0000000000..66a62259ce --- /dev/null +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestTaskContext.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution.tasks.annotated; + +import org.junit.After; +import org.junit.Test; + +import com.netflix.conductor.model.TaskModel; + +import static org.junit.Assert.*; + +public class TestTaskContext { + + @After + public void cleanup() { + TaskContext.clear(); + } + + @Test + public void testThreadLocalBehavior() { + TaskModel task = new TaskModel(); + task.setTaskId("id-1"); + + assertNull(TaskContext.get()); + + TaskContext context = TaskContext.set(task); + assertNotNull(context); + assertEquals("id-1", context.getTaskId()); + assertSame(context, TaskContext.get()); + + TaskContext.clear(); + assertNull(TaskContext.get()); + } + + @Test + public void testGettersAndSetters() { + TaskModel task = new TaskModel(); + task.setWorkflowInstanceId("workflow-1"); + task.setTaskId("task-1"); + task.setWorkerId("worker-1"); + task.setRetryCount(3); + task.setPollCount(5); + task.setCallbackAfterSeconds(10); + + TaskContext.set(task); + TaskContext context = TaskContext.get(); + + assertEquals("workflow-1", context.getWorkflowInstanceId()); + assertEquals("task-1", context.getTaskId()); + assertEquals("worker-1", context.getWorkerId()); + assertEquals(3, context.getRetryCount()); + assertEquals(5, context.getPollCount()); + assertEquals(10, context.getCallbackAfterSeconds()); + + context.setCallbackAfter(20); + assertEquals(20, task.getCallbackAfterSeconds()); + assertEquals(20, context.getCallbackAfterSeconds()); + } +} From 9fcf5eb49b613ec7fe00932bcc94342dd98b20a5 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 23 Jan 2026 21:57:30 +0530 Subject: [PATCH 03/39] Fix: Restore missing postpone logic from old sweeper (PR #703 regression) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed regression from PR #703 where workflows could hang indefinitely: 1. WorkflowExecutorOps.decide() - Removed buggy conditional that only rescheduled workflows if postpone != timeout. Now always reschedules non-terminal workflows. 2. ExecutorUtils.computePostpone() - Added missing logic from old sweeper: - HUMAN task handling (use default timeout) - Max postpone duration capping - Jitter (±1/3 of duration) to prevent thundering herd These changes restore the behavior from the old sweeper that was lost when PR #703 introduced the new sweeper implementation. --- .../core/execution/WorkflowExecutorOps.java | 22 +++++------ .../core/execution/ExecutorUtils.java | 39 +++++++++++++++++-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java index 6afe4f0e07..ac94f77e9f 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java @@ -1149,18 +1149,16 @@ private WorkflowModel decide(WorkflowModel workflow) { Duration timeout = properties.getWorkflowOffsetTimeout(); if (!workflow.getStatus().isTerminal()) { - Duration updatedOffset = computePostpone(workflow, timeout); - if (updatedOffset.getSeconds() != timeout.getSeconds()) { - // we have a new value, setUnack uses time in millis - LOGGER.debug( - "Pushing the workflow {} into decider queue by {} millis", - workflow.getWorkflowId(), - updatedOffset.getSeconds() * 1000); - queueDAO.setUnackTimeout( - DECIDER_QUEUE, - workflow.getWorkflowId(), - updatedOffset.getSeconds() * 1000); - } + Duration updatedOffset = + computePostpone( + workflow, timeout, properties.getMaxPostponeDurationSeconds()); + // Always reschedule non-terminal workflows to ensure they remain in the queue + LOGGER.debug( + "Rescheduling workflow {} to decider queue with postpone duration {} seconds", + workflow.getWorkflowId(), + updatedOffset.getSeconds()); + queueDAO.setUnackTimeout( + DECIDER_QUEUE, workflow.getWorkflowId(), updatedOffset.getSeconds() * 1000); } return workflow; diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java index bbdaa7be19..5a2904739b 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java @@ -13,6 +13,7 @@ package org.conductoross.conductor.core.execution; import java.time.Duration; +import java.util.Random; import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.model.TaskModel; @@ -26,9 +27,12 @@ public class ExecutorUtils { public static Duration computePostpone( - WorkflowModel workflowModel, Duration workflowOffsetTimeout) { + WorkflowModel workflowModel, + Duration workflowOffsetTimeout, + Duration maxPostponeDuration) { long currentTimeMillis = System.currentTimeMillis(); long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); + long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); long postponeDurationSeconds = 0; for (TaskModel taskModel : workflowModel.getTasks()) { @@ -40,6 +44,10 @@ public static Duration computePostpone( (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; } + } else if (taskModel.getTaskType().equals("HUMAN")) { + // HUMAN tasks always use the default workflowOffsetTimeout + // This matches the old sweeper behavior + postponeDurationSeconds = workflowOffsetTimeoutSeconds; } else { // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... // not sure but keeping it this way just in case. @@ -75,11 +83,16 @@ public static Duration computePostpone( workflowOffsetTimeoutSeconds, workflowModel.getWorkflowId()); + // Cap postpone duration to the configured maximum (matches old sweeper behavior) + if (postponeDurationSeconds > maxPostponeDurationSeconds) { + postponeDurationSeconds = maxPostponeDurationSeconds; + } + if (postponeDurationSeconds > 0) { - return Duration.ofSeconds( - Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds)); + long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); + return Duration.ofSeconds(applyJitter(capped)); } - return Duration.ofSeconds(workflowOffsetTimeoutSeconds); + return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); } private static long getMinTimeout(TaskDef taskDef) { @@ -89,4 +102,22 @@ private static long getMinTimeout(TaskDef taskDef) { taskDef.getTimeoutSeconds(), Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); } + + /** + * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the + * duration. + * + *

For example, if duration is 45 seconds, returns value between [30-60] seconds. + * + * @param durationSeconds postpone duration in seconds + * @return duration with jitter applied + */ + public static long applyJitter(long durationSeconds) { + if (durationSeconds <= 3) { + return durationSeconds; // Too small for jitter + } + long range = durationSeconds / 3; + long jitter = new Random().nextInt((int) (2 * range + 1)) - range; + return durationSeconds + jitter; + } } From df2ba8a565fdd2b8e2d8d146686ba38c1432bad3 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 23 Jan 2026 22:01:22 +0530 Subject: [PATCH 04/39] Refactor: Move sweeper to common package for Orkes compatibility Moved WorkflowSweeper and related classes from org.conductoross.conductor to com.netflix.conductor package to make them available as common implementation for both OSS and Orkes Conductor. Changes: - Deleted old disabled WorkflowSweeper from com.netflix.conductor.core.reconciliation - Moved WorkflowSweeper: org.conductoross...execution -> com.netflix...reconciliation - Moved ExecutorUtils: org.conductoross...execution -> com.netflix...execution - Moved SweeperProperties: org.conductoross...execution -> com.netflix...reconciliation - Added sweepAsync() method to WorkflowSweeper for compatibility - Updated imports in WorkflowExecutorOps --- .../core/execution/ExecutorUtils.java | 2 +- .../core/execution/WorkflowExecutorOps.java | 3 +- .../reconciliation}/SweeperProperties.java | 2 +- .../core/reconciliation/WorkflowSweeper.java | 388 +++++++++++------- .../core/execution/WorkflowSweeper.java | 294 ------------- 5 files changed, 242 insertions(+), 447 deletions(-) rename core/src/main/java/{org/conductoross => com/netflix}/conductor/core/execution/ExecutorUtils.java (99%) rename core/src/main/java/{org/conductoross/conductor/core/execution => com/netflix/conductor/core/reconciliation}/SweeperProperties.java (95%) delete mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java similarity index 99% rename from core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java rename to core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java index 5a2904739b..2720044ca2 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package org.conductoross.conductor.core.execution; +package com.netflix.conductor.core.execution; import java.time.Duration; import java.util.Random; diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java index ac94f77e9f..01025f8a0e 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java @@ -54,11 +54,10 @@ import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; +import static com.netflix.conductor.core.execution.ExecutorUtils.computePostpone; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; -import static org.conductoross.conductor.core.execution.ExecutorUtils.computePostpone; - /** Workflow services provider interface */ @Trace @Component diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java similarity index 95% rename from core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java rename to core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java index 813af44376..12af4ce025 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java @@ -10,7 +10,7 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ -package org.conductoross.conductor.core.execution; +package com.netflix.conductor.core.reconciliation; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index 45cda3d9b2..55294894c1 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Conductor Authors. + * Copyright 2025 Conductor Authors. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -12,61 +12,139 @@ */ package com.netflix.conductor.core.reconciliation; -import java.time.Instant; -import java.util.Optional; -import java.util.Random; +import java.time.Clock; +import java.time.Duration; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; -import com.netflix.conductor.annotations.VisibleForTesting; -import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.core.WorkflowContext; +import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.config.ConductorProperties; -import com.netflix.conductor.core.dal.ExecutionDAOFacade; -import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; +import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; +import com.netflix.conductor.core.utils.QueueUtils; +import com.netflix.conductor.core.utils.Utils; +import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; -import com.netflix.conductor.service.ExecutionLockService; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; -// @Component -public class WorkflowSweeper { - - private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowSweeper.class); +@Component +@Slf4j +@ConditionalOnProperty( + name = "conductor.app.sweeper.enabled", + havingValue = "true", + matchIfMissing = true) +public class WorkflowSweeper extends LifecycleAwareComponent { - private final ConductorProperties properties; - private final WorkflowExecutor workflowExecutor; - private final WorkflowRepairService workflowRepairService; private final QueueDAO queueDAO; - private final ExecutionDAOFacade executionDAOFacade; - private final ExecutionLockService executionLockService; - - private static final String CLASS_NAME = WorkflowSweeper.class.getSimpleName(); + private final SweeperProperties sweeperProperties; + private final WorkflowExecutor workflowExecutor; + private final ExecutionDAO executionDAO; + private final Duration worflowOffsetTimeout; + private final Executor sweeperExecutor; + private final ConductorProperties properties; + private final ObjectMapper objectMapper; + private SystemTaskRegistry systemTaskRegistry; + private final Clock clock = Clock.systemDefaultZone(); + private AtomicBoolean stop = new AtomicBoolean(false); public WorkflowSweeper( + @Qualifier(SWEEPER_EXECUTOR_NAME) Executor sweeperExecutor, + QueueDAO queueDAO, WorkflowExecutor workflowExecutor, - Optional workflowRepairService, + ExecutionDAO executionDAO, ConductorProperties properties, - QueueDAO queueDAO, - ExecutionDAOFacade executionDAOFacade, - ExecutionLockService executionLockService) { - this.properties = properties; + SweeperProperties sweeperProperties, + SystemTaskRegistry systemTaskRegistry, + ObjectMapper objectMapper) { this.queueDAO = queueDAO; + this.executionDAO = executionDAO; + this.sweeperProperties = sweeperProperties; this.workflowExecutor = workflowExecutor; - this.executionDAOFacade = executionDAOFacade; - this.workflowRepairService = workflowRepairService.orElse(null); - this.executionLockService = executionLockService; - LOGGER.info("WorkflowSweeper initialized."); + this.worflowOffsetTimeout = properties.getWorkflowOffsetTimeout(); + this.sweeperExecutor = sweeperExecutor; + this.properties = properties; + this.systemTaskRegistry = systemTaskRegistry; + this.objectMapper = objectMapper; + log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); + for (int i = 0; i < properties.getSweeperThreadCount(); i++) { + sweeperExecutor.execute(this::pollAndSweep); + } + } + + /* + For system task -> Verify the task isAsync() and not isAsyncComplete() or isAsyncComplete() in SCHEDULED state, + and in SCHEDULED or IN_PROGRESS state. (Example: SUB_WORKFLOW tasks in SCHEDULED state) + For simple task -> Verify the task is in SCHEDULED state. + */ + private final Predicate isTaskRepairable = + task -> { + if (systemTaskRegistry.isSystemTask(task.getTaskType())) { // If system task + WorkflowSystemTask workflowSystemTask = + systemTaskRegistry.get(task.getTaskType()); + return workflowSystemTask.isAsync() + && (!workflowSystemTask.isAsyncComplete(task) + || (workflowSystemTask.isAsyncComplete(task) + && task.getStatus() == TaskModel.Status.SCHEDULED)) + && (task.getStatus() == TaskModel.Status.IN_PROGRESS + || task.getStatus() == TaskModel.Status.SCHEDULED); + } else { // Else if simple task or wait task + return (task.getStatus() == TaskModel.Status.SCHEDULED + || (!task.getStatus().isTerminal() + && task.getWaitTimeout() > 0 + && (clock.millis() - task.getWaitTimeout() > 1000))); + } + }; + + private void pollAndSweep() { + try { + while (true) { + if (stop.get()) { + return; + } + try { + if (!isRunning()) { + log.trace("Component stopped, skip workflow sweep"); + } else { + List workflowIds = + queueDAO.pop( + DECIDER_QUEUE, + sweeperProperties.getSweepBatchSize(), + sweeperProperties.getQueuePopTimeout()); + log.trace("Found {} workflows to sweep", workflowIds.size()); + workflowIds.stream() + .parallel() + .forEach( + workflowId -> + Monitors.getTimer("workflowSweeper") + .record(() -> sweep(workflowId))); + } + } catch (Throwable e) { + log.warn("Error while running sweeper {}", e.getMessage(), e); + } + } + } catch (Throwable e) { + log.error("Error polling for sweep entries {}", e.getMessage(), e); + } } @Async(SWEEPER_EXECUTOR_NAME) @@ -76,137 +154,149 @@ public CompletableFuture sweepAsync(String workflowId) { } public void sweep(String workflowId) { - WorkflowContext workflowContext = new WorkflowContext(properties.getAppId()); - WorkflowContext.set(workflowContext); - WorkflowModel workflow = null; + log.info("Running sweeper for workflow {}", workflowId); + try { - if (!executionLockService.acquireLock(workflowId)) { + WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); + if (workflow == null || workflow.getStatus().isTerminal()) { + queueDAO.remove(DECIDER_QUEUE, workflowId); return; } - workflow = executionDAOFacade.getWorkflowModel(workflowId, true); - LOGGER.debug("Running sweeper for workflow {}", workflowId); - if (workflowRepairService != null) { - // Verify and repair tasks in the workflow. - workflowRepairService.verifyAndRepairWorkflowTasks(workflow); + + String tasks = + workflow.getTasks().stream() + .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) + .toList() + .toString(); + workflow = workflowExecutor.decideWithLock(workflow); + if (workflow == null) { + // couldn't get a lock + // Let's try again... with the lockTime timeout / 2 + int backoff = (int) (properties.getLockLeaseTime().toMillis() / 2); + log.info("can't get a lock on {}, will try after {} ms", workflowId, backoff); + queueDAO.push(DECIDER_QUEUE, workflowId, 0, Duration.ofMillis(backoff).toSeconds()); + return; } - long decideStartTime = System.currentTimeMillis(); - workflow = workflowExecutor.decide(workflow.getWorkflowId()); - Monitors.recordWorkflowDecisionTime(System.currentTimeMillis() - decideStartTime); - if (workflow != null && workflow.getStatus().isTerminal()) { - queueDAO.remove(DECIDER_QUEUE, workflowId); + if (workflow.getStatus().isTerminal()) { + queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); return; } - } catch (NotFoundException nfe) { - queueDAO.remove(DECIDER_QUEUE, workflowId); - LOGGER.info( - "Workflow NOT found for id:{}. Removed it from decider queue", workflowId, nfe); - return; - } catch (Exception e) { - Monitors.error(CLASS_NAME, "sweep"); - LOGGER.error("Error running sweep for " + workflowId, e); - } finally { - executionLockService.releaseLock(workflowId); - } - long workflowOffsetTimeout = - workflowOffsetWithJitter(properties.getWorkflowOffsetTimeout().getSeconds()); - if (workflow != null) { - long startTime = Instant.now().toEpochMilli(); - unack(workflow, workflowOffsetTimeout); - long endTime = Instant.now().toEpochMilli(); - Monitors.recordUnackTime(workflow.getWorkflowName(), endTime - startTime); - } else { - LOGGER.warn( - "Workflow with {} id can not be found. Attempting to unack using the id", - workflowId); - queueDAO.setUnackTimeout(DECIDER_QUEUE, workflowId, workflowOffsetTimeout * 1000); - } - } - /** - * Calculates the next decider queue unack delay by evaluating active tasks and choosing the - * smallest eligible delay. This prevents long-running tasks from delaying evaluation when a - * shorter timeout is due, while still honoring maxPostpone caps. - */ - @VisibleForTesting - void unack(WorkflowModel workflowModel, long workflowOffsetTimeout) { - // Pick the minimum next-evaluation delay across eligible tasks, capped by maxPostpone. - Long postponeDurationSeconds = null; - long maxPostponeSeconds = properties.getMaxPostponeDurationSeconds().getSeconds(); - for (TaskModel taskModel : workflowModel.getTasks()) { - Long candidateSeconds = null; - if (taskModel.getStatus() == Status.IN_PROGRESS) { - // Active tasks: delay based on wait/response timeout or workflow offset. - if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_WAIT)) { - if (taskModel.getWaitTimeout() == 0) { - candidateSeconds = workflowOffsetTimeout; - } else { - // waitTimeout is an absolute epoch ms; compute remaining seconds. - long deltaInSeconds = - (taskModel.getWaitTimeout() - System.currentTimeMillis()) / 1000; - candidateSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; - } - } else if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_HUMAN)) { - candidateSeconds = workflowOffsetTimeout; - } else { - candidateSeconds = - (taskModel.getResponseTimeoutSeconds() != 0) - // Add 1s so the response timeout window fully elapses. - ? taskModel.getResponseTimeoutSeconds() + 1 - : workflowOffsetTimeout; - } - } else if (taskModel.getStatus() == Status.SCHEDULED) { - // Scheduled tasks: use poll timeout when present, else workflow timeout or offset. - Optional taskDefinition = taskModel.getTaskDefinition(); - if (taskDefinition.isPresent()) { - TaskDef taskDef = taskDefinition.get(); - if (taskDef.getPollTimeoutSeconds() != null - && taskDef.getPollTimeoutSeconds() != 0) { - candidateSeconds = taskDef.getPollTimeoutSeconds().longValue() + 1; - } else { - candidateSeconds = - (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) - ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() - + 1 - : workflowOffsetTimeout; - } - } else { - candidateSeconds = - (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) - ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1 - : workflowOffsetTimeout; - } + String tasksAfterDecide = + workflow.getTasks().stream() + .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) + .toList() + .toString(); + + // Workflow has not completed and decide did not change the status of the tasks + // Every task that is running MUST be in the queue + if (tasks.equals(tasksAfterDecide)) { + workflow.getTasks().forEach(this::verifyAndRepairTask); + } + + // Workflow is in running status, there MUST be at-least one task that is not terminal + // (scheduled, in progress) + boolean hasRunningTasks = + workflow.getTasks().stream().anyMatch(task -> !task.getStatus().isTerminal()); + if (!hasRunningTasks) { + // Workflow is in RUNNING status but there are no tasks that are running + // This can happen in case of the database failures where the task scheduling failed + // after the last task was completed + // To fix, we reset the executed flag of the last task and re-run decide + forceSetLastTaskAsNotExecuted(workflow); + workflow = workflowExecutor.decideWithLock(workflow); } - if (candidateSeconds == null) { - continue; + // If parent workflow exists, call repair on that too - meaning ensure the parent is in + // the decider queue + if (workflow != null && StringUtils.isNotBlank(workflow.getParentWorkflowId())) { + ensureWorkflowExistsInDecider(workflow.getParentWorkflowId()); } - if (candidateSeconds < 0) { - candidateSeconds = 0L; + } catch (Throwable e) { + log.error("Error running sweep for {}, error = {}", workflowId, e.getMessage(), e); + } + } + + void verifyAndRepairTask(TaskModel task) { + if (isTaskRepairable.test(task)) { + // Ensure QueueDAO contains this taskId + String taskQueueName = QueueUtils.getQueueName(task); + if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { + queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); + log.info( + "Task {} in workflow {} re-queued for repairs", + task.getTaskId(), + task.getWorkflowInstanceId()); + Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); } - // Cap every candidate to avoid excessive unack delay. - if (candidateSeconds > maxPostponeSeconds) { - candidateSeconds = maxPostponeSeconds; + } + if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) + && task.getStatus() == TaskModel.Status.IN_PROGRESS) { + WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); + if (subWorkflow.getStatus().isTerminal()) { + log.info( + "Repairing sub workflow task {} for sub workflow {} in workflow {}", + task.getTaskId(), + task.getSubWorkflowId(), + task.getWorkflowInstanceId()); + repairSubWorkflowTask(task, subWorkflow); } - if (postponeDurationSeconds == null || candidateSeconds < postponeDurationSeconds) { - postponeDurationSeconds = candidateSeconds; + } + } + + private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { + switch (subWorkflow.getStatus()) { + case COMPLETED: + task.setStatus(TaskModel.Status.COMPLETED); + break; + case FAILED: + task.setStatus(TaskModel.Status.FAILED); + break; + case TERMINATED: + task.setStatus(TaskModel.Status.CANCELED); + break; + case TIMED_OUT: + task.setStatus(TaskModel.Status.TIMED_OUT); + break; + } + task.addOutput(subWorkflow.getOutput()); + executionDAO.updateTask(task); + } + + private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { + if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { + TaskModel taskModel = workflow.getTasks().getLast(); + log.warn( + "Force setting isExecuted to false for last task - {} - {} - {} - {} for workflow {}", + taskModel.getTaskId(), + taskModel.getReferenceTaskName(), + taskModel.getStatus(), + taskModel.getTaskDefName(), + taskModel.getWorkflowInstanceId()); + try { + log.debug( + "workflow {} JSON {}", + workflow.getWorkflowId(), + objectMapper.writeValueAsString(workflow)); + } catch (Exception e) { + log.error("Could not warn about workflow {}", workflow.getWorkflowId(), e); } + taskModel.setExecuted(false); + executionDAO.updateWorkflow(workflow); + } + } + + private void ensureWorkflowExistsInDecider(String workflowId) { + String queueName = Utils.DECIDER_QUEUE; + if (!queueDAO.containsMessage(queueName, workflowId)) { + queueDAO.push(queueName, workflowId, worflowOffsetTimeout.getSeconds()); + Monitors.recordQueueMessageRepushFromRepairService(queueName); } - // Default to immediate re-evaluation if no eligible task delay is found. - long unackSeconds = (postponeDurationSeconds != null) ? postponeDurationSeconds : 0; - queueDAO.setUnackTimeout(DECIDER_QUEUE, workflowModel.getWorkflowId(), unackSeconds * 1000); } - /** - * jitter will be +- (1/3) workflowOffsetTimeout for example, if workflowOffsetTimeout is 45 - * seconds, this function returns values between [30-60] seconds - * - * @param workflowOffsetTimeout - * @return - */ - @VisibleForTesting - long workflowOffsetWithJitter(long workflowOffsetTimeout) { - long range = workflowOffsetTimeout / 3; - long jitter = new Random().nextInt((int) (2 * range + 1)) - range; - return workflowOffsetTimeout + jitter; + @Override + public void doStop() { + stop.set(true); + ((ExecutorService) this.sweeperExecutor).shutdownNow(); } } diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java deleted file mode 100644 index 9aad1765ee..0000000000 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package org.conductoross.conductor.core.execution; - -import java.time.Clock; -import java.time.Duration; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Predicate; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.core.LifecycleAwareComponent; -import com.netflix.conductor.core.config.ConductorProperties; -import com.netflix.conductor.core.execution.WorkflowExecutor; -import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; -import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; -import com.netflix.conductor.core.utils.QueueUtils; -import com.netflix.conductor.core.utils.Utils; -import com.netflix.conductor.dao.ExecutionDAO; -import com.netflix.conductor.dao.QueueDAO; -import com.netflix.conductor.metrics.Monitors; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; - -import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; -import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; - -@Component -@Slf4j -@ConditionalOnProperty( - name = "conductor.app.sweeper.enabled", - havingValue = "true", - matchIfMissing = true) -public class WorkflowSweeper extends LifecycleAwareComponent { - - private final QueueDAO queueDAO; - private final SweeperProperties sweeperProperties; - private final WorkflowExecutor workflowExecutor; - private final ExecutionDAO executionDAO; - private final Duration worflowOffsetTimeout; - private final Executor sweeperExecutor; - private final ConductorProperties properties; - private final ObjectMapper objectMapper; - private SystemTaskRegistry systemTaskRegistry; - private final Clock clock = Clock.systemDefaultZone(); - private AtomicBoolean stop = new AtomicBoolean(false); - - public WorkflowSweeper( - @Qualifier(SWEEPER_EXECUTOR_NAME) Executor sweeperExecutor, - QueueDAO queueDAO, - WorkflowExecutor workflowExecutor, - ExecutionDAO executionDAO, - ConductorProperties properties, - SweeperProperties sweeperProperties, - SystemTaskRegistry systemTaskRegistry, - ObjectMapper objectMapper) { - this.queueDAO = queueDAO; - this.executionDAO = executionDAO; - this.sweeperProperties = sweeperProperties; - this.workflowExecutor = workflowExecutor; - this.worflowOffsetTimeout = properties.getWorkflowOffsetTimeout(); - this.sweeperExecutor = sweeperExecutor; - this.properties = properties; - this.systemTaskRegistry = systemTaskRegistry; - this.objectMapper = objectMapper; - log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); - for (int i = 0; i < properties.getSweeperThreadCount(); i++) { - sweeperExecutor.execute(this::pollAndSweep); - } - } - - /* - For system task -> Verify the task isAsync() and not isAsyncComplete() or isAsyncComplete() in SCHEDULED state, - and in SCHEDULED or IN_PROGRESS state. (Example: SUB_WORKFLOW tasks in SCHEDULED state) - For simple task -> Verify the task is in SCHEDULED state. - */ - private final Predicate isTaskRepairable = - task -> { - if (systemTaskRegistry.isSystemTask(task.getTaskType())) { // If system task - WorkflowSystemTask workflowSystemTask = - systemTaskRegistry.get(task.getTaskType()); - return workflowSystemTask.isAsync() - && (!workflowSystemTask.isAsyncComplete(task) - || (workflowSystemTask.isAsyncComplete(task) - && task.getStatus() == TaskModel.Status.SCHEDULED)) - && (task.getStatus() == TaskModel.Status.IN_PROGRESS - || task.getStatus() == TaskModel.Status.SCHEDULED); - } else { // Else if simple task or wait task - return (task.getStatus() == TaskModel.Status.SCHEDULED - || (!task.getStatus().isTerminal() - && task.getWaitTimeout() > 0 - && (clock.millis() - task.getWaitTimeout() > 1000))); - } - }; - - private void pollAndSweep() { - try { - while (true) { - if (stop.get()) { - return; - } - try { - if (!isRunning()) { - log.trace("Component stopped, skip workflow sweep"); - } else { - List workflowIds = - queueDAO.pop( - DECIDER_QUEUE, - sweeperProperties.getSweepBatchSize(), - sweeperProperties.getQueuePopTimeout()); - log.trace("Found {} workflows to sweep", workflowIds.size()); - workflowIds.stream() - .parallel() - .forEach( - workflowId -> - Monitors.getTimer("workflowSweeper") - .record(() -> sweep(workflowId))); - } - } catch (Throwable e) { - log.warn("Error while running sweeper {}", e.getMessage(), e); - } - } - } catch (Throwable e) { - log.error("Error polling for sweep entries {}", e.getMessage(), e); - } - } - - public void sweep(String workflowId) { - log.info("Running sweeper for workflow {}", workflowId); - - try { - WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); - if (workflow == null || workflow.getStatus().isTerminal()) { - queueDAO.remove(DECIDER_QUEUE, workflowId); - return; - } - - String tasks = - workflow.getTasks().stream() - .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) - .toList() - .toString(); - workflow = workflowExecutor.decideWithLock(workflow); - if (workflow == null) { - // couldn't get a lock - // Let's try again... with the lockTime timeout / 2 - int backoff = (int) (properties.getLockLeaseTime().toMillis() / 2); - log.info("can't get a lock on {}, will try after {} ms", workflowId, backoff); - queueDAO.push(DECIDER_QUEUE, workflowId, 0, Duration.ofMillis(backoff).toSeconds()); - return; - } - if (workflow.getStatus().isTerminal()) { - queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); - return; - } - - String tasksAfterDecide = - workflow.getTasks().stream() - .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) - .toList() - .toString(); - - // Workflow has not completed and decide did not change the status of the tasks - // Every task that is running MUST be in the queue - if (tasks.equals(tasksAfterDecide)) { - workflow.getTasks().forEach(this::verifyAndRepairTask); - } - - // Workflow is in running status, there MUST be at-least one task that is not terminal - // (scheduled, in progress) - boolean hasRunningTasks = - workflow.getTasks().stream().anyMatch(task -> !task.getStatus().isTerminal()); - if (!hasRunningTasks) { - // Workflow is in RUNNING status but there are no tasks that are running - // This can happen in case of the database failures where the task scheduling failed - // after the last task was completed - // To fix, we reset the executed flag of the last task and re-run decide - forceSetLastTaskAsNotExecuted(workflow); - workflow = workflowExecutor.decideWithLock(workflow); - } - - // If parent workflow exists, call repair on that too - meaning ensure the parent is in - // the decider queue - if (workflow != null && StringUtils.isNotBlank(workflow.getParentWorkflowId())) { - ensureWorkflowExistsInDecider(workflow.getParentWorkflowId()); - } - } catch (Throwable e) { - log.error("Error running sweep for {}, error = {}", workflowId, e.getMessage(), e); - } - } - - void verifyAndRepairTask(TaskModel task) { - if (isTaskRepairable.test(task)) { - // Ensure QueueDAO contains this taskId - String taskQueueName = QueueUtils.getQueueName(task); - if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { - queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); - log.info( - "Task {} in workflow {} re-queued for repairs", - task.getTaskId(), - task.getWorkflowInstanceId()); - Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); - } - } - if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) - && task.getStatus() == TaskModel.Status.IN_PROGRESS) { - WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); - if (subWorkflow.getStatus().isTerminal()) { - log.info( - "Repairing sub workflow task {} for sub workflow {} in workflow {}", - task.getTaskId(), - task.getSubWorkflowId(), - task.getWorkflowInstanceId()); - repairSubWorkflowTask(task, subWorkflow); - } - } - } - - private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { - switch (subWorkflow.getStatus()) { - case COMPLETED: - task.setStatus(TaskModel.Status.COMPLETED); - break; - case FAILED: - task.setStatus(TaskModel.Status.FAILED); - break; - case TERMINATED: - task.setStatus(TaskModel.Status.CANCELED); - break; - case TIMED_OUT: - task.setStatus(TaskModel.Status.TIMED_OUT); - break; - } - task.addOutput(subWorkflow.getOutput()); - executionDAO.updateTask(task); - } - - private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { - if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { - TaskModel taskModel = workflow.getTasks().getLast(); - log.warn( - "Force setting isExecuted to false for last task - {} - {} - {} - {} for workflow {}", - taskModel.getTaskId(), - taskModel.getReferenceTaskName(), - taskModel.getStatus(), - taskModel.getTaskDefName(), - taskModel.getWorkflowInstanceId()); - try { - log.debug( - "workflow {} JSON {}", - workflow.getWorkflowId(), - objectMapper.writeValueAsString(workflow)); - } catch (Exception e) { - log.error("Could not warn about workflow {}", workflow.getWorkflowId(), e); - } - taskModel.setExecuted(false); - executionDAO.updateWorkflow(workflow); - } - } - - private void ensureWorkflowExistsInDecider(String workflowId) { - String queueName = Utils.DECIDER_QUEUE; - if (!queueDAO.containsMessage(queueName, workflowId)) { - queueDAO.push(queueName, workflowId, worflowOffsetTimeout.getSeconds()); - Monitors.recordQueueMessageRepushFromRepairService(queueName); - } - } - - @Override - public void doStop() { - stop.set(true); - ((ExecutorService) this.sweeperExecutor).shutdownNow(); - } -} From 0caef279d7820123186c74843ca67d544e7e2353 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 22:08:04 +0530 Subject: [PATCH 05/39] Add tests for ExecutorUtils.computePostpone - Add TestExecutorUtils.java with 15 unit tests for postpone logic - Tests validate HUMAN task handling, max postpone capping, and jitter - Remove broken TestWorkflowSweeper.java (incompatible with new sweeper API) --- .../core/execution/TestExecutorUtils.java | 375 +++++++++++++++ .../reconciliation/TestWorkflowSweeper.java | 427 ------------------ gradle.properties | 3 +- 3 files changed, 377 insertions(+), 428 deletions(-) create mode 100644 core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java delete mode 100644 core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java new file mode 100644 index 0000000000..d35d162d76 --- /dev/null +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java @@ -0,0 +1,375 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.core.execution; + +import java.time.Duration; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import static org.junit.Assert.*; + +/** + * Unit tests for ExecutorUtils.computePostpone() to ensure correct postpone duration calculation + * for workflow rescheduling in the decider queue. + */ +public class TestExecutorUtils { + + private WorkflowModel workflow; + private WorkflowDef workflowDef; + private Duration workflowOffsetTimeout; + private Duration maxPostponeDuration; + + @Before + public void setup() { + workflowDef = new WorkflowDef(); + workflowDef.setName("test_workflow"); + workflowDef.setTimeoutSeconds(0); + + workflow = new WorkflowModel(); + workflow.setWorkflowDefinition(workflowDef); + workflow.setWorkflowId("test-workflow-id"); + + workflowOffsetTimeout = Duration.ofSeconds(30); + maxPostponeDuration = Duration.ofSeconds(300); + } + + @Test + public void testComputePostpone_WaitTaskWithTimeout() { + // Create a WAIT task with specific wait timeout + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + long futureTime = System.currentTimeMillis() + 60000; // 60 seconds in future + waitTask.setWaitTimeout(futureTime); + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be capped at workflowOffsetTimeout (30 seconds) with jitter + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_WaitTaskWithZeroTimeout() { + // Create a WAIT task without specific timeout + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(0); + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return the default workflow offset timeout (30 seconds, with jitter) + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_HumanTask() { + // HUMAN tasks should always use default workflowOffsetTimeout + TaskModel humanTask = new TaskModel(); + humanTask.setTaskType("HUMAN"); + humanTask.setStatus(TaskModel.Status.IN_PROGRESS); + humanTask.setStartTime(System.currentTimeMillis()); + + workflow.setTasks(List.of(humanTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return the default workflow offset timeout (30 seconds, with jitter) + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_InProgressTaskWithResponseTimeout() { + // Task with response timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(120L); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis() - 10000); // Started 10 seconds ago + task.setResponseTimeoutSeconds(120); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout (30s) + // Then jitter: 30 ± 10 = [20, 40] seconds + assertTrue( + "Postpone should be capped at workflow offset (20-40s)", + result.getSeconds() >= 20); + assertTrue( + "Postpone should be capped at workflow offset (20-40s)", + result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ScheduledTaskWithPollTimeout() { + // Task definition with poll timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setPollTimeoutSeconds(45); + taskDef.setResponseTimeoutSeconds(300); + taskDef.setTimeoutSeconds(600); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.SCHEDULED); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use minimum of pollTimeout, responseTimeout, timeout = 45 seconds (with jitter) + // But capped at workflowOffsetTimeout (30 seconds) + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue( + "Postpone should be between 20-40 seconds (capped at workflow offset)", + result.getSeconds() >= 20); + assertTrue( + "Postpone should be between 20-40 seconds (capped at workflow offset)", + result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ScheduledTaskWithWorkflowTimeout() { + // Workflow with timeout, task without specific timeout + workflowDef.setTimeoutSeconds(180); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.SCHEDULED); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be capped at workflowOffsetTimeout (30 seconds with jitter) + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_MaxPostponeCapping() { + // Test that postpone is capped at maxPostponeDuration + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(1000L); // Very large timeout + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis()); + task.setResponseTimeoutSeconds(1000); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration shortMaxPostpone = Duration.ofSeconds(60); + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, shortMaxPostpone); + + // Should be capped at workflowOffsetTimeout (30 seconds) which is smaller + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_NoTasksUsesDefault() { + // No tasks in workflow + workflow.setTasks(List.of()); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return workflowOffsetTimeout with jitter + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_OnlyTerminalTasks() { + // Only terminal tasks should use default + TaskModel completedTask = new TaskModel(); + completedTask.setTaskType("SIMPLE"); + completedTask.setStatus(TaskModel.Status.COMPLETED); + + TaskModel failedTask = new TaskModel(); + failedTask.setTaskType("SIMPLE"); + failedTask.setStatus(TaskModel.Status.FAILED); + + workflow.setTasks(List.of(completedTask, failedTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return workflowOffsetTimeout with jitter + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_FirstNonTerminalTaskIsUsed() { + // When multiple tasks exist, should use first non-terminal task + TaskModel completedTask = new TaskModel(); + completedTask.setTaskType("SIMPLE"); + completedTask.setStatus(TaskModel.Status.COMPLETED); + + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(System.currentTimeMillis() + 45000); // 45 seconds + + TaskModel scheduledTask = new TaskModel(); + scheduledTask.setTaskType("SIMPLE"); + scheduledTask.setStatus(TaskModel.Status.SCHEDULED); + + workflow.setTasks(List.of(completedTask, waitTask, scheduledTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use the WAIT task (first non-terminal), capped at workflowOffsetTimeout + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); + } + + @Test + public void testApplyJitter_SmallDuration() { + // Small durations (<=3) should not have jitter + long result = ExecutorUtils.applyJitter(3); + assertEquals("Small duration should not have jitter", 3, result); + + result = ExecutorUtils.applyJitter(2); + assertEquals("Small duration should not have jitter", 2, result); + + result = ExecutorUtils.applyJitter(1); + assertEquals("Small duration should not have jitter", 1, result); + } + + @Test + public void testApplyJitter_LargeDuration() { + // Test jitter is within bounds for larger duration + long duration = 90; + long result = ExecutorUtils.applyJitter(duration); + + // Jitter should be ±1/3, so 90 ± 30 = [60, 120] + assertTrue("Jitter should be within bounds", result >= 60); + assertTrue("Jitter should be within bounds", result <= 120); + } + + @Test + public void testApplyJitter_MultipleCallsVary() { + // Verify jitter actually produces different values (randomness check) + long duration = 60; + boolean foundDifferent = false; + + for (int i = 0; i < 100; i++) { + long result = ExecutorUtils.applyJitter(duration); + if (result != duration) { + foundDifferent = true; + break; + } + } + + assertTrue( + "Jitter should produce varied results over multiple calls (within range)", + foundDifferent); + } + + @Test + public void testComputePostpone_PastWaitTimeout() { + // WAIT task with timeout in the past should return 0 + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(System.currentTimeMillis() - 10000); // 10 seconds in past + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use workflowOffsetTimeout since calculated postpone is 0 + assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); + assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ResponseTimeoutAlreadyExceeded() { + // Task that has already exceeded its response timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(30L); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis() - 60000); // Started 60 seconds ago + task.setResponseTimeoutSeconds(30); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use default workflowOffsetTimeout + assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); + assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); + } +} diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java deleted file mode 100644 index e8689dc0f5..0000000000 --- a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Copyright 2022 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.reconciliation; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; - -import org.junit.Before; -import org.junit.Test; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.common.metadata.workflow.WorkflowDef; -import com.netflix.conductor.core.config.ConductorProperties; -import com.netflix.conductor.core.dal.ExecutionDAOFacade; -import com.netflix.conductor.core.execution.WorkflowExecutor; -import com.netflix.conductor.dao.QueueDAO; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.TaskModel.Status; -import com.netflix.conductor.model.WorkflowModel; -import com.netflix.conductor.service.ExecutionLockService; - -import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; - -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class TestWorkflowSweeper { - - private ConductorProperties properties; - private WorkflowExecutor workflowExecutor; - private WorkflowRepairService workflowRepairService; - private QueueDAO queueDAO; - private ExecutionDAOFacade executionDAOFacade; - private WorkflowSweeper workflowSweeper; - private ExecutionLockService executionLockService; - - private int defaultPostPoneOffSetSeconds = 1800; - private int defaulMmaxPostponeDurationSeconds = 2000000; - - @Before - public void setUp() { - properties = mock(ConductorProperties.class); - workflowExecutor = mock(WorkflowExecutor.class); - queueDAO = mock(QueueDAO.class); - workflowRepairService = mock(WorkflowRepairService.class); - executionDAOFacade = mock(ExecutionDAOFacade.class); - executionLockService = mock(ExecutionLockService.class); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper = - new WorkflowSweeper( - workflowExecutor, - Optional.of(workflowRepairService), - properties, - queueDAO, - executionDAOFacade, - executionLockService); - } - - @Test - public void testPostponeDurationForHumanTaskType() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_HUMAN); - taskModel.setStatus(Status.IN_PROGRESS); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForWaitTaskType() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); - taskModel.setStatus(Status.IN_PROGRESS); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForWaitTaskTypeWithLongWaitTime() { - long waitTimeout = 65845; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); - taskModel.setStatus(Status.IN_PROGRESS); - taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); - } - - @Test - public void testPostponeDurationForWaitTaskTypeWithLessOneSecondWaitTime() { - long waitTimeout = 180; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); - taskModel.setStatus(Status.IN_PROGRESS); - taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); - } - - @Test - public void testPostponeDurationForWaitTaskTypeWithZeroWaitTime() { - long waitTimeout = 0; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); - taskModel.setStatus(Status.IN_PROGRESS); - taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); - } - - @Test - public void testPostponeDurationForTaskInProgress() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); - taskModel.setStatus(Status.IN_PROGRESS); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForTaskInProgressWithResponseTimeoutSet() { - long responseTimeout = 200; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); - taskModel.setStatus(Status.IN_PROGRESS); - taskModel.setResponseTimeoutSeconds(responseTimeout); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (responseTimeout + 1) * 1000); - } - - @Test - public void - testPostponeDurationForTaskInProgressWithResponseTimeoutSetLongerThanMaxPostponeDuration() { - long responseTimeout = defaulMmaxPostponeDurationSeconds + 1; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); - taskModel.setStatus(Status.IN_PROGRESS); - taskModel.setResponseTimeoutSeconds(responseTimeout); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaulMmaxPostponeDurationSeconds * 1000L); - } - - @Test - public void testPostponeDurationForTaskInScheduled() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - WorkflowDef workflowDef = new WorkflowDef(); - workflowModel.setWorkflowDefinition(workflowDef); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); - taskModel.setStatus(Status.SCHEDULED); - taskModel.setReferenceTaskName("task1"); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSet() { - long workflowTimeout = 1800; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - WorkflowDef workflowDef = new WorkflowDef(); - workflowDef.setTimeoutSeconds(workflowTimeout); - workflowModel.setWorkflowDefinition(workflowDef); - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); - taskModel.setStatus(Status.SCHEDULED); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); - } - - @Test - public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSetAndNoPollTimeout() { - long workflowTimeout = 1800; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - WorkflowDef workflowDef = new WorkflowDef(); - workflowDef.setTimeoutSeconds(workflowTimeout); - workflowModel.setWorkflowDefinition(workflowDef); - TaskDef taskDef = new TaskDef(); - TaskModel taskModel = mock(TaskModel.class); - workflowModel.setTasks(List.of(taskModel)); - when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); - } - - @Test - public void testPostponeDurationForTaskInScheduledWithNoWorkflowTimeoutSetAndNoPollTimeout() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - WorkflowDef workflowDef = new WorkflowDef(); - workflowModel.setWorkflowDefinition(workflowDef); - TaskDef taskDef = new TaskDef(); - TaskModel taskModel = mock(TaskModel.class); - workflowModel.setTasks(List.of(taskModel)); - when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForTaskInScheduledWithNoPollTimeoutSet() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskDef taskDef = new TaskDef(); - WorkflowDef workflowDef = new WorkflowDef(); - workflowModel.setWorkflowDefinition(workflowDef); - TaskModel taskModel = mock(TaskModel.class); - workflowModel.setTasks(List.of(taskModel)); - when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); - when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, - workflowModel.getWorkflowId(), - defaultPostPoneOffSetSeconds * 1000); - } - - @Test - public void testPostponeDurationForTaskInScheduledWithPollTimeoutSet() { - int pollTimeout = 200; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskDef taskDef = new TaskDef(); - taskDef.setPollTimeoutSeconds(pollTimeout); - TaskModel taskModel = mock(TaskModel.class); - ; - workflowModel.setTasks(List.of(taskModel)); - when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); - when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (pollTimeout + 1) * 1000); - } - - @Test - public void testPostponeDurationChoosesMinimumAcrossTasks() { - long responseTimeout = 500; - int pollTimeout = 120; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskModel inProgressTask = new TaskModel(); - inProgressTask.setTaskId("task1"); - inProgressTask.setTaskType(TaskType.TASK_TYPE_SIMPLE); - inProgressTask.setStatus(Status.IN_PROGRESS); - inProgressTask.setResponseTimeoutSeconds(responseTimeout); - TaskDef taskDef = new TaskDef(); - taskDef.setPollTimeoutSeconds(pollTimeout); - TaskModel scheduledTask = mock(TaskModel.class); - when(scheduledTask.getStatus()).thenReturn(Status.SCHEDULED); - when(scheduledTask.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - workflowModel.setTasks(List.of(inProgressTask, scheduledTask)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), (pollTimeout + 1) * 1000L); - } - - @Test - public void testPostponeDurationForScheduledTaskCappedByMaxPostpone() { - int pollTimeout = 1000; - int maxPostponeSeconds = 100; - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - TaskDef taskDef = new TaskDef(); - taskDef.setPollTimeoutSeconds(pollTimeout); - TaskModel taskModel = mock(TaskModel.class); - when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); - when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); - workflowModel.setTasks(List.of(taskModel)); - when(properties.getWorkflowOffsetTimeout()) - .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); - when(properties.getMaxPostponeDurationSeconds()) - .thenReturn(Duration.ofSeconds(maxPostponeSeconds)); - - workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - - verify(queueDAO) - .setUnackTimeout( - DECIDER_QUEUE, workflowModel.getWorkflowId(), maxPostponeSeconds * 1000L); - } - - @Test - public void testWorkflowOffsetJitter() { - long offset = 45; - for (int i = 0; i < 10; i++) { - long offsetWithJitter = workflowSweeper.workflowOffsetWithJitter(offset); - assertTrue(offsetWithJitter >= 30); - assertTrue(offsetWithJitter <= 60); - } - } -} diff --git a/gradle.properties b/gradle.properties index c6fc37252f..2cd0ce4481 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ org.gradle.daemon=true org.gradle.jvmargs=-Xmx4g -org.gradle.caching=true \ No newline at end of file +org.gradle.caching=true +version=3.21.19-SNAPSHOT From 8699b91a4351eafbc9e551dbba2e170f81a1efec Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 22:09:58 +0530 Subject: [PATCH 06/39] revert --- gradle.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 2cd0ce4481..c6fc37252f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,3 @@ org.gradle.daemon=true org.gradle.jvmargs=-Xmx4g -org.gradle.caching=true -version=3.21.19-SNAPSHOT +org.gradle.caching=true \ No newline at end of file From a03f4ffd5693d5408d2a3a6695276e8931bf9875 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 22:17:09 +0530 Subject: [PATCH 07/39] Add tests for ExecutorUtils.computePostpone - Add TestExecutorUtils.java with 15 unit tests for postpone logic - Tests validate HUMAN task handling, max postpone capping, and jitter --- .../conductor/core/execution/TestExecutorUtils.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java index d35d162d76..226aadf0d9 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java @@ -132,14 +132,13 @@ public void testComputePostpone_InProgressTaskWithResponseTimeout() { Duration result = ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout (30s) + // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout + // (30s) // Then jitter: 30 ± 10 = [20, 40] seconds assertTrue( - "Postpone should be capped at workflow offset (20-40s)", - result.getSeconds() >= 20); + "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() >= 20); assertTrue( - "Postpone should be capped at workflow offset (20-40s)", - result.getSeconds() <= 40); + "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() <= 40); } @Test From ee3596d53fa0c2a0e37c1dd896a8e1cf9e0fb0df Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 22:24:26 +0530 Subject: [PATCH 08/39] Fix TestWorkflowSweeper for new WorkflowSweeper API - Rewrite tests to use new WorkflowSweeper constructor signature - Update tests to test sweep() method instead of removed unack() method - Tests now verify workflow sweep behavior: terminal workflow removal, lock handling, task repair, subworkflow handling --- .../reconciliation/TestWorkflowSweeper.java | 226 ++++++++++++++++++ .../src/main/resources/application.properties | 8 +- 2 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java new file mode 100644 index 0000000000..fb2f07d6dd --- /dev/null +++ b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java @@ -0,0 +1,226 @@ +/* + * Copyright 2022 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.core.reconciliation; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executor; + +import org.junit.Before; +import org.junit.Test; + +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.core.config.ConductorProperties; +import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; +import com.netflix.conductor.dao.ExecutionDAO; +import com.netflix.conductor.dao.QueueDAO; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.TaskModel.Status; +import com.netflix.conductor.model.WorkflowModel; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class TestWorkflowSweeper { + + private ConductorProperties properties; + private SweeperProperties sweeperProperties; + private WorkflowExecutor workflowExecutor; + private QueueDAO queueDAO; + private ExecutionDAO executionDAO; + private WorkflowSweeper workflowSweeper; + private SystemTaskRegistry systemTaskRegistry; + private ObjectMapper objectMapper; + private Executor executor; + + @Before + public void setUp() { + properties = mock(ConductorProperties.class); + sweeperProperties = mock(SweeperProperties.class); + workflowExecutor = mock(WorkflowExecutor.class); + queueDAO = mock(QueueDAO.class); + executionDAO = mock(ExecutionDAO.class); + systemTaskRegistry = mock(SystemTaskRegistry.class); + objectMapper = new ObjectMapper(); + executor = mock(Executor.class); + + when(properties.getMaxPostponeDurationSeconds()).thenReturn(Duration.ofSeconds(2000000)); + when(properties.getSweeperThreadCount()).thenReturn(1); + when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(1800)); + when(properties.getLockLeaseTime()).thenReturn(Duration.ofSeconds(60)); + when(sweeperProperties.getSweepBatchSize()).thenReturn(10); + when(sweeperProperties.getQueuePopTimeout()).thenReturn(100); + + workflowSweeper = + new WorkflowSweeper( + executor, + queueDAO, + workflowExecutor, + executionDAO, + properties, + sweeperProperties, + systemTaskRegistry, + objectMapper); + } + + @Test + public void testSweepTerminalWorkflow() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + workflowModel.setStatus(WorkflowModel.Status.COMPLETED); + + when(workflowExecutor.getWorkflow("1", true)).thenReturn(workflowModel); + + workflowSweeper.sweep("1"); + + verify(queueDAO).remove(DECIDER_QUEUE, "1"); + verify(workflowExecutor, never()).decideWithLock(any(WorkflowModel.class)); + } + + @Test + public void testSweepNullWorkflow() { + when(workflowExecutor.getWorkflow("1", true)).thenReturn(null); + + workflowSweeper.sweep("1"); + + verify(queueDAO).remove(DECIDER_QUEUE, "1"); + } + + @Test + public void testSweepRunningWorkflow() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("2"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("2", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + + workflowSweeper.sweep("2"); + + verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); + } + + @Test + public void testSweepCannotGetLock() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("3"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("3", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(null); + + workflowSweeper.sweep("3"); + + verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); + verify(queueDAO).push(eq(DECIDER_QUEUE), eq("3"), eq(0), anyLong()); + } + + @Test + public void testSweepRepairsScheduledTask() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("4"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.SCHEDULED); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("4", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); + when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); + + workflowSweeper.sweep("4"); + + verify(queueDAO).push(anyString(), eq("task1"), anyLong()); + } + + @Test + public void testSweepDoesNotRepairTaskAlreadyInQueue() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("5"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.SCHEDULED); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("5", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); + when(queueDAO.containsMessage(anyString(), eq("task1"))).thenReturn(true); + + workflowSweeper.sweep("5"); + + verify(queueDAO, never()).push(anyString(), eq("task1"), anyLong()); + } + + @Test + public void testSweepSubWorkflowTask() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("6"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel subWorkflowTask = new TaskModel(); + subWorkflowTask.setTaskId("task1"); + subWorkflowTask.setReferenceTaskName("task1"); + subWorkflowTask.setTaskType(TaskType.TASK_TYPE_SUB_WORKFLOW); + subWorkflowTask.setStatus(Status.IN_PROGRESS); + subWorkflowTask.setSubWorkflowId("sub-wf-1"); + workflowModel.setTasks(List.of(subWorkflowTask)); + + WorkflowModel subWorkflow = new WorkflowModel(); + subWorkflow.setWorkflowId("sub-wf-1"); + subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); + + when(workflowExecutor.getWorkflow("6", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(executionDAO.getWorkflow("sub-wf-1", false)).thenReturn(subWorkflow); + + workflowSweeper.sweep("6"); + + verify(executionDAO).updateTask(subWorkflowTask); + } +} diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 960508f05f..9030aed920 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -15,7 +15,13 @@ spring.application.name=conductor springdoc.api-docs.path=/api-docs loadSample=true -conductor.db.type=memory +conductor.db.type=postgres + +spring.datasource.url=jdbc:postgresql://localhost:5432/conductor +spring.datasource.username=mananbhatt +spring.datasource.password= +spring.datasource.hikari.maximum-pool-size=8 +spring.datasource.hikari.auto-commit=false conductor.queue.type=redis_standalone # Redis cluster connection in SSL mode From d1aee1080a8e9066cff1e15f87bb2bec3dd207ec Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 22:26:46 +0530 Subject: [PATCH 09/39] revert --- server/src/main/resources/application.properties | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 9030aed920..960508f05f 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -15,13 +15,7 @@ spring.application.name=conductor springdoc.api-docs.path=/api-docs loadSample=true -conductor.db.type=postgres - -spring.datasource.url=jdbc:postgresql://localhost:5432/conductor -spring.datasource.username=mananbhatt -spring.datasource.password= -spring.datasource.hikari.maximum-pool-size=8 -spring.datasource.hikari.auto-commit=false +conductor.db.type=memory conductor.queue.type=redis_standalone # Redis cluster connection in SSL mode From 9e8e7b3387d64614d2e4332dbbf254966824ddac Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Wed, 28 Jan 2026 23:12:57 +0530 Subject: [PATCH 10/39] spotless. --- .../conductor/core/reconciliation/TestWorkflowSweeper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java index fb2f07d6dd..44be365b46 100644 --- a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java +++ b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java @@ -34,7 +34,6 @@ import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; From 5d5ade92c826f387748ee8ff5bfe1c2371f70fa6 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 00:11:54 +0530 Subject: [PATCH 11/39] Move Orkes WorkflowSweeper from org.conductoross to com.netflix.conductor OSS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replaced complex OSS WorkflowSweeper with simple Orkes version from org.conductoross - Added SweeperProperties to OSS reconciliation package - Removed ExecutorUtils with complex computePostpone() logic - Simplified WorkflowExecutorOps.decide() to use basic workflowOffsetTimeout - Updated WorkflowReconciler to use sweep() instead of sweepAsync() - Simplified TestWorkflowSweeper (removed tests for removed methods) - Deleted TestExecutorUtils The Orkes WorkflowSweeper now in OSS has: ✅ Simple pollAndSweep() with continuous loop ✅ Task repair logic (verifyAndRepairTask) ✅ Subworkflow repair logic ✅ Parent workflow queue management ✅ NO @Async annotation ✅ NO sweepAsync() method ✅ NO complex postpone calculations --- .../core/execution/ExecutorUtils.java | 123 ------ .../core/execution/WorkflowExecutorOps.java | 13 +- .../reconciliation/WorkflowReconciler.java | 9 +- .../core/reconciliation/WorkflowSweeper.java | 8 - .../core/execution/TestExecutorUtils.java | 374 ------------------ .../reconciliation/TestWorkflowSweeper.java | 179 +-------- 6 files changed, 20 insertions(+), 686 deletions(-) delete mode 100644 core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java delete mode 100644 core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java diff --git a/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java deleted file mode 100644 index 2720044ca2..0000000000 --- a/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.execution; - -import java.time.Duration; -import java.util.Random; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import lombok.extern.slf4j.Slf4j; - -import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; - -@Slf4j -public class ExecutorUtils { - - public static Duration computePostpone( - WorkflowModel workflowModel, - Duration workflowOffsetTimeout, - Duration maxPostponeDuration) { - long currentTimeMillis = System.currentTimeMillis(); - long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); - long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); - - long postponeDurationSeconds = 0; - for (TaskModel taskModel : workflowModel.getTasks()) { - TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); - if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { - if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { - if (taskModel.getWaitTimeout() != 0) { - long deltaInSeconds = - (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; - postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; - } - } else if (taskModel.getTaskType().equals("HUMAN")) { - // HUMAN tasks always use the default workflowOffsetTimeout - // This matches the old sweeper behavior - postponeDurationSeconds = workflowOffsetTimeoutSeconds; - } else { - // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... - // not sure but keeping it this way just in case. - long responseTimeoutSeconds = - taskDef != null - ? taskDef.getResponseTimeoutSeconds() - : taskModel.getResponseTimeoutSeconds(); - if (responseTimeoutSeconds != 0) { - long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; - if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { - postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; - } - } - } - break; - } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { - if (taskDef != null) { - postponeDurationSeconds = getMinTimeout(taskDef); - } - - if (postponeDurationSeconds <= 0) { - if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { - postponeDurationSeconds = - workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; - } - } - break; - } - } - log.trace( - "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", - postponeDurationSeconds, - workflowOffsetTimeoutSeconds, - workflowModel.getWorkflowId()); - - // Cap postpone duration to the configured maximum (matches old sweeper behavior) - if (postponeDurationSeconds > maxPostponeDurationSeconds) { - postponeDurationSeconds = maxPostponeDurationSeconds; - } - - if (postponeDurationSeconds > 0) { - long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); - return Duration.ofSeconds(applyJitter(capped)); - } - return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); - } - - private static long getMinTimeout(TaskDef taskDef) { - long pollTimeoutSeconds = - taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; - return Math.min( - taskDef.getTimeoutSeconds(), - Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); - } - - /** - * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the - * duration. - * - *

For example, if duration is 45 seconds, returns value between [30-60] seconds. - * - * @param durationSeconds postpone duration in seconds - * @return duration with jitter applied - */ - public static long applyJitter(long durationSeconds) { - if (durationSeconds <= 3) { - return durationSeconds; // Too small for jitter - } - long range = durationSeconds / 3; - long jitter = new Random().nextInt((int) (2 * range + 1)) - range; - return durationSeconds + jitter; - } -} diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java index 01025f8a0e..e2288b555b 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java @@ -54,7 +54,6 @@ import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; -import static com.netflix.conductor.core.execution.ExecutorUtils.computePostpone; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; @@ -1146,18 +1145,10 @@ private WorkflowModel decide(WorkflowModel workflow) { executionDAOFacade.updateWorkflow(workflow); } - Duration timeout = properties.getWorkflowOffsetTimeout(); if (!workflow.getStatus().isTerminal()) { - Duration updatedOffset = - computePostpone( - workflow, timeout, properties.getMaxPostponeDurationSeconds()); - // Always reschedule non-terminal workflows to ensure they remain in the queue - LOGGER.debug( - "Rescheduling workflow {} to decider queue with postpone duration {} seconds", - workflow.getWorkflowId(), - updatedOffset.getSeconds()); + Duration timeout = properties.getWorkflowOffsetTimeout(); queueDAO.setUnackTimeout( - DECIDER_QUEUE, workflow.getWorkflowId(), updatedOffset.getSeconds() * 1000); + DECIDER_QUEUE, workflow.getWorkflowId(), timeout.getSeconds() * 1000); } return workflow; diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java index 9e7e6e43f4..ecb98a54b9 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java @@ -13,7 +13,6 @@ package com.netflix.conductor.core.reconciliation; import java.util.List; -import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,12 +71,8 @@ public void pollAndSweep() { List workflowIds = queueDAO.pop(DECIDER_QUEUE, sweeperThreadCount, sweeperWorkflowPollTimeout); if (workflowIds != null) { - // wait for all workflow ids to be "swept" - CompletableFuture.allOf( - workflowIds.stream() - .map(workflowSweeper::sweepAsync) - .toArray(CompletableFuture[]::new)) - .get(); + // Process all workflow ids synchronously + workflowIds.forEach(workflowSweeper::sweep); LOGGER.debug( "Sweeper processed {} from the decider queue", String.join(",", workflowIds)); diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index 55294894c1..f2bf16fdb0 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -15,7 +15,6 @@ import java.time.Clock; import java.time.Duration; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; @@ -24,7 +23,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; @@ -147,12 +145,6 @@ private void pollAndSweep() { } } - @Async(SWEEPER_EXECUTOR_NAME) - public CompletableFuture sweepAsync(String workflowId) { - sweep(workflowId); - return CompletableFuture.completedFuture(null); - } - public void sweep(String workflowId) { log.info("Running sweeper for workflow {}", workflowId); diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java deleted file mode 100644 index 226aadf0d9..0000000000 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.execution; - -import java.time.Duration; -import java.util.List; - -import org.junit.Before; -import org.junit.Test; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.common.metadata.workflow.WorkflowDef; -import com.netflix.conductor.common.metadata.workflow.WorkflowTask; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import static org.junit.Assert.*; - -/** - * Unit tests for ExecutorUtils.computePostpone() to ensure correct postpone duration calculation - * for workflow rescheduling in the decider queue. - */ -public class TestExecutorUtils { - - private WorkflowModel workflow; - private WorkflowDef workflowDef; - private Duration workflowOffsetTimeout; - private Duration maxPostponeDuration; - - @Before - public void setup() { - workflowDef = new WorkflowDef(); - workflowDef.setName("test_workflow"); - workflowDef.setTimeoutSeconds(0); - - workflow = new WorkflowModel(); - workflow.setWorkflowDefinition(workflowDef); - workflow.setWorkflowId("test-workflow-id"); - - workflowOffsetTimeout = Duration.ofSeconds(30); - maxPostponeDuration = Duration.ofSeconds(300); - } - - @Test - public void testComputePostpone_WaitTaskWithTimeout() { - // Create a WAIT task with specific wait timeout - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - long futureTime = System.currentTimeMillis() + 60000; // 60 seconds in future - waitTask.setWaitTimeout(futureTime); - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be capped at workflowOffsetTimeout (30 seconds) with jitter - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_WaitTaskWithZeroTimeout() { - // Create a WAIT task without specific timeout - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(0); - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return the default workflow offset timeout (30 seconds, with jitter) - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_HumanTask() { - // HUMAN tasks should always use default workflowOffsetTimeout - TaskModel humanTask = new TaskModel(); - humanTask.setTaskType("HUMAN"); - humanTask.setStatus(TaskModel.Status.IN_PROGRESS); - humanTask.setStartTime(System.currentTimeMillis()); - - workflow.setTasks(List.of(humanTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return the default workflow offset timeout (30 seconds, with jitter) - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_InProgressTaskWithResponseTimeout() { - // Task with response timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(120L); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis() - 10000); // Started 10 seconds ago - task.setResponseTimeoutSeconds(120); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout - // (30s) - // Then jitter: 30 ± 10 = [20, 40] seconds - assertTrue( - "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() >= 20); - assertTrue( - "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ScheduledTaskWithPollTimeout() { - // Task definition with poll timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setPollTimeoutSeconds(45); - taskDef.setResponseTimeoutSeconds(300); - taskDef.setTimeoutSeconds(600); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.SCHEDULED); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use minimum of pollTimeout, responseTimeout, timeout = 45 seconds (with jitter) - // But capped at workflowOffsetTimeout (30 seconds) - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue( - "Postpone should be between 20-40 seconds (capped at workflow offset)", - result.getSeconds() >= 20); - assertTrue( - "Postpone should be between 20-40 seconds (capped at workflow offset)", - result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ScheduledTaskWithWorkflowTimeout() { - // Workflow with timeout, task without specific timeout - workflowDef.setTimeoutSeconds(180); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.SCHEDULED); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be capped at workflowOffsetTimeout (30 seconds with jitter) - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_MaxPostponeCapping() { - // Test that postpone is capped at maxPostponeDuration - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(1000L); // Very large timeout - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis()); - task.setResponseTimeoutSeconds(1000); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration shortMaxPostpone = Duration.ofSeconds(60); - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, shortMaxPostpone); - - // Should be capped at workflowOffsetTimeout (30 seconds) which is smaller - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_NoTasksUsesDefault() { - // No tasks in workflow - workflow.setTasks(List.of()); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return workflowOffsetTimeout with jitter - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_OnlyTerminalTasks() { - // Only terminal tasks should use default - TaskModel completedTask = new TaskModel(); - completedTask.setTaskType("SIMPLE"); - completedTask.setStatus(TaskModel.Status.COMPLETED); - - TaskModel failedTask = new TaskModel(); - failedTask.setTaskType("SIMPLE"); - failedTask.setStatus(TaskModel.Status.FAILED); - - workflow.setTasks(List.of(completedTask, failedTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return workflowOffsetTimeout with jitter - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_FirstNonTerminalTaskIsUsed() { - // When multiple tasks exist, should use first non-terminal task - TaskModel completedTask = new TaskModel(); - completedTask.setTaskType("SIMPLE"); - completedTask.setStatus(TaskModel.Status.COMPLETED); - - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(System.currentTimeMillis() + 45000); // 45 seconds - - TaskModel scheduledTask = new TaskModel(); - scheduledTask.setTaskType("SIMPLE"); - scheduledTask.setStatus(TaskModel.Status.SCHEDULED); - - workflow.setTasks(List.of(completedTask, waitTask, scheduledTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use the WAIT task (first non-terminal), capped at workflowOffsetTimeout - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); - } - - @Test - public void testApplyJitter_SmallDuration() { - // Small durations (<=3) should not have jitter - long result = ExecutorUtils.applyJitter(3); - assertEquals("Small duration should not have jitter", 3, result); - - result = ExecutorUtils.applyJitter(2); - assertEquals("Small duration should not have jitter", 2, result); - - result = ExecutorUtils.applyJitter(1); - assertEquals("Small duration should not have jitter", 1, result); - } - - @Test - public void testApplyJitter_LargeDuration() { - // Test jitter is within bounds for larger duration - long duration = 90; - long result = ExecutorUtils.applyJitter(duration); - - // Jitter should be ±1/3, so 90 ± 30 = [60, 120] - assertTrue("Jitter should be within bounds", result >= 60); - assertTrue("Jitter should be within bounds", result <= 120); - } - - @Test - public void testApplyJitter_MultipleCallsVary() { - // Verify jitter actually produces different values (randomness check) - long duration = 60; - boolean foundDifferent = false; - - for (int i = 0; i < 100; i++) { - long result = ExecutorUtils.applyJitter(duration); - if (result != duration) { - foundDifferent = true; - break; - } - } - - assertTrue( - "Jitter should produce varied results over multiple calls (within range)", - foundDifferent); - } - - @Test - public void testComputePostpone_PastWaitTimeout() { - // WAIT task with timeout in the past should return 0 - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(System.currentTimeMillis() - 10000); // 10 seconds in past - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use workflowOffsetTimeout since calculated postpone is 0 - assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); - assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ResponseTimeoutAlreadyExceeded() { - // Task that has already exceeded its response timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(30L); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis() - 60000); // Started 60 seconds ago - task.setResponseTimeoutSeconds(30); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use default workflowOffsetTimeout - assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); - assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); - } -} diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java index 44be365b46..7fb6599671 100644 --- a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java +++ b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java @@ -13,68 +13,57 @@ package com.netflix.conductor.core.reconciliation; import java.time.Duration; -import java.util.List; import java.util.concurrent.Executor; import org.junit.Before; import org.junit.Test; -import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.TaskModel.Status; -import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; -import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestWorkflowSweeper { private ConductorProperties properties; - private SweeperProperties sweeperProperties; private WorkflowExecutor workflowExecutor; private QueueDAO queueDAO; private ExecutionDAO executionDAO; - private WorkflowSweeper workflowSweeper; + private SweeperProperties sweeperProperties; private SystemTaskRegistry systemTaskRegistry; private ObjectMapper objectMapper; - private Executor executor; + private Executor sweeperExecutor; @Before public void setUp() { properties = mock(ConductorProperties.class); - sweeperProperties = mock(SweeperProperties.class); workflowExecutor = mock(WorkflowExecutor.class); queueDAO = mock(QueueDAO.class); executionDAO = mock(ExecutionDAO.class); + sweeperProperties = mock(SweeperProperties.class); systemTaskRegistry = mock(SystemTaskRegistry.class); - objectMapper = new ObjectMapper(); - executor = mock(Executor.class); + objectMapper = mock(ObjectMapper.class); + sweeperExecutor = mock(Executor.class); - when(properties.getMaxPostponeDurationSeconds()).thenReturn(Duration.ofSeconds(2000000)); - when(properties.getSweeperThreadCount()).thenReturn(1); - when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(1800)); + when(properties.getSweeperThreadCount()).thenReturn(0); // Don't start threads in tests + when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(30)); when(properties.getLockLeaseTime()).thenReturn(Duration.ofSeconds(60)); - when(sweeperProperties.getSweepBatchSize()).thenReturn(10); + when(sweeperProperties.getSweepBatchSize()).thenReturn(2); when(sweeperProperties.getQueuePopTimeout()).thenReturn(100); + } - workflowSweeper = + @Test + public void testWorkflowSweeperInitialization() { + // Test that WorkflowSweeper can be instantiated + WorkflowSweeper workflowSweeper = new WorkflowSweeper( - executor, + sweeperExecutor, queueDAO, workflowExecutor, executionDAO, @@ -82,144 +71,8 @@ public void setUp() { sweeperProperties, systemTaskRegistry, objectMapper); - } - - @Test - public void testSweepTerminalWorkflow() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("1"); - workflowModel.setStatus(WorkflowModel.Status.COMPLETED); - - when(workflowExecutor.getWorkflow("1", true)).thenReturn(workflowModel); - - workflowSweeper.sweep("1"); - - verify(queueDAO).remove(DECIDER_QUEUE, "1"); - verify(workflowExecutor, never()).decideWithLock(any(WorkflowModel.class)); - } - - @Test - public void testSweepNullWorkflow() { - when(workflowExecutor.getWorkflow("1", true)).thenReturn(null); - - workflowSweeper.sweep("1"); - - verify(queueDAO).remove(DECIDER_QUEUE, "1"); - } - - @Test - public void testSweepRunningWorkflow() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("2"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); - taskModel.setStatus(Status.IN_PROGRESS); - workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("2", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - - workflowSweeper.sweep("2"); - - verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); - } - - @Test - public void testSweepCannotGetLock() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("3"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); - taskModel.setStatus(Status.IN_PROGRESS); - workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("3", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(null); - - workflowSweeper.sweep("3"); - - verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); - verify(queueDAO).push(eq(DECIDER_QUEUE), eq("3"), eq(0), anyLong()); - } - - @Test - public void testSweepRepairsScheduledTask() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("4"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); - taskModel.setStatus(Status.SCHEDULED); - workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("4", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); - when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); - - workflowSweeper.sweep("4"); - - verify(queueDAO).push(anyString(), eq("task1"), anyLong()); - } - - @Test - public void testSweepDoesNotRepairTaskAlreadyInQueue() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("5"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - - TaskModel taskModel = new TaskModel(); - taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); - taskModel.setStatus(Status.SCHEDULED); - workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("5", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); - when(queueDAO.containsMessage(anyString(), eq("task1"))).thenReturn(true); - - workflowSweeper.sweep("5"); - - verify(queueDAO, never()).push(anyString(), eq("task1"), anyLong()); - } - - @Test - public void testSweepSubWorkflowTask() { - WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("6"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - - TaskModel subWorkflowTask = new TaskModel(); - subWorkflowTask.setTaskId("task1"); - subWorkflowTask.setReferenceTaskName("task1"); - subWorkflowTask.setTaskType(TaskType.TASK_TYPE_SUB_WORKFLOW); - subWorkflowTask.setStatus(Status.IN_PROGRESS); - subWorkflowTask.setSubWorkflowId("sub-wf-1"); - workflowModel.setTasks(List.of(subWorkflowTask)); - - WorkflowModel subWorkflow = new WorkflowModel(); - subWorkflow.setWorkflowId("sub-wf-1"); - subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); - - when(workflowExecutor.getWorkflow("6", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(executionDAO.getWorkflow("sub-wf-1", false)).thenReturn(subWorkflow); - - workflowSweeper.sweep("6"); - verify(executionDAO).updateTask(subWorkflowTask); + // Basic test to ensure constructor works + // The actual sweep logic is tested via integration tests } } From e174e2c899bcd2c2fbebb9f219b8fda2097f3c49 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 18:50:25 +0530 Subject: [PATCH 12/39] spotless --- .../conductor/core/reconciliation/WorkflowReconciler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java index ecb98a54b9..3667763890 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java @@ -31,8 +31,7 @@ * Periodically polls all running workflows in the system and evaluates them for timeouts and/or * maintain consistency. */ -// Deprecated - and superseeded by new WorkflowSweeper in org.conductoross.conductor.core.execution -// package +// Deprecated - superseded by WorkflowSweeper in com.netflix.conductor.core.reconciliation package @Deprecated(forRemoval = true) @Component @ConditionalOnProperty( From 85783d963dd43046ce1985e3d21299942f3e75ae Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 19:13:47 +0530 Subject: [PATCH 13/39] changes --- .../netflix/conductor/test/base/AbstractSpecification.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 24b9bd3216..beb077a1c5 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -12,7 +12,7 @@ */ package com.netflix.conductor.test.base -import org.conductoross.conductor.core.execution.WorkflowSweeper +import com.netflix.conductor.core.reconciliation.WorkflowSweeper import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.DynamicPropertyRegistry From 58628ee92ef5a6dd74b7a4098b7187023dc7751a Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 19:14:30 +0530 Subject: [PATCH 14/39] Fix: Update WorkflowSweeper import in AbstractSpecification Changed import from: org.conductoross.conductor.core.execution.WorkflowSweeper to: com.netflix.conductor.core.reconciliation.WorkflowSweeper This completes the package migration after moving WorkflowSweeper from org.conductoross to com.netflix.conductor OSS package. Fixes compilation error in test-harness module. Co-Authored-By: Claude Sonnet 4.5 --- .../netflix/conductor/test/base/AbstractSpecification.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index beb077a1c5..fb4856fde6 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -12,7 +12,6 @@ */ package com.netflix.conductor.test.base -import com.netflix.conductor.core.reconciliation.WorkflowSweeper import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.DynamicPropertyRegistry @@ -26,6 +25,7 @@ import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.WorkflowExecutor +import com.netflix.conductor.core.reconciliation.WorkflowSweeper import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.netflix.conductor.test.util.WorkflowTestUtil From bf0f1f7680b287cab8e8f3815ec16056bfb28753 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 21:48:42 +0530 Subject: [PATCH 15/39] revert --- .../core/reconciliation/WorkflowSweeper.java | 72 +++++++------------ local-publish.gradle | 21 ++++++ .../src/main/resources/application.properties | 1 - 3 files changed, 47 insertions(+), 47 deletions(-) create mode 100644 local-publish.gradle diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index f2bf16fdb0..7124d55e5b 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -183,7 +183,30 @@ public void sweep(String workflowId) { // Workflow has not completed and decide did not change the status of the tasks // Every task that is running MUST be in the queue if (tasks.equals(tasksAfterDecide)) { - workflow.getTasks().forEach(this::verifyAndRepairTask); + long now = System.currentTimeMillis(); + workflow.getTasks().stream() + .filter(isTaskRepairable) + .forEach( + task -> { + log.warn( + "Going to repair the task {} / {}, with status {}, workflow = {}, timeout = {}, now-wait = {}", + task.getTaskId(), + task.getReferenceTaskName(), + task.getStatus(), + workflowId, + task.getWaitTimeout(), + (now - task.getWaitTimeout())); + Monitors.recordQueueMessageRepushFromRepairService( + task.getTaskDefName()); + String queueName = QueueUtils.getQueueName(task); + if (!queueDAO.containsMessage(queueName, task.getTaskId())) { + queueDAO.push( + queueName, + task.getTaskId(), + 0, + task.getCallbackAfterSeconds()); + } + }); } // Workflow is in running status, there MUST be at-least one task that is not terminal @@ -195,6 +218,8 @@ public void sweep(String workflowId) { // This can happen in case of the database failures where the task scheduling failed // after the last task was completed // To fix, we reset the executed flag of the last task and re-run decide + // TODO: This MIGHT not be completely accurate, write a test to validate what + // happens if the last task is a JOIN task forceSetLastTaskAsNotExecuted(workflow); workflow = workflowExecutor.decideWithLock(workflow); } @@ -209,51 +234,6 @@ public void sweep(String workflowId) { } } - void verifyAndRepairTask(TaskModel task) { - if (isTaskRepairable.test(task)) { - // Ensure QueueDAO contains this taskId - String taskQueueName = QueueUtils.getQueueName(task); - if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { - queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); - log.info( - "Task {} in workflow {} re-queued for repairs", - task.getTaskId(), - task.getWorkflowInstanceId()); - Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); - } - } - if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) - && task.getStatus() == TaskModel.Status.IN_PROGRESS) { - WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); - if (subWorkflow.getStatus().isTerminal()) { - log.info( - "Repairing sub workflow task {} for sub workflow {} in workflow {}", - task.getTaskId(), - task.getSubWorkflowId(), - task.getWorkflowInstanceId()); - repairSubWorkflowTask(task, subWorkflow); - } - } - } - - private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { - switch (subWorkflow.getStatus()) { - case COMPLETED: - task.setStatus(TaskModel.Status.COMPLETED); - break; - case FAILED: - task.setStatus(TaskModel.Status.FAILED); - break; - case TERMINATED: - task.setStatus(TaskModel.Status.CANCELED); - break; - case TIMED_OUT: - task.setStatus(TaskModel.Status.TIMED_OUT); - break; - } - task.addOutput(subWorkflow.getOutput()); - executionDAO.updateTask(task); - } private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { diff --git a/local-publish.gradle b/local-publish.gradle new file mode 100644 index 0000000000..0404627214 --- /dev/null +++ b/local-publish.gradle @@ -0,0 +1,21 @@ +subprojects { + apply plugin: 'maven-publish' + apply plugin: 'java-library' + + group = 'org.conductoross' + version = '3.20.1-SNAPSHOT' // Set your desired version + + publishing { + publications { + mavenJava(MavenPublication) { + from components.java + groupId = 'org.conductoross' + artifactId = project.name + } + } + + repositories { + mavenLocal() + } + } +} diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 960508f05f..974f06e318 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -181,7 +181,6 @@ management.cloudwatch.metrics.export.namespace=conductor management.azuremonitor.metrics.export.enabled=false management.azuremonitor.metrics.export.instrumentationKey=INSTRUMENTATION_KEY -management.jmx.metrics.export.enabled=false # When enabled logs metrics as info level logs conductor.metrics-logger.enabled=false \ No newline at end of file From 537c4ddd0729bfce4913ebeffc96582cb2596341 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 23:58:24 +0530 Subject: [PATCH 16/39] Add Orkes improvements to WorkflowSweeper (without Orkes-specific code) - Inline task repair with detailed warning logs - Fixed queueDAO.push to use 3-parameter version - Added subworkflow repair logic for terminal subworkflow states - Handles COMPLETED, FAILED, TERMINATED, TIMED_OUT states - All tests pass --- .../core/reconciliation/WorkflowSweeper.java | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index 7124d55e5b..7b67fd0a72 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -201,10 +201,27 @@ public void sweep(String workflowId) { String queueName = QueueUtils.getQueueName(task); if (!queueDAO.containsMessage(queueName, task.getTaskId())) { queueDAO.push( - queueName, + queueName, task.getTaskId(), task.getCallbackAfterSeconds()); + } + }); + + // Repair subworkflow tasks if needed + workflow.getTasks().stream() + .filter( + task -> + task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) + && task.getStatus() == TaskModel.Status.IN_PROGRESS) + .forEach( + task -> { + WorkflowModel subWorkflow = + executionDAO.getWorkflow(task.getSubWorkflowId(), false); + if (subWorkflow.getStatus().isTerminal()) { + log.info( + "Repairing sub workflow task {} for sub workflow {} in workflow {}", task.getTaskId(), - 0, - task.getCallbackAfterSeconds()); + task.getSubWorkflowId(), + task.getWorkflowInstanceId()); + repairSubWorkflowTask(task, subWorkflow); } }); } @@ -235,6 +252,25 @@ public void sweep(String workflowId) { } + private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { + switch (subWorkflow.getStatus()) { + case COMPLETED: + task.setStatus(TaskModel.Status.COMPLETED); + break; + case FAILED: + task.setStatus(TaskModel.Status.FAILED); + break; + case TERMINATED: + task.setStatus(TaskModel.Status.CANCELED); + break; + case TIMED_OUT: + task.setStatus(TaskModel.Status.TIMED_OUT); + break; + } + task.addOutput(subWorkflow.getOutput()); + executionDAO.updateTask(task); + } + private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { TaskModel taskModel = workflow.getTasks().getLast(); From efd376821c1a30b19fa7d9779d5859214001b074 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Thu, 29 Jan 2026 23:38:42 +0530 Subject: [PATCH 17/39] Revert "Move Orkes WorkflowSweeper from org.conductoross to com.netflix.conductor OSS" This reverts commit 7a710b5fead45d9b1de3d74df35c5481231b9877. --- .../core/execution/ExecutorUtils.java | 123 ++++++ .../core/execution/WorkflowExecutorOps.java | 13 +- .../reconciliation/WorkflowReconciler.java | 9 +- .../core/reconciliation/WorkflowSweeper.java | 8 + .../core/execution/TestExecutorUtils.java | 374 ++++++++++++++++++ .../reconciliation/TestWorkflowSweeper.java | 179 ++++++++- 6 files changed, 686 insertions(+), 20 deletions(-) create mode 100644 core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java create mode 100644 core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java diff --git a/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java new file mode 100644 index 0000000000..2720044ca2 --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.core.execution; + +import java.time.Duration; +import java.util.Random; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import lombok.extern.slf4j.Slf4j; + +import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; + +@Slf4j +public class ExecutorUtils { + + public static Duration computePostpone( + WorkflowModel workflowModel, + Duration workflowOffsetTimeout, + Duration maxPostponeDuration) { + long currentTimeMillis = System.currentTimeMillis(); + long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); + long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); + + long postponeDurationSeconds = 0; + for (TaskModel taskModel : workflowModel.getTasks()) { + TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); + if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { + if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { + if (taskModel.getWaitTimeout() != 0) { + long deltaInSeconds = + (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; + postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; + } + } else if (taskModel.getTaskType().equals("HUMAN")) { + // HUMAN tasks always use the default workflowOffsetTimeout + // This matches the old sweeper behavior + postponeDurationSeconds = workflowOffsetTimeoutSeconds; + } else { + // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... + // not sure but keeping it this way just in case. + long responseTimeoutSeconds = + taskDef != null + ? taskDef.getResponseTimeoutSeconds() + : taskModel.getResponseTimeoutSeconds(); + if (responseTimeoutSeconds != 0) { + long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; + if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { + postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; + } + } + } + break; + } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { + if (taskDef != null) { + postponeDurationSeconds = getMinTimeout(taskDef); + } + + if (postponeDurationSeconds <= 0) { + if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { + postponeDurationSeconds = + workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; + } + } + break; + } + } + log.trace( + "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", + postponeDurationSeconds, + workflowOffsetTimeoutSeconds, + workflowModel.getWorkflowId()); + + // Cap postpone duration to the configured maximum (matches old sweeper behavior) + if (postponeDurationSeconds > maxPostponeDurationSeconds) { + postponeDurationSeconds = maxPostponeDurationSeconds; + } + + if (postponeDurationSeconds > 0) { + long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); + return Duration.ofSeconds(applyJitter(capped)); + } + return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); + } + + private static long getMinTimeout(TaskDef taskDef) { + long pollTimeoutSeconds = + taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; + return Math.min( + taskDef.getTimeoutSeconds(), + Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); + } + + /** + * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the + * duration. + * + *

For example, if duration is 45 seconds, returns value between [30-60] seconds. + * + * @param durationSeconds postpone duration in seconds + * @return duration with jitter applied + */ + public static long applyJitter(long durationSeconds) { + if (durationSeconds <= 3) { + return durationSeconds; // Too small for jitter + } + long range = durationSeconds / 3; + long jitter = new Random().nextInt((int) (2 * range + 1)) - range; + return durationSeconds + jitter; + } +} diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java index e2288b555b..01025f8a0e 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java @@ -54,6 +54,7 @@ import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; +import static com.netflix.conductor.core.execution.ExecutorUtils.computePostpone; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; @@ -1145,10 +1146,18 @@ private WorkflowModel decide(WorkflowModel workflow) { executionDAOFacade.updateWorkflow(workflow); } + Duration timeout = properties.getWorkflowOffsetTimeout(); if (!workflow.getStatus().isTerminal()) { - Duration timeout = properties.getWorkflowOffsetTimeout(); + Duration updatedOffset = + computePostpone( + workflow, timeout, properties.getMaxPostponeDurationSeconds()); + // Always reschedule non-terminal workflows to ensure they remain in the queue + LOGGER.debug( + "Rescheduling workflow {} to decider queue with postpone duration {} seconds", + workflow.getWorkflowId(), + updatedOffset.getSeconds()); queueDAO.setUnackTimeout( - DECIDER_QUEUE, workflow.getWorkflowId(), timeout.getSeconds() * 1000); + DECIDER_QUEUE, workflow.getWorkflowId(), updatedOffset.getSeconds() * 1000); } return workflow; diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java index 3667763890..555f445387 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java @@ -13,6 +13,7 @@ package com.netflix.conductor.core.reconciliation; import java.util.List; +import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,8 +71,12 @@ public void pollAndSweep() { List workflowIds = queueDAO.pop(DECIDER_QUEUE, sweeperThreadCount, sweeperWorkflowPollTimeout); if (workflowIds != null) { - // Process all workflow ids synchronously - workflowIds.forEach(workflowSweeper::sweep); + // wait for all workflow ids to be "swept" + CompletableFuture.allOf( + workflowIds.stream() + .map(workflowSweeper::sweepAsync) + .toArray(CompletableFuture[]::new)) + .get(); LOGGER.debug( "Sweeper processed {} from the decider queue", String.join(",", workflowIds)); diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index 7b67fd0a72..ab489b870c 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -15,6 +15,7 @@ import java.time.Clock; import java.time.Duration; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; @@ -23,6 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; @@ -145,6 +147,12 @@ private void pollAndSweep() { } } + @Async(SWEEPER_EXECUTOR_NAME) + public CompletableFuture sweepAsync(String workflowId) { + sweep(workflowId); + return CompletableFuture.completedFuture(null); + } + public void sweep(String workflowId) { log.info("Running sweeper for workflow {}", workflowId); diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java new file mode 100644 index 0000000000..226aadf0d9 --- /dev/null +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java @@ -0,0 +1,374 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.core.execution; + +import java.time.Duration; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import static org.junit.Assert.*; + +/** + * Unit tests for ExecutorUtils.computePostpone() to ensure correct postpone duration calculation + * for workflow rescheduling in the decider queue. + */ +public class TestExecutorUtils { + + private WorkflowModel workflow; + private WorkflowDef workflowDef; + private Duration workflowOffsetTimeout; + private Duration maxPostponeDuration; + + @Before + public void setup() { + workflowDef = new WorkflowDef(); + workflowDef.setName("test_workflow"); + workflowDef.setTimeoutSeconds(0); + + workflow = new WorkflowModel(); + workflow.setWorkflowDefinition(workflowDef); + workflow.setWorkflowId("test-workflow-id"); + + workflowOffsetTimeout = Duration.ofSeconds(30); + maxPostponeDuration = Duration.ofSeconds(300); + } + + @Test + public void testComputePostpone_WaitTaskWithTimeout() { + // Create a WAIT task with specific wait timeout + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + long futureTime = System.currentTimeMillis() + 60000; // 60 seconds in future + waitTask.setWaitTimeout(futureTime); + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be capped at workflowOffsetTimeout (30 seconds) with jitter + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_WaitTaskWithZeroTimeout() { + // Create a WAIT task without specific timeout + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(0); + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return the default workflow offset timeout (30 seconds, with jitter) + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_HumanTask() { + // HUMAN tasks should always use default workflowOffsetTimeout + TaskModel humanTask = new TaskModel(); + humanTask.setTaskType("HUMAN"); + humanTask.setStatus(TaskModel.Status.IN_PROGRESS); + humanTask.setStartTime(System.currentTimeMillis()); + + workflow.setTasks(List.of(humanTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return the default workflow offset timeout (30 seconds, with jitter) + // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_InProgressTaskWithResponseTimeout() { + // Task with response timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(120L); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis() - 10000); // Started 10 seconds ago + task.setResponseTimeoutSeconds(120); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout + // (30s) + // Then jitter: 30 ± 10 = [20, 40] seconds + assertTrue( + "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() >= 20); + assertTrue( + "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ScheduledTaskWithPollTimeout() { + // Task definition with poll timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setPollTimeoutSeconds(45); + taskDef.setResponseTimeoutSeconds(300); + taskDef.setTimeoutSeconds(600); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.SCHEDULED); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use minimum of pollTimeout, responseTimeout, timeout = 45 seconds (with jitter) + // But capped at workflowOffsetTimeout (30 seconds) + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue( + "Postpone should be between 20-40 seconds (capped at workflow offset)", + result.getSeconds() >= 20); + assertTrue( + "Postpone should be between 20-40 seconds (capped at workflow offset)", + result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ScheduledTaskWithWorkflowTimeout() { + // Workflow with timeout, task without specific timeout + workflowDef.setTimeoutSeconds(180); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.SCHEDULED); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should be capped at workflowOffsetTimeout (30 seconds with jitter) + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_MaxPostponeCapping() { + // Test that postpone is capped at maxPostponeDuration + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(1000L); // Very large timeout + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis()); + task.setResponseTimeoutSeconds(1000); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration shortMaxPostpone = Duration.ofSeconds(60); + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, shortMaxPostpone); + + // Should be capped at workflowOffsetTimeout (30 seconds) which is smaller + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_NoTasksUsesDefault() { + // No tasks in workflow + workflow.setTasks(List.of()); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return workflowOffsetTimeout with jitter + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_OnlyTerminalTasks() { + // Only terminal tasks should use default + TaskModel completedTask = new TaskModel(); + completedTask.setTaskType("SIMPLE"); + completedTask.setStatus(TaskModel.Status.COMPLETED); + + TaskModel failedTask = new TaskModel(); + failedTask.setTaskType("SIMPLE"); + failedTask.setStatus(TaskModel.Status.FAILED); + + workflow.setTasks(List.of(completedTask, failedTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should return workflowOffsetTimeout with jitter + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); + assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_FirstNonTerminalTaskIsUsed() { + // When multiple tasks exist, should use first non-terminal task + TaskModel completedTask = new TaskModel(); + completedTask.setTaskType("SIMPLE"); + completedTask.setStatus(TaskModel.Status.COMPLETED); + + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(System.currentTimeMillis() + 45000); // 45 seconds + + TaskModel scheduledTask = new TaskModel(); + scheduledTask.setTaskType("SIMPLE"); + scheduledTask.setStatus(TaskModel.Status.SCHEDULED); + + workflow.setTasks(List.of(completedTask, waitTask, scheduledTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use the WAIT task (first non-terminal), capped at workflowOffsetTimeout + // Jitter: 30 ± 10 = [20, 40] seconds + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); + assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); + } + + @Test + public void testApplyJitter_SmallDuration() { + // Small durations (<=3) should not have jitter + long result = ExecutorUtils.applyJitter(3); + assertEquals("Small duration should not have jitter", 3, result); + + result = ExecutorUtils.applyJitter(2); + assertEquals("Small duration should not have jitter", 2, result); + + result = ExecutorUtils.applyJitter(1); + assertEquals("Small duration should not have jitter", 1, result); + } + + @Test + public void testApplyJitter_LargeDuration() { + // Test jitter is within bounds for larger duration + long duration = 90; + long result = ExecutorUtils.applyJitter(duration); + + // Jitter should be ±1/3, so 90 ± 30 = [60, 120] + assertTrue("Jitter should be within bounds", result >= 60); + assertTrue("Jitter should be within bounds", result <= 120); + } + + @Test + public void testApplyJitter_MultipleCallsVary() { + // Verify jitter actually produces different values (randomness check) + long duration = 60; + boolean foundDifferent = false; + + for (int i = 0; i < 100; i++) { + long result = ExecutorUtils.applyJitter(duration); + if (result != duration) { + foundDifferent = true; + break; + } + } + + assertTrue( + "Jitter should produce varied results over multiple calls (within range)", + foundDifferent); + } + + @Test + public void testComputePostpone_PastWaitTimeout() { + // WAIT task with timeout in the past should return 0 + TaskModel waitTask = new TaskModel(); + waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); + waitTask.setStatus(TaskModel.Status.IN_PROGRESS); + waitTask.setWaitTimeout(System.currentTimeMillis() - 10000); // 10 seconds in past + + workflow.setTasks(List.of(waitTask)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use workflowOffsetTimeout since calculated postpone is 0 + assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); + assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); + } + + @Test + public void testComputePostpone_ResponseTimeoutAlreadyExceeded() { + // Task that has already exceeded its response timeout + TaskDef taskDef = new TaskDef(); + taskDef.setName("test_task"); + taskDef.setResponseTimeoutSeconds(30L); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setTaskDefinition(taskDef); + + TaskModel task = new TaskModel(); + task.setTaskType("SIMPLE"); + task.setStatus(TaskModel.Status.IN_PROGRESS); + task.setStartTime(System.currentTimeMillis() - 60000); // Started 60 seconds ago + task.setResponseTimeoutSeconds(30); + task.setWorkflowTask(workflowTask); + + workflow.setTasks(List.of(task)); + + Duration result = + ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); + + // Should use default workflowOffsetTimeout + assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); + assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); + } +} diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java index 7fb6599671..44be365b46 100644 --- a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java +++ b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java @@ -13,57 +13,68 @@ package com.netflix.conductor.core.reconciliation; import java.time.Duration; +import java.util.List; import java.util.concurrent.Executor; import org.junit.Before; import org.junit.Test; +import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.TaskModel.Status; +import com.netflix.conductor.model.WorkflowModel; import com.fasterxml.jackson.databind.ObjectMapper; +import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestWorkflowSweeper { private ConductorProperties properties; + private SweeperProperties sweeperProperties; private WorkflowExecutor workflowExecutor; private QueueDAO queueDAO; private ExecutionDAO executionDAO; - private SweeperProperties sweeperProperties; + private WorkflowSweeper workflowSweeper; private SystemTaskRegistry systemTaskRegistry; private ObjectMapper objectMapper; - private Executor sweeperExecutor; + private Executor executor; @Before public void setUp() { properties = mock(ConductorProperties.class); + sweeperProperties = mock(SweeperProperties.class); workflowExecutor = mock(WorkflowExecutor.class); queueDAO = mock(QueueDAO.class); executionDAO = mock(ExecutionDAO.class); - sweeperProperties = mock(SweeperProperties.class); systemTaskRegistry = mock(SystemTaskRegistry.class); - objectMapper = mock(ObjectMapper.class); - sweeperExecutor = mock(Executor.class); + objectMapper = new ObjectMapper(); + executor = mock(Executor.class); - when(properties.getSweeperThreadCount()).thenReturn(0); // Don't start threads in tests - when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(30)); + when(properties.getMaxPostponeDurationSeconds()).thenReturn(Duration.ofSeconds(2000000)); + when(properties.getSweeperThreadCount()).thenReturn(1); + when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(1800)); when(properties.getLockLeaseTime()).thenReturn(Duration.ofSeconds(60)); - when(sweeperProperties.getSweepBatchSize()).thenReturn(2); + when(sweeperProperties.getSweepBatchSize()).thenReturn(10); when(sweeperProperties.getQueuePopTimeout()).thenReturn(100); - } - @Test - public void testWorkflowSweeperInitialization() { - // Test that WorkflowSweeper can be instantiated - WorkflowSweeper workflowSweeper = + workflowSweeper = new WorkflowSweeper( - sweeperExecutor, + executor, queueDAO, workflowExecutor, executionDAO, @@ -71,8 +82,144 @@ public void testWorkflowSweeperInitialization() { sweeperProperties, systemTaskRegistry, objectMapper); + } + + @Test + public void testSweepTerminalWorkflow() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + workflowModel.setStatus(WorkflowModel.Status.COMPLETED); + + when(workflowExecutor.getWorkflow("1", true)).thenReturn(workflowModel); + + workflowSweeper.sweep("1"); + + verify(queueDAO).remove(DECIDER_QUEUE, "1"); + verify(workflowExecutor, never()).decideWithLock(any(WorkflowModel.class)); + } + + @Test + public void testSweepNullWorkflow() { + when(workflowExecutor.getWorkflow("1", true)).thenReturn(null); + + workflowSweeper.sweep("1"); + + verify(queueDAO).remove(DECIDER_QUEUE, "1"); + } + + @Test + public void testSweepRunningWorkflow() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("2"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("2", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + + workflowSweeper.sweep("2"); + + verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); + } + + @Test + public void testSweepCannotGetLock() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("3"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("3", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(null); + + workflowSweeper.sweep("3"); + + verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); + verify(queueDAO).push(eq(DECIDER_QUEUE), eq("3"), eq(0), anyLong()); + } + + @Test + public void testSweepRepairsScheduledTask() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("4"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.SCHEDULED); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("4", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); + when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); + + workflowSweeper.sweep("4"); + + verify(queueDAO).push(anyString(), eq("task1"), anyLong()); + } + + @Test + public void testSweepDoesNotRepairTaskAlreadyInQueue() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("5"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setReferenceTaskName("task1"); + taskModel.setTaskType("SIMPLE"); + taskModel.setStatus(Status.SCHEDULED); + workflowModel.setTasks(List.of(taskModel)); + + when(workflowExecutor.getWorkflow("5", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); + when(queueDAO.containsMessage(anyString(), eq("task1"))).thenReturn(true); + + workflowSweeper.sweep("5"); + + verify(queueDAO, never()).push(anyString(), eq("task1"), anyLong()); + } + + @Test + public void testSweepSubWorkflowTask() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("6"); + workflowModel.setStatus(WorkflowModel.Status.RUNNING); + + TaskModel subWorkflowTask = new TaskModel(); + subWorkflowTask.setTaskId("task1"); + subWorkflowTask.setReferenceTaskName("task1"); + subWorkflowTask.setTaskType(TaskType.TASK_TYPE_SUB_WORKFLOW); + subWorkflowTask.setStatus(Status.IN_PROGRESS); + subWorkflowTask.setSubWorkflowId("sub-wf-1"); + workflowModel.setTasks(List.of(subWorkflowTask)); + + WorkflowModel subWorkflow = new WorkflowModel(); + subWorkflow.setWorkflowId("sub-wf-1"); + subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); + + when(workflowExecutor.getWorkflow("6", true)).thenReturn(workflowModel); + when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); + when(executionDAO.getWorkflow("sub-wf-1", false)).thenReturn(subWorkflow); + + workflowSweeper.sweep("6"); - // Basic test to ensure constructor works - // The actual sweep logic is tested via integration tests + verify(executionDAO).updateTask(subWorkflowTask); } } From 17c92cf2daf21cf3b5c9ef18078b899ec3944c9e Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:01:41 +0530 Subject: [PATCH 18/39] Restore old WorkflowSweeper in org.conductoross package - Restored org.conductoross.conductor.core.execution.WorkflowSweeper - Restored org.conductoross.conductor.core.execution.SweeperProperties - Keep both old and new sweeper implementations - Old sweeper (org.conductoross): original simple implementation - New sweeper (com.netflix): has simplified improvements without Orkes-specific code --- .../core/execution/SweeperProperties.java | 30 ++ .../core/execution/WorkflowSweeper.java | 294 ++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java b/core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java new file mode 100644 index 0000000000..813af44376 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/SweeperProperties.java @@ -0,0 +1,30 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Configuration +@ConfigurationProperties("conductor.app.sweeper") +@Getter +@Setter +@ToString +public class SweeperProperties { + private int sweepBatchSize = 2; + private int queuePopTimeout = 100; +} diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java new file mode 100644 index 0000000000..9aad1765ee --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -0,0 +1,294 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.core.LifecycleAwareComponent; +import com.netflix.conductor.core.config.ConductorProperties; +import com.netflix.conductor.core.execution.WorkflowExecutor; +import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; +import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; +import com.netflix.conductor.core.utils.QueueUtils; +import com.netflix.conductor.core.utils.Utils; +import com.netflix.conductor.dao.ExecutionDAO; +import com.netflix.conductor.dao.QueueDAO; +import com.netflix.conductor.metrics.Monitors; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; +import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; + +@Component +@Slf4j +@ConditionalOnProperty( + name = "conductor.app.sweeper.enabled", + havingValue = "true", + matchIfMissing = true) +public class WorkflowSweeper extends LifecycleAwareComponent { + + private final QueueDAO queueDAO; + private final SweeperProperties sweeperProperties; + private final WorkflowExecutor workflowExecutor; + private final ExecutionDAO executionDAO; + private final Duration worflowOffsetTimeout; + private final Executor sweeperExecutor; + private final ConductorProperties properties; + private final ObjectMapper objectMapper; + private SystemTaskRegistry systemTaskRegistry; + private final Clock clock = Clock.systemDefaultZone(); + private AtomicBoolean stop = new AtomicBoolean(false); + + public WorkflowSweeper( + @Qualifier(SWEEPER_EXECUTOR_NAME) Executor sweeperExecutor, + QueueDAO queueDAO, + WorkflowExecutor workflowExecutor, + ExecutionDAO executionDAO, + ConductorProperties properties, + SweeperProperties sweeperProperties, + SystemTaskRegistry systemTaskRegistry, + ObjectMapper objectMapper) { + this.queueDAO = queueDAO; + this.executionDAO = executionDAO; + this.sweeperProperties = sweeperProperties; + this.workflowExecutor = workflowExecutor; + this.worflowOffsetTimeout = properties.getWorkflowOffsetTimeout(); + this.sweeperExecutor = sweeperExecutor; + this.properties = properties; + this.systemTaskRegistry = systemTaskRegistry; + this.objectMapper = objectMapper; + log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); + for (int i = 0; i < properties.getSweeperThreadCount(); i++) { + sweeperExecutor.execute(this::pollAndSweep); + } + } + + /* + For system task -> Verify the task isAsync() and not isAsyncComplete() or isAsyncComplete() in SCHEDULED state, + and in SCHEDULED or IN_PROGRESS state. (Example: SUB_WORKFLOW tasks in SCHEDULED state) + For simple task -> Verify the task is in SCHEDULED state. + */ + private final Predicate isTaskRepairable = + task -> { + if (systemTaskRegistry.isSystemTask(task.getTaskType())) { // If system task + WorkflowSystemTask workflowSystemTask = + systemTaskRegistry.get(task.getTaskType()); + return workflowSystemTask.isAsync() + && (!workflowSystemTask.isAsyncComplete(task) + || (workflowSystemTask.isAsyncComplete(task) + && task.getStatus() == TaskModel.Status.SCHEDULED)) + && (task.getStatus() == TaskModel.Status.IN_PROGRESS + || task.getStatus() == TaskModel.Status.SCHEDULED); + } else { // Else if simple task or wait task + return (task.getStatus() == TaskModel.Status.SCHEDULED + || (!task.getStatus().isTerminal() + && task.getWaitTimeout() > 0 + && (clock.millis() - task.getWaitTimeout() > 1000))); + } + }; + + private void pollAndSweep() { + try { + while (true) { + if (stop.get()) { + return; + } + try { + if (!isRunning()) { + log.trace("Component stopped, skip workflow sweep"); + } else { + List workflowIds = + queueDAO.pop( + DECIDER_QUEUE, + sweeperProperties.getSweepBatchSize(), + sweeperProperties.getQueuePopTimeout()); + log.trace("Found {} workflows to sweep", workflowIds.size()); + workflowIds.stream() + .parallel() + .forEach( + workflowId -> + Monitors.getTimer("workflowSweeper") + .record(() -> sweep(workflowId))); + } + } catch (Throwable e) { + log.warn("Error while running sweeper {}", e.getMessage(), e); + } + } + } catch (Throwable e) { + log.error("Error polling for sweep entries {}", e.getMessage(), e); + } + } + + public void sweep(String workflowId) { + log.info("Running sweeper for workflow {}", workflowId); + + try { + WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); + if (workflow == null || workflow.getStatus().isTerminal()) { + queueDAO.remove(DECIDER_QUEUE, workflowId); + return; + } + + String tasks = + workflow.getTasks().stream() + .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) + .toList() + .toString(); + workflow = workflowExecutor.decideWithLock(workflow); + if (workflow == null) { + // couldn't get a lock + // Let's try again... with the lockTime timeout / 2 + int backoff = (int) (properties.getLockLeaseTime().toMillis() / 2); + log.info("can't get a lock on {}, will try after {} ms", workflowId, backoff); + queueDAO.push(DECIDER_QUEUE, workflowId, 0, Duration.ofMillis(backoff).toSeconds()); + return; + } + if (workflow.getStatus().isTerminal()) { + queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); + return; + } + + String tasksAfterDecide = + workflow.getTasks().stream() + .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) + .toList() + .toString(); + + // Workflow has not completed and decide did not change the status of the tasks + // Every task that is running MUST be in the queue + if (tasks.equals(tasksAfterDecide)) { + workflow.getTasks().forEach(this::verifyAndRepairTask); + } + + // Workflow is in running status, there MUST be at-least one task that is not terminal + // (scheduled, in progress) + boolean hasRunningTasks = + workflow.getTasks().stream().anyMatch(task -> !task.getStatus().isTerminal()); + if (!hasRunningTasks) { + // Workflow is in RUNNING status but there are no tasks that are running + // This can happen in case of the database failures where the task scheduling failed + // after the last task was completed + // To fix, we reset the executed flag of the last task and re-run decide + forceSetLastTaskAsNotExecuted(workflow); + workflow = workflowExecutor.decideWithLock(workflow); + } + + // If parent workflow exists, call repair on that too - meaning ensure the parent is in + // the decider queue + if (workflow != null && StringUtils.isNotBlank(workflow.getParentWorkflowId())) { + ensureWorkflowExistsInDecider(workflow.getParentWorkflowId()); + } + } catch (Throwable e) { + log.error("Error running sweep for {}, error = {}", workflowId, e.getMessage(), e); + } + } + + void verifyAndRepairTask(TaskModel task) { + if (isTaskRepairable.test(task)) { + // Ensure QueueDAO contains this taskId + String taskQueueName = QueueUtils.getQueueName(task); + if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { + queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); + log.info( + "Task {} in workflow {} re-queued for repairs", + task.getTaskId(), + task.getWorkflowInstanceId()); + Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); + } + } + if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) + && task.getStatus() == TaskModel.Status.IN_PROGRESS) { + WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); + if (subWorkflow.getStatus().isTerminal()) { + log.info( + "Repairing sub workflow task {} for sub workflow {} in workflow {}", + task.getTaskId(), + task.getSubWorkflowId(), + task.getWorkflowInstanceId()); + repairSubWorkflowTask(task, subWorkflow); + } + } + } + + private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { + switch (subWorkflow.getStatus()) { + case COMPLETED: + task.setStatus(TaskModel.Status.COMPLETED); + break; + case FAILED: + task.setStatus(TaskModel.Status.FAILED); + break; + case TERMINATED: + task.setStatus(TaskModel.Status.CANCELED); + break; + case TIMED_OUT: + task.setStatus(TaskModel.Status.TIMED_OUT); + break; + } + task.addOutput(subWorkflow.getOutput()); + executionDAO.updateTask(task); + } + + private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { + if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { + TaskModel taskModel = workflow.getTasks().getLast(); + log.warn( + "Force setting isExecuted to false for last task - {} - {} - {} - {} for workflow {}", + taskModel.getTaskId(), + taskModel.getReferenceTaskName(), + taskModel.getStatus(), + taskModel.getTaskDefName(), + taskModel.getWorkflowInstanceId()); + try { + log.debug( + "workflow {} JSON {}", + workflow.getWorkflowId(), + objectMapper.writeValueAsString(workflow)); + } catch (Exception e) { + log.error("Could not warn about workflow {}", workflow.getWorkflowId(), e); + } + taskModel.setExecuted(false); + executionDAO.updateWorkflow(workflow); + } + } + + private void ensureWorkflowExistsInDecider(String workflowId) { + String queueName = Utils.DECIDER_QUEUE; + if (!queueDAO.containsMessage(queueName, workflowId)) { + queueDAO.push(queueName, workflowId, worflowOffsetTimeout.getSeconds()); + Monitors.recordQueueMessageRepushFromRepairService(queueName); + } + } + + @Override + public void doStop() { + stop.set(true); + ((ExecutorService) this.sweeperExecutor).shutdownNow(); + } +} From a29a407889bc68b957889a08a76df958c6f3e5f6 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:03:38 +0530 Subject: [PATCH 19/39] Restore ExecutorUtils to org.conductoross package - Restored org.conductoross.conductor.core.execution.ExecutorUtils.java - Now both old (org.conductoross) and new (com.netflix) packages have ExecutorUtils - No files moved, both versions exist --- .../core/execution/ExecutorUtils.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java new file mode 100644 index 0000000000..5a2904739b --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution; + +import java.time.Duration; +import java.util.Random; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import lombok.extern.slf4j.Slf4j; + +import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; + +@Slf4j +public class ExecutorUtils { + + public static Duration computePostpone( + WorkflowModel workflowModel, + Duration workflowOffsetTimeout, + Duration maxPostponeDuration) { + long currentTimeMillis = System.currentTimeMillis(); + long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); + long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); + + long postponeDurationSeconds = 0; + for (TaskModel taskModel : workflowModel.getTasks()) { + TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); + if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { + if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { + if (taskModel.getWaitTimeout() != 0) { + long deltaInSeconds = + (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; + postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; + } + } else if (taskModel.getTaskType().equals("HUMAN")) { + // HUMAN tasks always use the default workflowOffsetTimeout + // This matches the old sweeper behavior + postponeDurationSeconds = workflowOffsetTimeoutSeconds; + } else { + // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... + // not sure but keeping it this way just in case. + long responseTimeoutSeconds = + taskDef != null + ? taskDef.getResponseTimeoutSeconds() + : taskModel.getResponseTimeoutSeconds(); + if (responseTimeoutSeconds != 0) { + long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; + if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { + postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; + } + } + } + break; + } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { + if (taskDef != null) { + postponeDurationSeconds = getMinTimeout(taskDef); + } + + if (postponeDurationSeconds <= 0) { + if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { + postponeDurationSeconds = + workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; + } + } + break; + } + } + log.trace( + "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", + postponeDurationSeconds, + workflowOffsetTimeoutSeconds, + workflowModel.getWorkflowId()); + + // Cap postpone duration to the configured maximum (matches old sweeper behavior) + if (postponeDurationSeconds > maxPostponeDurationSeconds) { + postponeDurationSeconds = maxPostponeDurationSeconds; + } + + if (postponeDurationSeconds > 0) { + long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); + return Duration.ofSeconds(applyJitter(capped)); + } + return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); + } + + private static long getMinTimeout(TaskDef taskDef) { + long pollTimeoutSeconds = + taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; + return Math.min( + taskDef.getTimeoutSeconds(), + Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); + } + + /** + * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the + * duration. + * + *

For example, if duration is 45 seconds, returns value between [30-60] seconds. + * + * @param durationSeconds postpone duration in seconds + * @return duration with jitter applied + */ + public static long applyJitter(long durationSeconds) { + if (durationSeconds <= 3) { + return durationSeconds; // Too small for jitter + } + long range = durationSeconds / 3; + long jitter = new Random().nextInt((int) (2 * range + 1)) - range; + return durationSeconds + jitter; + } +} From 3f1d02a2e0e6cf2d90e958bc11f820de2f2009e1 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:10:05 +0530 Subject: [PATCH 20/39] simpler sweeper. --- .../core/execution/WorkflowSweeper.java | 76 ++++++++++++------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java index 9aad1765ee..ce231d5a01 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -15,6 +15,7 @@ import java.time.Clock; import java.time.Duration; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; @@ -23,6 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.netflix.conductor.common.metadata.tasks.TaskType; @@ -145,6 +147,12 @@ private void pollAndSweep() { } } + @Async(SWEEPER_EXECUTOR_NAME) + public CompletableFuture sweepAsync(String workflowId) { + sweep(workflowId); + return CompletableFuture.completedFuture(null); + } + public void sweep(String workflowId) { log.info("Running sweeper for workflow {}", workflowId); @@ -183,7 +191,47 @@ public void sweep(String workflowId) { // Workflow has not completed and decide did not change the status of the tasks // Every task that is running MUST be in the queue if (tasks.equals(tasksAfterDecide)) { - workflow.getTasks().forEach(this::verifyAndRepairTask); + long now = System.currentTimeMillis(); + workflow.getTasks().stream() + .filter(isTaskRepairable) + .forEach( + task -> { + log.warn( + "Going to repair the task {} / {}, with status {}, workflow = {}, timeout = {}, now-wait = {}", + task.getTaskId(), + task.getReferenceTaskName(), + task.getStatus(), + workflowId, + task.getWaitTimeout(), + (now - task.getWaitTimeout())); + Monitors.recordQueueMessageRepushFromRepairService( + task.getTaskDefName()); + String queueName = QueueUtils.getQueueName(task); + if (!queueDAO.containsMessage(queueName, task.getTaskId())) { + queueDAO.push( + queueName, task.getTaskId(), task.getCallbackAfterSeconds()); + } + }); + + // Repair subworkflow tasks if needed + workflow.getTasks().stream() + .filter( + task -> + task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) + && task.getStatus() == TaskModel.Status.IN_PROGRESS) + .forEach( + task -> { + WorkflowModel subWorkflow = + executionDAO.getWorkflow(task.getSubWorkflowId(), false); + if (subWorkflow.getStatus().isTerminal()) { + log.info( + "Repairing sub workflow task {} for sub workflow {} in workflow {}", + task.getTaskId(), + task.getSubWorkflowId(), + task.getWorkflowInstanceId()); + repairSubWorkflowTask(task, subWorkflow); + } + }); } // Workflow is in running status, there MUST be at-least one task that is not terminal @@ -209,32 +257,6 @@ public void sweep(String workflowId) { } } - void verifyAndRepairTask(TaskModel task) { - if (isTaskRepairable.test(task)) { - // Ensure QueueDAO contains this taskId - String taskQueueName = QueueUtils.getQueueName(task); - if (!queueDAO.containsMessage(taskQueueName, task.getTaskId())) { - queueDAO.push(taskQueueName, task.getTaskId(), task.getCallbackAfterSeconds()); - log.info( - "Task {} in workflow {} re-queued for repairs", - task.getTaskId(), - task.getWorkflowInstanceId()); - Monitors.recordQueueMessageRepushFromRepairService(task.getTaskDefName()); - } - } - if (task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) - && task.getStatus() == TaskModel.Status.IN_PROGRESS) { - WorkflowModel subWorkflow = executionDAO.getWorkflow(task.getSubWorkflowId(), false); - if (subWorkflow.getStatus().isTerminal()) { - log.info( - "Repairing sub workflow task {} for sub workflow {} in workflow {}", - task.getTaskId(), - task.getSubWorkflowId(), - task.getWorkflowInstanceId()); - repairSubWorkflowTask(task, subWorkflow); - } - } - } private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { switch (subWorkflow.getStatus()) { From 6eba38ab59b4d3c6426a5b771cfdfd2c52e0948e Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:15:28 +0530 Subject: [PATCH 21/39] changes --- .../core/execution/WorkflowSweeper.java | 40 ------------------- local-publish.gradle | 21 ---------- .../src/main/resources/application.properties | 2 +- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 local-publish.gradle diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java index ce231d5a01..d2da65f0d6 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -212,26 +212,6 @@ public void sweep(String workflowId) { queueName, task.getTaskId(), task.getCallbackAfterSeconds()); } }); - - // Repair subworkflow tasks if needed - workflow.getTasks().stream() - .filter( - task -> - task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) - && task.getStatus() == TaskModel.Status.IN_PROGRESS) - .forEach( - task -> { - WorkflowModel subWorkflow = - executionDAO.getWorkflow(task.getSubWorkflowId(), false); - if (subWorkflow.getStatus().isTerminal()) { - log.info( - "Repairing sub workflow task {} for sub workflow {} in workflow {}", - task.getTaskId(), - task.getSubWorkflowId(), - task.getWorkflowInstanceId()); - repairSubWorkflowTask(task, subWorkflow); - } - }); } // Workflow is in running status, there MUST be at-least one task that is not terminal @@ -257,26 +237,6 @@ public void sweep(String workflowId) { } } - - private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { - switch (subWorkflow.getStatus()) { - case COMPLETED: - task.setStatus(TaskModel.Status.COMPLETED); - break; - case FAILED: - task.setStatus(TaskModel.Status.FAILED); - break; - case TERMINATED: - task.setStatus(TaskModel.Status.CANCELED); - break; - case TIMED_OUT: - task.setStatus(TaskModel.Status.TIMED_OUT); - break; - } - task.addOutput(subWorkflow.getOutput()); - executionDAO.updateTask(task); - } - private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { TaskModel taskModel = workflow.getTasks().getLast(); diff --git a/local-publish.gradle b/local-publish.gradle deleted file mode 100644 index 0404627214..0000000000 --- a/local-publish.gradle +++ /dev/null @@ -1,21 +0,0 @@ -subprojects { - apply plugin: 'maven-publish' - apply plugin: 'java-library' - - group = 'org.conductoross' - version = '3.20.1-SNAPSHOT' // Set your desired version - - publishing { - publications { - mavenJava(MavenPublication) { - from components.java - groupId = 'org.conductoross' - artifactId = project.name - } - } - - repositories { - mavenLocal() - } - } -} diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 974f06e318..8c8481bb5d 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -183,4 +183,4 @@ management.azuremonitor.metrics.export.enabled=false management.azuremonitor.metrics.export.instrumentationKey=INSTRUMENTATION_KEY # When enabled logs metrics as info level logs -conductor.metrics-logger.enabled=false \ No newline at end of file +conductor.metrics-logger.enablnoed=false \ No newline at end of file From 6445e3b916083cd8e5f30333193b0c3627c3c402 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:17:52 +0530 Subject: [PATCH 22/39] Revert changes to com.netflix WorkflowSweeper - keep it disabled as-is --- .../reconciliation/WorkflowReconciler.java | 3 +- .../core/reconciliation/WorkflowSweeper.java | 406 +++++++----------- 2 files changed, 152 insertions(+), 257 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java index 555f445387..9e7e6e43f4 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowReconciler.java @@ -32,7 +32,8 @@ * Periodically polls all running workflows in the system and evaluates them for timeouts and/or * maintain consistency. */ -// Deprecated - superseded by WorkflowSweeper in com.netflix.conductor.core.reconciliation package +// Deprecated - and superseeded by new WorkflowSweeper in org.conductoross.conductor.core.execution +// package @Deprecated(forRemoval = true) @Component @ConditionalOnProperty( diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index ab489b870c..45cda3d9b2 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 Conductor Authors. + * Copyright 2022 Conductor Authors. *

* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -12,139 +12,61 @@ */ package com.netflix.conductor.core.reconciliation; -import java.time.Clock; -import java.time.Duration; -import java.util.List; +import java.time.Instant; +import java.util.Optional; +import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Predicate; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; +import com.netflix.conductor.annotations.VisibleForTesting; +import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.core.LifecycleAwareComponent; +import com.netflix.conductor.core.WorkflowContext; import com.netflix.conductor.core.config.ConductorProperties; +import com.netflix.conductor.core.dal.ExecutionDAOFacade; +import com.netflix.conductor.core.exception.NotFoundException; import com.netflix.conductor.core.execution.WorkflowExecutor; -import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; -import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; -import com.netflix.conductor.core.utils.QueueUtils; -import com.netflix.conductor.core.utils.Utils; -import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; - -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; +import com.netflix.conductor.service.ExecutionLockService; import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; -@Component -@Slf4j -@ConditionalOnProperty( - name = "conductor.app.sweeper.enabled", - havingValue = "true", - matchIfMissing = true) -public class WorkflowSweeper extends LifecycleAwareComponent { +// @Component +public class WorkflowSweeper { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowSweeper.class); - private final QueueDAO queueDAO; - private final SweeperProperties sweeperProperties; - private final WorkflowExecutor workflowExecutor; - private final ExecutionDAO executionDAO; - private final Duration worflowOffsetTimeout; - private final Executor sweeperExecutor; private final ConductorProperties properties; - private final ObjectMapper objectMapper; - private SystemTaskRegistry systemTaskRegistry; - private final Clock clock = Clock.systemDefaultZone(); - private AtomicBoolean stop = new AtomicBoolean(false); + private final WorkflowExecutor workflowExecutor; + private final WorkflowRepairService workflowRepairService; + private final QueueDAO queueDAO; + private final ExecutionDAOFacade executionDAOFacade; + private final ExecutionLockService executionLockService; + + private static final String CLASS_NAME = WorkflowSweeper.class.getSimpleName(); public WorkflowSweeper( - @Qualifier(SWEEPER_EXECUTOR_NAME) Executor sweeperExecutor, - QueueDAO queueDAO, WorkflowExecutor workflowExecutor, - ExecutionDAO executionDAO, + Optional workflowRepairService, ConductorProperties properties, - SweeperProperties sweeperProperties, - SystemTaskRegistry systemTaskRegistry, - ObjectMapper objectMapper) { + QueueDAO queueDAO, + ExecutionDAOFacade executionDAOFacade, + ExecutionLockService executionLockService) { + this.properties = properties; this.queueDAO = queueDAO; - this.executionDAO = executionDAO; - this.sweeperProperties = sweeperProperties; this.workflowExecutor = workflowExecutor; - this.worflowOffsetTimeout = properties.getWorkflowOffsetTimeout(); - this.sweeperExecutor = sweeperExecutor; - this.properties = properties; - this.systemTaskRegistry = systemTaskRegistry; - this.objectMapper = objectMapper; - log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); - for (int i = 0; i < properties.getSweeperThreadCount(); i++) { - sweeperExecutor.execute(this::pollAndSweep); - } - } - - /* - For system task -> Verify the task isAsync() and not isAsyncComplete() or isAsyncComplete() in SCHEDULED state, - and in SCHEDULED or IN_PROGRESS state. (Example: SUB_WORKFLOW tasks in SCHEDULED state) - For simple task -> Verify the task is in SCHEDULED state. - */ - private final Predicate isTaskRepairable = - task -> { - if (systemTaskRegistry.isSystemTask(task.getTaskType())) { // If system task - WorkflowSystemTask workflowSystemTask = - systemTaskRegistry.get(task.getTaskType()); - return workflowSystemTask.isAsync() - && (!workflowSystemTask.isAsyncComplete(task) - || (workflowSystemTask.isAsyncComplete(task) - && task.getStatus() == TaskModel.Status.SCHEDULED)) - && (task.getStatus() == TaskModel.Status.IN_PROGRESS - || task.getStatus() == TaskModel.Status.SCHEDULED); - } else { // Else if simple task or wait task - return (task.getStatus() == TaskModel.Status.SCHEDULED - || (!task.getStatus().isTerminal() - && task.getWaitTimeout() > 0 - && (clock.millis() - task.getWaitTimeout() > 1000))); - } - }; - - private void pollAndSweep() { - try { - while (true) { - if (stop.get()) { - return; - } - try { - if (!isRunning()) { - log.trace("Component stopped, skip workflow sweep"); - } else { - List workflowIds = - queueDAO.pop( - DECIDER_QUEUE, - sweeperProperties.getSweepBatchSize(), - sweeperProperties.getQueuePopTimeout()); - log.trace("Found {} workflows to sweep", workflowIds.size()); - workflowIds.stream() - .parallel() - .forEach( - workflowId -> - Monitors.getTimer("workflowSweeper") - .record(() -> sweep(workflowId))); - } - } catch (Throwable e) { - log.warn("Error while running sweeper {}", e.getMessage(), e); - } - } - } catch (Throwable e) { - log.error("Error polling for sweep entries {}", e.getMessage(), e); - } + this.executionDAOFacade = executionDAOFacade; + this.workflowRepairService = workflowRepairService.orElse(null); + this.executionLockService = executionLockService; + LOGGER.info("WorkflowSweeper initialized."); } @Async(SWEEPER_EXECUTOR_NAME) @@ -154,165 +76,137 @@ public CompletableFuture sweepAsync(String workflowId) { } public void sweep(String workflowId) { - log.info("Running sweeper for workflow {}", workflowId); - + WorkflowContext workflowContext = new WorkflowContext(properties.getAppId()); + WorkflowContext.set(workflowContext); + WorkflowModel workflow = null; try { - WorkflowModel workflow = workflowExecutor.getWorkflow(workflowId, true); - if (workflow == null || workflow.getStatus().isTerminal()) { - queueDAO.remove(DECIDER_QUEUE, workflowId); + if (!executionLockService.acquireLock(workflowId)) { return; } - - String tasks = - workflow.getTasks().stream() - .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) - .toList() - .toString(); - workflow = workflowExecutor.decideWithLock(workflow); - if (workflow == null) { - // couldn't get a lock - // Let's try again... with the lockTime timeout / 2 - int backoff = (int) (properties.getLockLeaseTime().toMillis() / 2); - log.info("can't get a lock on {}, will try after {} ms", workflowId, backoff); - queueDAO.push(DECIDER_QUEUE, workflowId, 0, Duration.ofMillis(backoff).toSeconds()); - return; + workflow = executionDAOFacade.getWorkflowModel(workflowId, true); + LOGGER.debug("Running sweeper for workflow {}", workflowId); + if (workflowRepairService != null) { + // Verify and repair tasks in the workflow. + workflowRepairService.verifyAndRepairWorkflowTasks(workflow); } - if (workflow.getStatus().isTerminal()) { - queueDAO.remove(DECIDER_QUEUE, workflow.getWorkflowId()); + long decideStartTime = System.currentTimeMillis(); + workflow = workflowExecutor.decide(workflow.getWorkflowId()); + Monitors.recordWorkflowDecisionTime(System.currentTimeMillis() - decideStartTime); + if (workflow != null && workflow.getStatus().isTerminal()) { + queueDAO.remove(DECIDER_QUEUE, workflowId); return; } - - String tasksAfterDecide = - workflow.getTasks().stream() - .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) - .toList() - .toString(); - - // Workflow has not completed and decide did not change the status of the tasks - // Every task that is running MUST be in the queue - if (tasks.equals(tasksAfterDecide)) { - long now = System.currentTimeMillis(); - workflow.getTasks().stream() - .filter(isTaskRepairable) - .forEach( - task -> { - log.warn( - "Going to repair the task {} / {}, with status {}, workflow = {}, timeout = {}, now-wait = {}", - task.getTaskId(), - task.getReferenceTaskName(), - task.getStatus(), - workflowId, - task.getWaitTimeout(), - (now - task.getWaitTimeout())); - Monitors.recordQueueMessageRepushFromRepairService( - task.getTaskDefName()); - String queueName = QueueUtils.getQueueName(task); - if (!queueDAO.containsMessage(queueName, task.getTaskId())) { - queueDAO.push( - queueName, task.getTaskId(), task.getCallbackAfterSeconds()); - } - }); - - // Repair subworkflow tasks if needed - workflow.getTasks().stream() - .filter( - task -> - task.getTaskType().equals(TaskType.TASK_TYPE_SUB_WORKFLOW) - && task.getStatus() == TaskModel.Status.IN_PROGRESS) - .forEach( - task -> { - WorkflowModel subWorkflow = - executionDAO.getWorkflow(task.getSubWorkflowId(), false); - if (subWorkflow.getStatus().isTerminal()) { - log.info( - "Repairing sub workflow task {} for sub workflow {} in workflow {}", - task.getTaskId(), - task.getSubWorkflowId(), - task.getWorkflowInstanceId()); - repairSubWorkflowTask(task, subWorkflow); - } - }); - } - - // Workflow is in running status, there MUST be at-least one task that is not terminal - // (scheduled, in progress) - boolean hasRunningTasks = - workflow.getTasks().stream().anyMatch(task -> !task.getStatus().isTerminal()); - if (!hasRunningTasks) { - // Workflow is in RUNNING status but there are no tasks that are running - // This can happen in case of the database failures where the task scheduling failed - // after the last task was completed - // To fix, we reset the executed flag of the last task and re-run decide - // TODO: This MIGHT not be completely accurate, write a test to validate what - // happens if the last task is a JOIN task - forceSetLastTaskAsNotExecuted(workflow); - workflow = workflowExecutor.decideWithLock(workflow); - } - - // If parent workflow exists, call repair on that too - meaning ensure the parent is in - // the decider queue - if (workflow != null && StringUtils.isNotBlank(workflow.getParentWorkflowId())) { - ensureWorkflowExistsInDecider(workflow.getParentWorkflowId()); - } - } catch (Throwable e) { - log.error("Error running sweep for {}, error = {}", workflowId, e.getMessage(), e); + } catch (NotFoundException nfe) { + queueDAO.remove(DECIDER_QUEUE, workflowId); + LOGGER.info( + "Workflow NOT found for id:{}. Removed it from decider queue", workflowId, nfe); + return; + } catch (Exception e) { + Monitors.error(CLASS_NAME, "sweep"); + LOGGER.error("Error running sweep for " + workflowId, e); + } finally { + executionLockService.releaseLock(workflowId); } - } - - - private void repairSubWorkflowTask(TaskModel task, WorkflowModel subWorkflow) { - switch (subWorkflow.getStatus()) { - case COMPLETED: - task.setStatus(TaskModel.Status.COMPLETED); - break; - case FAILED: - task.setStatus(TaskModel.Status.FAILED); - break; - case TERMINATED: - task.setStatus(TaskModel.Status.CANCELED); - break; - case TIMED_OUT: - task.setStatus(TaskModel.Status.TIMED_OUT); - break; + long workflowOffsetTimeout = + workflowOffsetWithJitter(properties.getWorkflowOffsetTimeout().getSeconds()); + if (workflow != null) { + long startTime = Instant.now().toEpochMilli(); + unack(workflow, workflowOffsetTimeout); + long endTime = Instant.now().toEpochMilli(); + Monitors.recordUnackTime(workflow.getWorkflowName(), endTime - startTime); + } else { + LOGGER.warn( + "Workflow with {} id can not be found. Attempting to unack using the id", + workflowId); + queueDAO.setUnackTimeout(DECIDER_QUEUE, workflowId, workflowOffsetTimeout * 1000); } - task.addOutput(subWorkflow.getOutput()); - executionDAO.updateTask(task); } - private void forceSetLastTaskAsNotExecuted(WorkflowModel workflow) { - if (workflow.getTasks() != null && !workflow.getTasks().isEmpty()) { - TaskModel taskModel = workflow.getTasks().getLast(); - log.warn( - "Force setting isExecuted to false for last task - {} - {} - {} - {} for workflow {}", - taskModel.getTaskId(), - taskModel.getReferenceTaskName(), - taskModel.getStatus(), - taskModel.getTaskDefName(), - taskModel.getWorkflowInstanceId()); - try { - log.debug( - "workflow {} JSON {}", - workflow.getWorkflowId(), - objectMapper.writeValueAsString(workflow)); - } catch (Exception e) { - log.error("Could not warn about workflow {}", workflow.getWorkflowId(), e); + /** + * Calculates the next decider queue unack delay by evaluating active tasks and choosing the + * smallest eligible delay. This prevents long-running tasks from delaying evaluation when a + * shorter timeout is due, while still honoring maxPostpone caps. + */ + @VisibleForTesting + void unack(WorkflowModel workflowModel, long workflowOffsetTimeout) { + // Pick the minimum next-evaluation delay across eligible tasks, capped by maxPostpone. + Long postponeDurationSeconds = null; + long maxPostponeSeconds = properties.getMaxPostponeDurationSeconds().getSeconds(); + for (TaskModel taskModel : workflowModel.getTasks()) { + Long candidateSeconds = null; + if (taskModel.getStatus() == Status.IN_PROGRESS) { + // Active tasks: delay based on wait/response timeout or workflow offset. + if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_WAIT)) { + if (taskModel.getWaitTimeout() == 0) { + candidateSeconds = workflowOffsetTimeout; + } else { + // waitTimeout is an absolute epoch ms; compute remaining seconds. + long deltaInSeconds = + (taskModel.getWaitTimeout() - System.currentTimeMillis()) / 1000; + candidateSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; + } + } else if (taskModel.getTaskType().equals(TaskType.TASK_TYPE_HUMAN)) { + candidateSeconds = workflowOffsetTimeout; + } else { + candidateSeconds = + (taskModel.getResponseTimeoutSeconds() != 0) + // Add 1s so the response timeout window fully elapses. + ? taskModel.getResponseTimeoutSeconds() + 1 + : workflowOffsetTimeout; + } + } else if (taskModel.getStatus() == Status.SCHEDULED) { + // Scheduled tasks: use poll timeout when present, else workflow timeout or offset. + Optional taskDefinition = taskModel.getTaskDefinition(); + if (taskDefinition.isPresent()) { + TaskDef taskDef = taskDefinition.get(); + if (taskDef.getPollTimeoutSeconds() != null + && taskDef.getPollTimeoutSeconds() != 0) { + candidateSeconds = taskDef.getPollTimeoutSeconds().longValue() + 1; + } else { + candidateSeconds = + (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) + ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() + + 1 + : workflowOffsetTimeout; + } + } else { + candidateSeconds = + (workflowModel.getWorkflowDefinition().getTimeoutSeconds() != 0) + ? workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1 + : workflowOffsetTimeout; + } } - taskModel.setExecuted(false); - executionDAO.updateWorkflow(workflow); - } - } - private void ensureWorkflowExistsInDecider(String workflowId) { - String queueName = Utils.DECIDER_QUEUE; - if (!queueDAO.containsMessage(queueName, workflowId)) { - queueDAO.push(queueName, workflowId, worflowOffsetTimeout.getSeconds()); - Monitors.recordQueueMessageRepushFromRepairService(queueName); + if (candidateSeconds == null) { + continue; + } + if (candidateSeconds < 0) { + candidateSeconds = 0L; + } + // Cap every candidate to avoid excessive unack delay. + if (candidateSeconds > maxPostponeSeconds) { + candidateSeconds = maxPostponeSeconds; + } + if (postponeDurationSeconds == null || candidateSeconds < postponeDurationSeconds) { + postponeDurationSeconds = candidateSeconds; + } } + // Default to immediate re-evaluation if no eligible task delay is found. + long unackSeconds = (postponeDurationSeconds != null) ? postponeDurationSeconds : 0; + queueDAO.setUnackTimeout(DECIDER_QUEUE, workflowModel.getWorkflowId(), unackSeconds * 1000); } - @Override - public void doStop() { - stop.set(true); - ((ExecutorService) this.sweeperExecutor).shutdownNow(); + /** + * jitter will be +- (1/3) workflowOffsetTimeout for example, if workflowOffsetTimeout is 45 + * seconds, this function returns values between [30-60] seconds + * + * @param workflowOffsetTimeout + * @return + */ + @VisibleForTesting + long workflowOffsetWithJitter(long workflowOffsetTimeout) { + long range = workflowOffsetTimeout / 3; + long jitter = new Random().nextInt((int) (2 * range + 1)) - range; + return workflowOffsetTimeout + jitter; } } From e558781d3fad9e85bad14751464f9fd50e1ab814 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:19:11 +0530 Subject: [PATCH 23/39] Remove ExecutorUtils from com.netflix package - only needed in org.conductoross --- .../core/execution/ExecutorUtils.java | 123 ------------------ 1 file changed, 123 deletions(-) delete mode 100644 core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java diff --git a/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java deleted file mode 100644 index 2720044ca2..0000000000 --- a/core/src/main/java/com/netflix/conductor/core/execution/ExecutorUtils.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.execution; - -import java.time.Duration; -import java.util.Random; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import lombok.extern.slf4j.Slf4j; - -import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; - -@Slf4j -public class ExecutorUtils { - - public static Duration computePostpone( - WorkflowModel workflowModel, - Duration workflowOffsetTimeout, - Duration maxPostponeDuration) { - long currentTimeMillis = System.currentTimeMillis(); - long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); - long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); - - long postponeDurationSeconds = 0; - for (TaskModel taskModel : workflowModel.getTasks()) { - TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); - if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { - if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { - if (taskModel.getWaitTimeout() != 0) { - long deltaInSeconds = - (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; - postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; - } - } else if (taskModel.getTaskType().equals("HUMAN")) { - // HUMAN tasks always use the default workflowOffsetTimeout - // This matches the old sweeper behavior - postponeDurationSeconds = workflowOffsetTimeoutSeconds; - } else { - // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... - // not sure but keeping it this way just in case. - long responseTimeoutSeconds = - taskDef != null - ? taskDef.getResponseTimeoutSeconds() - : taskModel.getResponseTimeoutSeconds(); - if (responseTimeoutSeconds != 0) { - long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; - if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { - postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; - } - } - } - break; - } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { - if (taskDef != null) { - postponeDurationSeconds = getMinTimeout(taskDef); - } - - if (postponeDurationSeconds <= 0) { - if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { - postponeDurationSeconds = - workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; - } - } - break; - } - } - log.trace( - "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", - postponeDurationSeconds, - workflowOffsetTimeoutSeconds, - workflowModel.getWorkflowId()); - - // Cap postpone duration to the configured maximum (matches old sweeper behavior) - if (postponeDurationSeconds > maxPostponeDurationSeconds) { - postponeDurationSeconds = maxPostponeDurationSeconds; - } - - if (postponeDurationSeconds > 0) { - long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); - return Duration.ofSeconds(applyJitter(capped)); - } - return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); - } - - private static long getMinTimeout(TaskDef taskDef) { - long pollTimeoutSeconds = - taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; - return Math.min( - taskDef.getTimeoutSeconds(), - Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); - } - - /** - * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the - * duration. - * - *

For example, if duration is 45 seconds, returns value between [30-60] seconds. - * - * @param durationSeconds postpone duration in seconds - * @return duration with jitter applied - */ - public static long applyJitter(long durationSeconds) { - if (durationSeconds <= 3) { - return durationSeconds; // Too small for jitter - } - long range = durationSeconds / 3; - long jitter = new Random().nextInt((int) (2 * range + 1)) - range; - return durationSeconds + jitter; - } -} From 5a4de9a171e05d02623e437f0a5838231e98c05f Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:20:05 +0530 Subject: [PATCH 24/39] Fix TestExecutorUtils to import ExecutorUtils from org.conductoross package --- .../com/netflix/conductor/core/execution/TestExecutorUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java index 226aadf0d9..106279e264 100644 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java +++ b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java @@ -25,6 +25,8 @@ import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; +import org.conductoross.conductor.core.execution.ExecutorUtils; + import static org.junit.Assert.*; /** From 3b4c88c8e6562548164dc74b75040ac08d0a514e Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:21:30 +0530 Subject: [PATCH 25/39] Remove ExecutorUtils - not needed for OrkesWorkflowSweeper parity --- .../core/execution/ExecutorUtils.java | 123 ------ .../core/execution/TestExecutorUtils.java | 376 ------------------ 2 files changed, 499 deletions(-) delete mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java delete mode 100644 core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java deleted file mode 100644 index 5a2904739b..0000000000 --- a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package org.conductoross.conductor.core.execution; - -import java.time.Duration; -import java.util.Random; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import lombok.extern.slf4j.Slf4j; - -import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; - -@Slf4j -public class ExecutorUtils { - - public static Duration computePostpone( - WorkflowModel workflowModel, - Duration workflowOffsetTimeout, - Duration maxPostponeDuration) { - long currentTimeMillis = System.currentTimeMillis(); - long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); - long maxPostponeDurationSeconds = maxPostponeDuration.getSeconds(); - - long postponeDurationSeconds = 0; - for (TaskModel taskModel : workflowModel.getTasks()) { - TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); - if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { - if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { - if (taskModel.getWaitTimeout() != 0) { - long deltaInSeconds = - (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; - postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; - } - } else if (taskModel.getTaskType().equals("HUMAN")) { - // HUMAN tasks always use the default workflowOffsetTimeout - // This matches the old sweeper behavior - postponeDurationSeconds = workflowOffsetTimeoutSeconds; - } else { - // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... - // not sure but keeping it this way just in case. - long responseTimeoutSeconds = - taskDef != null - ? taskDef.getResponseTimeoutSeconds() - : taskModel.getResponseTimeoutSeconds(); - if (responseTimeoutSeconds != 0) { - long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; - if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { - postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; - } - } - } - break; - } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { - if (taskDef != null) { - postponeDurationSeconds = getMinTimeout(taskDef); - } - - if (postponeDurationSeconds <= 0) { - if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { - postponeDurationSeconds = - workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; - } - } - break; - } - } - log.trace( - "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", - postponeDurationSeconds, - workflowOffsetTimeoutSeconds, - workflowModel.getWorkflowId()); - - // Cap postpone duration to the configured maximum (matches old sweeper behavior) - if (postponeDurationSeconds > maxPostponeDurationSeconds) { - postponeDurationSeconds = maxPostponeDurationSeconds; - } - - if (postponeDurationSeconds > 0) { - long capped = Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds); - return Duration.ofSeconds(applyJitter(capped)); - } - return Duration.ofSeconds(applyJitter(workflowOffsetTimeoutSeconds)); - } - - private static long getMinTimeout(TaskDef taskDef) { - long pollTimeoutSeconds = - taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; - return Math.min( - taskDef.getTimeoutSeconds(), - Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); - } - - /** - * Apply jitter to postpone duration to prevent thundering herd. Jitter will be +- (1/3) of the - * duration. - * - *

For example, if duration is 45 seconds, returns value between [30-60] seconds. - * - * @param durationSeconds postpone duration in seconds - * @return duration with jitter applied - */ - public static long applyJitter(long durationSeconds) { - if (durationSeconds <= 3) { - return durationSeconds; // Too small for jitter - } - long range = durationSeconds / 3; - long jitter = new Random().nextInt((int) (2 * range + 1)) - range; - return durationSeconds + jitter; - } -} diff --git a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java b/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java deleted file mode 100644 index 106279e264..0000000000 --- a/core/src/test/java/com/netflix/conductor/core/execution/TestExecutorUtils.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.execution; - -import java.time.Duration; -import java.util.List; - -import org.junit.Before; -import org.junit.Test; - -import com.netflix.conductor.common.metadata.tasks.TaskDef; -import com.netflix.conductor.common.metadata.tasks.TaskType; -import com.netflix.conductor.common.metadata.workflow.WorkflowDef; -import com.netflix.conductor.common.metadata.workflow.WorkflowTask; -import com.netflix.conductor.model.TaskModel; -import com.netflix.conductor.model.WorkflowModel; - -import org.conductoross.conductor.core.execution.ExecutorUtils; - -import static org.junit.Assert.*; - -/** - * Unit tests for ExecutorUtils.computePostpone() to ensure correct postpone duration calculation - * for workflow rescheduling in the decider queue. - */ -public class TestExecutorUtils { - - private WorkflowModel workflow; - private WorkflowDef workflowDef; - private Duration workflowOffsetTimeout; - private Duration maxPostponeDuration; - - @Before - public void setup() { - workflowDef = new WorkflowDef(); - workflowDef.setName("test_workflow"); - workflowDef.setTimeoutSeconds(0); - - workflow = new WorkflowModel(); - workflow.setWorkflowDefinition(workflowDef); - workflow.setWorkflowId("test-workflow-id"); - - workflowOffsetTimeout = Duration.ofSeconds(30); - maxPostponeDuration = Duration.ofSeconds(300); - } - - @Test - public void testComputePostpone_WaitTaskWithTimeout() { - // Create a WAIT task with specific wait timeout - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - long futureTime = System.currentTimeMillis() + 60000; // 60 seconds in future - waitTask.setWaitTimeout(futureTime); - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be capped at workflowOffsetTimeout (30 seconds) with jitter - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_WaitTaskWithZeroTimeout() { - // Create a WAIT task without specific timeout - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(0); - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return the default workflow offset timeout (30 seconds, with jitter) - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_HumanTask() { - // HUMAN tasks should always use default workflowOffsetTimeout - TaskModel humanTask = new TaskModel(); - humanTask.setTaskType("HUMAN"); - humanTask.setStatus(TaskModel.Status.IN_PROGRESS); - humanTask.setStartTime(System.currentTimeMillis()); - - workflow.setTasks(List.of(humanTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return the default workflow offset timeout (30 seconds, with jitter) - // Jitter can be ±1/3, so 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_InProgressTaskWithResponseTimeout() { - // Task with response timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(120L); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis() - 10000); // Started 10 seconds ago - task.setResponseTimeoutSeconds(120); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be approximately 120 - 10 + 1 = 111 seconds, but capped at workflowOffsetTimeout - // (30s) - // Then jitter: 30 ± 10 = [20, 40] seconds - assertTrue( - "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() >= 20); - assertTrue( - "Postpone should be capped at workflow offset (20-40s)", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ScheduledTaskWithPollTimeout() { - // Task definition with poll timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setPollTimeoutSeconds(45); - taskDef.setResponseTimeoutSeconds(300); - taskDef.setTimeoutSeconds(600); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.SCHEDULED); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use minimum of pollTimeout, responseTimeout, timeout = 45 seconds (with jitter) - // But capped at workflowOffsetTimeout (30 seconds) - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue( - "Postpone should be between 20-40 seconds (capped at workflow offset)", - result.getSeconds() >= 20); - assertTrue( - "Postpone should be between 20-40 seconds (capped at workflow offset)", - result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ScheduledTaskWithWorkflowTimeout() { - // Workflow with timeout, task without specific timeout - workflowDef.setTimeoutSeconds(180); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.SCHEDULED); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should be capped at workflowOffsetTimeout (30 seconds with jitter) - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_MaxPostponeCapping() { - // Test that postpone is capped at maxPostponeDuration - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(1000L); // Very large timeout - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis()); - task.setResponseTimeoutSeconds(1000); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration shortMaxPostpone = Duration.ofSeconds(60); - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, shortMaxPostpone); - - // Should be capped at workflowOffsetTimeout (30 seconds) which is smaller - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_NoTasksUsesDefault() { - // No tasks in workflow - workflow.setTasks(List.of()); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return workflowOffsetTimeout with jitter - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_OnlyTerminalTasks() { - // Only terminal tasks should use default - TaskModel completedTask = new TaskModel(); - completedTask.setTaskType("SIMPLE"); - completedTask.setStatus(TaskModel.Status.COMPLETED); - - TaskModel failedTask = new TaskModel(); - failedTask.setTaskType("SIMPLE"); - failedTask.setStatus(TaskModel.Status.FAILED); - - workflow.setTasks(List.of(completedTask, failedTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should return workflowOffsetTimeout with jitter - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() >= 20); - assertTrue("Postpone should be between 20-40 seconds", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_FirstNonTerminalTaskIsUsed() { - // When multiple tasks exist, should use first non-terminal task - TaskModel completedTask = new TaskModel(); - completedTask.setTaskType("SIMPLE"); - completedTask.setStatus(TaskModel.Status.COMPLETED); - - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(System.currentTimeMillis() + 45000); // 45 seconds - - TaskModel scheduledTask = new TaskModel(); - scheduledTask.setTaskType("SIMPLE"); - scheduledTask.setStatus(TaskModel.Status.SCHEDULED); - - workflow.setTasks(List.of(completedTask, waitTask, scheduledTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use the WAIT task (first non-terminal), capped at workflowOffsetTimeout - // Jitter: 30 ± 10 = [20, 40] seconds - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() >= 20); - assertTrue("Postpone should be capped at workflow offset", result.getSeconds() <= 40); - } - - @Test - public void testApplyJitter_SmallDuration() { - // Small durations (<=3) should not have jitter - long result = ExecutorUtils.applyJitter(3); - assertEquals("Small duration should not have jitter", 3, result); - - result = ExecutorUtils.applyJitter(2); - assertEquals("Small duration should not have jitter", 2, result); - - result = ExecutorUtils.applyJitter(1); - assertEquals("Small duration should not have jitter", 1, result); - } - - @Test - public void testApplyJitter_LargeDuration() { - // Test jitter is within bounds for larger duration - long duration = 90; - long result = ExecutorUtils.applyJitter(duration); - - // Jitter should be ±1/3, so 90 ± 30 = [60, 120] - assertTrue("Jitter should be within bounds", result >= 60); - assertTrue("Jitter should be within bounds", result <= 120); - } - - @Test - public void testApplyJitter_MultipleCallsVary() { - // Verify jitter actually produces different values (randomness check) - long duration = 60; - boolean foundDifferent = false; - - for (int i = 0; i < 100; i++) { - long result = ExecutorUtils.applyJitter(duration); - if (result != duration) { - foundDifferent = true; - break; - } - } - - assertTrue( - "Jitter should produce varied results over multiple calls (within range)", - foundDifferent); - } - - @Test - public void testComputePostpone_PastWaitTimeout() { - // WAIT task with timeout in the past should return 0 - TaskModel waitTask = new TaskModel(); - waitTask.setTaskType(TaskType.TASK_TYPE_WAIT); - waitTask.setStatus(TaskModel.Status.IN_PROGRESS); - waitTask.setWaitTimeout(System.currentTimeMillis() - 10000); // 10 seconds in past - - workflow.setTasks(List.of(waitTask)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use workflowOffsetTimeout since calculated postpone is 0 - assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); - assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); - } - - @Test - public void testComputePostpone_ResponseTimeoutAlreadyExceeded() { - // Task that has already exceeded its response timeout - TaskDef taskDef = new TaskDef(); - taskDef.setName("test_task"); - taskDef.setResponseTimeoutSeconds(30L); - - WorkflowTask workflowTask = new WorkflowTask(); - workflowTask.setTaskDefinition(taskDef); - - TaskModel task = new TaskModel(); - task.setTaskType("SIMPLE"); - task.setStatus(TaskModel.Status.IN_PROGRESS); - task.setStartTime(System.currentTimeMillis() - 60000); // Started 60 seconds ago - task.setResponseTimeoutSeconds(30); - task.setWorkflowTask(workflowTask); - - workflow.setTasks(List.of(task)); - - Duration result = - ExecutorUtils.computePostpone(workflow, workflowOffsetTimeout, maxPostponeDuration); - - // Should use default workflowOffsetTimeout - assertTrue("Postpone should use default timeout", result.getSeconds() >= 20); - assertTrue("Postpone should use default timeout", result.getSeconds() <= 40); - } -} From 7f208476f4c34d5c39cbd9320aa4733bc182f49c Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:25:43 +0530 Subject: [PATCH 26/39] Remove unused sweeper-related code: SweeperProperties in com.netflix, revert WorkflowExecutorOps and TestWorkflowSweeper --- .../core/execution/WorkflowExecutorOps.java | 25 +- .../reconciliation/SweeperProperties.java | 30 -- .../reconciliation/TestWorkflowSweeper.java | 436 +++++++++++++----- 3 files changed, 333 insertions(+), 158 deletions(-) delete mode 100644 core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java diff --git a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java index 01025f8a0e..6afe4f0e07 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/WorkflowExecutorOps.java @@ -54,10 +54,11 @@ import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; -import static com.netflix.conductor.core.execution.ExecutorUtils.computePostpone; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; import static com.netflix.conductor.model.TaskModel.Status.*; +import static org.conductoross.conductor.core.execution.ExecutorUtils.computePostpone; + /** Workflow services provider interface */ @Trace @Component @@ -1148,16 +1149,18 @@ private WorkflowModel decide(WorkflowModel workflow) { Duration timeout = properties.getWorkflowOffsetTimeout(); if (!workflow.getStatus().isTerminal()) { - Duration updatedOffset = - computePostpone( - workflow, timeout, properties.getMaxPostponeDurationSeconds()); - // Always reschedule non-terminal workflows to ensure they remain in the queue - LOGGER.debug( - "Rescheduling workflow {} to decider queue with postpone duration {} seconds", - workflow.getWorkflowId(), - updatedOffset.getSeconds()); - queueDAO.setUnackTimeout( - DECIDER_QUEUE, workflow.getWorkflowId(), updatedOffset.getSeconds() * 1000); + Duration updatedOffset = computePostpone(workflow, timeout); + if (updatedOffset.getSeconds() != timeout.getSeconds()) { + // we have a new value, setUnack uses time in millis + LOGGER.debug( + "Pushing the workflow {} into decider queue by {} millis", + workflow.getWorkflowId(), + updatedOffset.getSeconds() * 1000); + queueDAO.setUnackTimeout( + DECIDER_QUEUE, + workflow.getWorkflowId(), + updatedOffset.getSeconds() * 1000); + } } return workflow; diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java deleted file mode 100644 index 12af4ce025..0000000000 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/SweeperProperties.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2025 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package com.netflix.conductor.core.reconciliation; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Configuration; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -@Configuration -@ConfigurationProperties("conductor.app.sweeper") -@Getter -@Setter -@ToString -public class SweeperProperties { - private int sweepBatchSize = 2; - private int queuePopTimeout = 100; -} diff --git a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java index 44be365b46..e8689dc0f5 100644 --- a/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java +++ b/core/src/test/java/com/netflix/conductor/core/reconciliation/TestWorkflowSweeper.java @@ -14,212 +14,414 @@ import java.time.Duration; import java.util.List; -import java.util.concurrent.Executor; +import java.util.Optional; import org.junit.Before; import org.junit.Test; +import com.netflix.conductor.common.metadata.tasks.TaskDef; import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; import com.netflix.conductor.core.config.ConductorProperties; +import com.netflix.conductor.core.dal.ExecutionDAOFacade; import com.netflix.conductor.core.execution.WorkflowExecutor; -import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; -import com.netflix.conductor.dao.ExecutionDAO; import com.netflix.conductor.dao.QueueDAO; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; - -import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.conductor.service.ExecutionLockService; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class TestWorkflowSweeper { private ConductorProperties properties; - private SweeperProperties sweeperProperties; private WorkflowExecutor workflowExecutor; + private WorkflowRepairService workflowRepairService; private QueueDAO queueDAO; - private ExecutionDAO executionDAO; + private ExecutionDAOFacade executionDAOFacade; private WorkflowSweeper workflowSweeper; - private SystemTaskRegistry systemTaskRegistry; - private ObjectMapper objectMapper; - private Executor executor; + private ExecutionLockService executionLockService; + + private int defaultPostPoneOffSetSeconds = 1800; + private int defaulMmaxPostponeDurationSeconds = 2000000; @Before public void setUp() { properties = mock(ConductorProperties.class); - sweeperProperties = mock(SweeperProperties.class); workflowExecutor = mock(WorkflowExecutor.class); queueDAO = mock(QueueDAO.class); - executionDAO = mock(ExecutionDAO.class); - systemTaskRegistry = mock(SystemTaskRegistry.class); - objectMapper = new ObjectMapper(); - executor = mock(Executor.class); - - when(properties.getMaxPostponeDurationSeconds()).thenReturn(Duration.ofSeconds(2000000)); - when(properties.getSweeperThreadCount()).thenReturn(1); - when(properties.getWorkflowOffsetTimeout()).thenReturn(Duration.ofSeconds(1800)); - when(properties.getLockLeaseTime()).thenReturn(Duration.ofSeconds(60)); - when(sweeperProperties.getSweepBatchSize()).thenReturn(10); - when(sweeperProperties.getQueuePopTimeout()).thenReturn(100); - + workflowRepairService = mock(WorkflowRepairService.class); + executionDAOFacade = mock(ExecutionDAOFacade.class); + executionLockService = mock(ExecutionLockService.class); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); workflowSweeper = new WorkflowSweeper( - executor, - queueDAO, workflowExecutor, - executionDAO, + Optional.of(workflowRepairService), properties, - sweeperProperties, - systemTaskRegistry, - objectMapper); + queueDAO, + executionDAOFacade, + executionLockService); } @Test - public void testSweepTerminalWorkflow() { + public void testPostponeDurationForHumanTaskType() { WorkflowModel workflowModel = new WorkflowModel(); workflowModel.setWorkflowId("1"); - workflowModel.setStatus(WorkflowModel.Status.COMPLETED); - - when(workflowExecutor.getWorkflow("1", true)).thenReturn(workflowModel); - - workflowSweeper.sweep("1"); - - verify(queueDAO).remove(DECIDER_QUEUE, "1"); - verify(workflowExecutor, never()).decideWithLock(any(WorkflowModel.class)); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_HUMAN); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); } @Test - public void testSweepNullWorkflow() { - when(workflowExecutor.getWorkflow("1", true)).thenReturn(null); - - workflowSweeper.sweep("1"); - - verify(queueDAO).remove(DECIDER_QUEUE, "1"); + public void testPostponeDurationForWaitTaskType() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); } @Test - public void testSweepRunningWorkflow() { + public void testPostponeDurationForWaitTaskTypeWithLongWaitTime() { + long waitTimeout = 65845; WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("2"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - + workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); + taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); + taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("2", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - - workflowSweeper.sweep("2"); - - verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); } @Test - public void testSweepCannotGetLock() { + public void testPostponeDurationForWaitTaskTypeWithLessOneSecondWaitTime() { + long waitTimeout = 180; WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("3"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - + workflowModel.setWorkflowId("1"); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); + taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); taskModel.setStatus(Status.IN_PROGRESS); + taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); + } - when(workflowExecutor.getWorkflow("3", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(null); + @Test + public void testPostponeDurationForWaitTaskTypeWithZeroWaitTime() { + long waitTimeout = 0; + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_WAIT); + taskModel.setStatus(Status.IN_PROGRESS); + taskModel.setWaitTimeout(System.currentTimeMillis() + waitTimeout); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (waitTimeout / 1000) * 1000); + } - workflowSweeper.sweep("3"); + @Test + public void testPostponeDurationForTaskInProgress() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); + taskModel.setStatus(Status.IN_PROGRESS); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); + } - verify(workflowExecutor).decideWithLock(any(WorkflowModel.class)); - verify(queueDAO).push(eq(DECIDER_QUEUE), eq("3"), eq(0), anyLong()); + @Test + public void testPostponeDurationForTaskInProgressWithResponseTimeoutSet() { + long responseTimeout = 200; + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); + taskModel.setStatus(Status.IN_PROGRESS); + taskModel.setResponseTimeoutSeconds(responseTimeout); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (responseTimeout + 1) * 1000); } @Test - public void testSweepRepairsScheduledTask() { + public void + testPostponeDurationForTaskInProgressWithResponseTimeoutSetLongerThanMaxPostponeDuration() { + long responseTimeout = defaulMmaxPostponeDurationSeconds + 1; WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("4"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); + workflowModel.setWorkflowId("1"); + TaskModel taskModel = new TaskModel(); + taskModel.setTaskId("task1"); + taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); + taskModel.setStatus(Status.IN_PROGRESS); + taskModel.setResponseTimeoutSeconds(responseTimeout); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(defaulMmaxPostponeDurationSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaulMmaxPostponeDurationSeconds * 1000L); + } + @Test + public void testPostponeDurationForTaskInScheduled() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + WorkflowDef workflowDef = new WorkflowDef(); + workflowModel.setWorkflowDefinition(workflowDef); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); + taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.SCHEDULED); + taskModel.setReferenceTaskName("task1"); workflowModel.setTasks(List.of(taskModel)); - - when(workflowExecutor.getWorkflow("4", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); - when(queueDAO.containsMessage(anyString(), anyString())).thenReturn(false); - - workflowSweeper.sweep("4"); - - verify(queueDAO).push(anyString(), eq("task1"), anyLong()); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); } @Test - public void testSweepDoesNotRepairTaskAlreadyInQueue() { + public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSet() { + long workflowTimeout = 1800; WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("5"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); - + workflowModel.setWorkflowId("1"); + WorkflowDef workflowDef = new WorkflowDef(); + workflowDef.setTimeoutSeconds(workflowTimeout); + workflowModel.setWorkflowDefinition(workflowDef); TaskModel taskModel = new TaskModel(); taskModel.setTaskId("task1"); - taskModel.setReferenceTaskName("task1"); - taskModel.setTaskType("SIMPLE"); + taskModel.setTaskType(TaskType.TASK_TYPE_SIMPLE); taskModel.setStatus(Status.SCHEDULED); workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); + } - when(workflowExecutor.getWorkflow("5", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(systemTaskRegistry.isSystemTask("SIMPLE")).thenReturn(false); - when(queueDAO.containsMessage(anyString(), eq("task1"))).thenReturn(true); + @Test + public void testPostponeDurationForTaskInScheduledWithWorkflowTimeoutSetAndNoPollTimeout() { + long workflowTimeout = 1800; + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + WorkflowDef workflowDef = new WorkflowDef(); + workflowDef.setTimeoutSeconds(workflowTimeout); + workflowModel.setWorkflowDefinition(workflowDef); + TaskDef taskDef = new TaskDef(); + TaskModel taskModel = mock(TaskModel.class); + workflowModel.setTasks(List.of(taskModel)); + when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (workflowTimeout + 1) * 1000); + } - workflowSweeper.sweep("5"); + @Test + public void testPostponeDurationForTaskInScheduledWithNoWorkflowTimeoutSetAndNoPollTimeout() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + WorkflowDef workflowDef = new WorkflowDef(); + workflowModel.setWorkflowDefinition(workflowDef); + TaskDef taskDef = new TaskDef(); + TaskModel taskModel = mock(TaskModel.class); + workflowModel.setTasks(List.of(taskModel)); + when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); + } - verify(queueDAO, never()).push(anyString(), eq("task1"), anyLong()); + @Test + public void testPostponeDurationForTaskInScheduledWithNoPollTimeoutSet() { + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskDef taskDef = new TaskDef(); + WorkflowDef workflowDef = new WorkflowDef(); + workflowModel.setWorkflowDefinition(workflowDef); + TaskModel taskModel = mock(TaskModel.class); + workflowModel.setTasks(List.of(taskModel)); + when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); + when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, + workflowModel.getWorkflowId(), + defaultPostPoneOffSetSeconds * 1000); } @Test - public void testSweepSubWorkflowTask() { + public void testPostponeDurationForTaskInScheduledWithPollTimeoutSet() { + int pollTimeout = 200; WorkflowModel workflowModel = new WorkflowModel(); - workflowModel.setWorkflowId("6"); - workflowModel.setStatus(WorkflowModel.Status.RUNNING); + workflowModel.setWorkflowId("1"); + TaskDef taskDef = new TaskDef(); + taskDef.setPollTimeoutSeconds(pollTimeout); + TaskModel taskModel = mock(TaskModel.class); + ; + workflowModel.setTasks(List.of(taskModel)); + when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); + when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (pollTimeout + 1) * 1000); + } - TaskModel subWorkflowTask = new TaskModel(); - subWorkflowTask.setTaskId("task1"); - subWorkflowTask.setReferenceTaskName("task1"); - subWorkflowTask.setTaskType(TaskType.TASK_TYPE_SUB_WORKFLOW); - subWorkflowTask.setStatus(Status.IN_PROGRESS); - subWorkflowTask.setSubWorkflowId("sub-wf-1"); - workflowModel.setTasks(List.of(subWorkflowTask)); + @Test + public void testPostponeDurationChoosesMinimumAcrossTasks() { + long responseTimeout = 500; + int pollTimeout = 120; + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskModel inProgressTask = new TaskModel(); + inProgressTask.setTaskId("task1"); + inProgressTask.setTaskType(TaskType.TASK_TYPE_SIMPLE); + inProgressTask.setStatus(Status.IN_PROGRESS); + inProgressTask.setResponseTimeoutSeconds(responseTimeout); + TaskDef taskDef = new TaskDef(); + taskDef.setPollTimeoutSeconds(pollTimeout); + TaskModel scheduledTask = mock(TaskModel.class); + when(scheduledTask.getStatus()).thenReturn(Status.SCHEDULED); + when(scheduledTask.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + workflowModel.setTasks(List.of(inProgressTask, scheduledTask)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); + + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), (pollTimeout + 1) * 1000L); + } - WorkflowModel subWorkflow = new WorkflowModel(); - subWorkflow.setWorkflowId("sub-wf-1"); - subWorkflow.setStatus(WorkflowModel.Status.COMPLETED); + @Test + public void testPostponeDurationForScheduledTaskCappedByMaxPostpone() { + int pollTimeout = 1000; + int maxPostponeSeconds = 100; + WorkflowModel workflowModel = new WorkflowModel(); + workflowModel.setWorkflowId("1"); + TaskDef taskDef = new TaskDef(); + taskDef.setPollTimeoutSeconds(pollTimeout); + TaskModel taskModel = mock(TaskModel.class); + when(taskModel.getStatus()).thenReturn(Status.SCHEDULED); + when(taskModel.getTaskDefinition()).thenReturn(Optional.of(taskDef)); + workflowModel.setTasks(List.of(taskModel)); + when(properties.getWorkflowOffsetTimeout()) + .thenReturn(Duration.ofSeconds(defaultPostPoneOffSetSeconds)); + when(properties.getMaxPostponeDurationSeconds()) + .thenReturn(Duration.ofSeconds(maxPostponeSeconds)); - when(workflowExecutor.getWorkflow("6", true)).thenReturn(workflowModel); - when(workflowExecutor.decideWithLock(any(WorkflowModel.class))).thenReturn(workflowModel); - when(executionDAO.getWorkflow("sub-wf-1", false)).thenReturn(subWorkflow); + workflowSweeper.unack(workflowModel, defaultPostPoneOffSetSeconds); - workflowSweeper.sweep("6"); + verify(queueDAO) + .setUnackTimeout( + DECIDER_QUEUE, workflowModel.getWorkflowId(), maxPostponeSeconds * 1000L); + } - verify(executionDAO).updateTask(subWorkflowTask); + @Test + public void testWorkflowOffsetJitter() { + long offset = 45; + for (int i = 0; i < 10; i++) { + long offsetWithJitter = workflowSweeper.workflowOffsetWithJitter(offset); + assertTrue(offsetWithJitter >= 30); + assertTrue(offsetWithJitter <= 60); + } } } From d18d4547fadf9063c616be2b5eece0ee68859ae1 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:23:17 +0530 Subject: [PATCH 27/39] Remove all unrelated changes - keep only sweeper changes --- server/src/main/resources/application.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 8c8481bb5d..960508f05f 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -181,6 +181,7 @@ management.cloudwatch.metrics.export.namespace=conductor management.azuremonitor.metrics.export.enabled=false management.azuremonitor.metrics.export.instrumentationKey=INSTRUMENTATION_KEY +management.jmx.metrics.export.enabled=false # When enabled logs metrics as info level logs -conductor.metrics-logger.enablnoed=false \ No newline at end of file +conductor.metrics-logger.enabled=false \ No newline at end of file From b93eab4cf4ceb13163dec61a92dd75288d9254ae Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 00:36:29 +0530 Subject: [PATCH 28/39] Restore ExecutorUtils - needed by WorkflowExecutorOps --- .../core/execution/ExecutorUtils.java | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java new file mode 100644 index 0000000000..bbdaa7be19 --- /dev/null +++ b/core/src/main/java/org/conductoross/conductor/core/execution/ExecutorUtils.java @@ -0,0 +1,92 @@ +/* + * Copyright 2025 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.conductoross.conductor.core.execution; + +import java.time.Duration; + +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.model.WorkflowModel; + +import lombok.extern.slf4j.Slf4j; + +import static com.netflix.conductor.common.metadata.tasks.TaskType.TASK_TYPE_WAIT; + +@Slf4j +public class ExecutorUtils { + + public static Duration computePostpone( + WorkflowModel workflowModel, Duration workflowOffsetTimeout) { + long currentTimeMillis = System.currentTimeMillis(); + long workflowOffsetTimeoutSeconds = workflowOffsetTimeout.getSeconds(); + + long postponeDurationSeconds = 0; + for (TaskModel taskModel : workflowModel.getTasks()) { + TaskDef taskDef = taskModel.getTaskDefinition().orElse(null); + if (taskModel.getStatus() == TaskModel.Status.IN_PROGRESS) { + if (taskModel.getTaskType().equals(TASK_TYPE_WAIT)) { + if (taskModel.getWaitTimeout() != 0) { + long deltaInSeconds = + (taskModel.getWaitTimeout() - currentTimeMillis) / 1000; + postponeDurationSeconds = (deltaInSeconds > 0) ? deltaInSeconds : 0; + } + } else { + // Could taskModel.getResponseTimeoutSeconds() be set and no taskDef?... + // not sure but keeping it this way just in case. + long responseTimeoutSeconds = + taskDef != null + ? taskDef.getResponseTimeoutSeconds() + : taskModel.getResponseTimeoutSeconds(); + if (responseTimeoutSeconds != 0) { + long deltaInSeconds = (currentTimeMillis - taskModel.getStartTime()) / 1000; + if (deltaInSeconds > 0 && (responseTimeoutSeconds - deltaInSeconds) >= 0) { + postponeDurationSeconds = responseTimeoutSeconds - deltaInSeconds + 1; + } + } + } + break; + } else if (taskModel.getStatus() == TaskModel.Status.SCHEDULED) { + if (taskDef != null) { + postponeDurationSeconds = getMinTimeout(taskDef); + } + + if (postponeDurationSeconds <= 0) { + if (workflowModel.getWorkflowDefinition().getTimeoutSeconds() > 0) { + postponeDurationSeconds = + workflowModel.getWorkflowDefinition().getTimeoutSeconds() + 1; + } + } + break; + } + } + log.trace( + "postponeDurationSeconds calculated is {} and workflowOffsetTimeoutSeconds is {} for workflow {}", + postponeDurationSeconds, + workflowOffsetTimeoutSeconds, + workflowModel.getWorkflowId()); + + if (postponeDurationSeconds > 0) { + return Duration.ofSeconds( + Math.min(postponeDurationSeconds, workflowOffsetTimeoutSeconds)); + } + return Duration.ofSeconds(workflowOffsetTimeoutSeconds); + } + + private static long getMinTimeout(TaskDef taskDef) { + long pollTimeoutSeconds = + taskDef.getPollTimeoutSeconds() != null ? taskDef.getPollTimeoutSeconds() : 0; + return Math.min( + taskDef.getTimeoutSeconds(), + Math.min(taskDef.getResponseTimeoutSeconds(), pollTimeoutSeconds)); + } +} From 454fb8fd113fc4e136f7940cab59f40ba504fb57 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 11:44:54 +0530 Subject: [PATCH 29/39] Add explicit lock management to WorkflowSweeper - Added ExecutionLockService with explicit acquireLock/releaseLock - Changed from decideWithLock() to decide() after acquiring lock - Lock is now acquired at start of sweep() and released in finally block - Aligns OSS WorkflowSweeper locking pattern with OrkesWorkflowSweeper - Makes OSS the common base implementation for both OSS and Orkes This fix ensures workflows are properly locked before running decide, preventing race conditions and concurrent modifications. Co-Authored-By: Claude Sonnet 4.5 --- .../core/execution/WorkflowSweeper.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java index d2da65f0d6..783f1fd115 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -27,7 +27,6 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import com.netflix.conductor.common.metadata.tasks.TaskType; import com.netflix.conductor.core.LifecycleAwareComponent; import com.netflix.conductor.core.config.ConductorProperties; import com.netflix.conductor.core.execution.WorkflowExecutor; @@ -40,6 +39,7 @@ import com.netflix.conductor.metrics.Monitors; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; +import com.netflix.conductor.service.ExecutionLockService; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @@ -64,6 +64,7 @@ public class WorkflowSweeper extends LifecycleAwareComponent { private final ConductorProperties properties; private final ObjectMapper objectMapper; private SystemTaskRegistry systemTaskRegistry; + private final ExecutionLockService executionLockService; private final Clock clock = Clock.systemDefaultZone(); private AtomicBoolean stop = new AtomicBoolean(false); @@ -75,7 +76,8 @@ public WorkflowSweeper( ConductorProperties properties, SweeperProperties sweeperProperties, SystemTaskRegistry systemTaskRegistry, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + ExecutionLockService executionLockService) { this.queueDAO = queueDAO; this.executionDAO = executionDAO; this.sweeperProperties = sweeperProperties; @@ -85,6 +87,7 @@ public WorkflowSweeper( this.properties = properties; this.systemTaskRegistry = systemTaskRegistry; this.objectMapper = objectMapper; + this.executionLockService = executionLockService; log.info("Initializing sweeper with {} threads", properties.getSweeperThreadCount()); for (int i = 0; i < properties.getSweeperThreadCount(); i++) { sweeperExecutor.execute(this::pollAndSweep); @@ -154,6 +157,10 @@ public CompletableFuture sweepAsync(String workflowId) { } public void sweep(String workflowId) { + if (!executionLockService.acquireLock(workflowId)) { + log.error("Couldn't acquire lock to sweep workflow {}", workflowId); + return; + } log.info("Running sweeper for workflow {}", workflowId); try { @@ -168,13 +175,9 @@ public void sweep(String workflowId) { .map(t -> t.getReferenceTaskName() + ":" + t.getStatus()) .toList() .toString(); - workflow = workflowExecutor.decideWithLock(workflow); + workflow = workflowExecutor.decide(workflowId); if (workflow == null) { - // couldn't get a lock - // Let's try again... with the lockTime timeout / 2 - int backoff = (int) (properties.getLockLeaseTime().toMillis() / 2); - log.info("can't get a lock on {}, will try after {} ms", workflowId, backoff); - queueDAO.push(DECIDER_QUEUE, workflowId, 0, Duration.ofMillis(backoff).toSeconds()); + log.warn("Workflow {} not found after decide", workflowId); return; } if (workflow.getStatus().isTerminal()) { @@ -209,7 +212,9 @@ public void sweep(String workflowId) { String queueName = QueueUtils.getQueueName(task); if (!queueDAO.containsMessage(queueName, task.getTaskId())) { queueDAO.push( - queueName, task.getTaskId(), task.getCallbackAfterSeconds()); + queueName, + task.getTaskId(), + task.getCallbackAfterSeconds()); } }); } @@ -224,7 +229,7 @@ public void sweep(String workflowId) { // after the last task was completed // To fix, we reset the executed flag of the last task and re-run decide forceSetLastTaskAsNotExecuted(workflow); - workflow = workflowExecutor.decideWithLock(workflow); + workflow = workflowExecutor.decide(workflowId); } // If parent workflow exists, call repair on that too - meaning ensure the parent is in @@ -234,6 +239,8 @@ public void sweep(String workflowId) { } } catch (Throwable e) { log.error("Error running sweep for {}, error = {}", workflowId, e.getMessage(), e); + } finally { + executionLockService.releaseLock(workflowId); } } From 2b46e47d40c7061b65600aaaecca41a66ac48c5b Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 14:26:21 +0530 Subject: [PATCH 30/39] Fix OpenSearchProperties Environment injection for integration tests Add @Autowired annotation to setEnvironment() method to ensure Spring properly injects Environment instance. This enables legacy property fallback logic in @PostConstruct init() method during integration tests. Co-Authored-By: Claude Sonnet 4.5 --- .../com/netflix/conductor/os/config/OpenSearchProperties.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/os-persistence/src/main/java/com/netflix/conductor/os/config/OpenSearchProperties.java b/os-persistence/src/main/java/com/netflix/conductor/os/config/OpenSearchProperties.java index ef6a367cf5..b70d3dd468 100644 --- a/os-persistence/src/main/java/com/netflix/conductor/os/config/OpenSearchProperties.java +++ b/os-persistence/src/main/java/com/netflix/conductor/os/config/OpenSearchProperties.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; import org.springframework.core.env.Environment; @@ -307,6 +308,7 @@ private boolean hasNewProperty(String propertyName) { return environment != null && environment.containsProperty(NEW_PREFIX + propertyName); } + @Autowired public void setEnvironment(Environment environment) { this.environment = environment; } From d015be42fc9ec22290d9c47917de4adfe8ea6d00 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 14:44:11 +0530 Subject: [PATCH 31/39] Fix WorkflowSweeper imports in test base classes Update AbstractSpecification in both test-harness and test-util to import WorkflowSweeper from new package location org.conductoross.conductor.core.execution instead of deprecated com.netflix.conductor.core.reconciliation package. Co-Authored-By: Claude Sonnet 4.5 --- .../netflix/conductor/test/base/AbstractSpecification.groovy | 2 +- .../netflix/conductor/test/base/AbstractSpecification.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index fb4856fde6..141615f8ad 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -25,10 +25,10 @@ import com.netflix.conductor.common.metadata.workflow.WorkflowDef import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.StartWorkflowInput import com.netflix.conductor.core.execution.WorkflowExecutor -import com.netflix.conductor.core.reconciliation.WorkflowSweeper import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.netflix.conductor.test.util.WorkflowTestUtil +import org.conductoross.conductor.core.execution.WorkflowSweeper import spock.lang.Specification import spock.util.concurrent.PollingConditions diff --git a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index bd4b569483..c5aaee4d07 100644 --- a/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-util/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -22,10 +22,10 @@ import org.springframework.test.context.TestPropertySource import com.netflix.conductor.core.execution.AsyncSystemTaskExecutor import com.netflix.conductor.core.execution.WorkflowExecutor -import com.netflix.conductor.core.reconciliation.WorkflowSweeper import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.netflix.conductor.test.util.WorkflowTestUtil +import org.conductoross.conductor.core.execution.WorkflowSweeper import org.testcontainers.containers.GenericContainer import org.testcontainers.utility.DockerImageName import spock.lang.Specification From 81bd987c1397d4254eb743ca51384d25bfef02bc Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 15:22:52 +0530 Subject: [PATCH 32/39] spotless --- .../netflix/conductor/test/base/AbstractSpecification.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy index 141615f8ad..24b9bd3216 100644 --- a/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy +++ b/test-harness/src/test/groovy/com/netflix/conductor/test/base/AbstractSpecification.groovy @@ -12,6 +12,7 @@ */ package com.netflix.conductor.test.base +import org.conductoross.conductor.core.execution.WorkflowSweeper import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.DynamicPropertyRegistry @@ -28,7 +29,6 @@ import com.netflix.conductor.core.execution.WorkflowExecutor import com.netflix.conductor.service.ExecutionService import com.netflix.conductor.service.MetadataService import com.netflix.conductor.test.util.WorkflowTestUtil -import org.conductoross.conductor.core.execution.WorkflowSweeper import spock.lang.Specification import spock.util.concurrent.PollingConditions From d67357ef911620be86b2b31582004c7e506f8ec4 Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 15:42:18 +0530 Subject: [PATCH 33/39] remove async annotation --- .../conductoross/conductor/core/execution/WorkflowSweeper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java index 783f1fd115..034c857c27 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -150,7 +150,6 @@ private void pollAndSweep() { } } - @Async(SWEEPER_EXECUTOR_NAME) public CompletableFuture sweepAsync(String workflowId) { sweep(workflowId); return CompletableFuture.completedFuture(null); From fc97206cb0905b54b5b613ca61ceff46fba893bf Mon Sep 17 00:00:00 2001 From: Manan Bhatt Date: Fri, 30 Jan 2026 16:57:34 +0530 Subject: [PATCH 34/39] spotless --- .../conductoross/conductor/core/execution/WorkflowSweeper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java index 034c857c27..9cb11957e6 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/WorkflowSweeper.java @@ -24,7 +24,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import com.netflix.conductor.core.LifecycleAwareComponent; From fc888f084692f8dbac090cec89a3733dc76d0fa7 Mon Sep 17 00:00:00 2001 From: manan164 <1897158+manan164@users.noreply.github.com> Date: Fri, 30 Jan 2026 23:05:27 +0530 Subject: [PATCH 35/39] Keep legacy sweeper --- .../conductor/core/reconciliation/WorkflowSweeper.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index 45cda3d9b2..f650514b81 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -19,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Async; import com.netflix.conductor.annotations.VisibleForTesting; @@ -35,11 +36,18 @@ import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; +import org.springframework.stereotype.Component; import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; -// @Component +// Deprecated in favor of org.conductoross.conductor.core.execution.WorkflowSweeper +@Deprecated(forRemoval = true) +@Component +@ConditionalOnProperty( + name = "conductor.app.legacy.sweeper.enabled", + havingValue = "true", + matchIfMissing = true) public class WorkflowSweeper { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowSweeper.class); From e789f81e2f41d5af203bcb7c5a3998d0f93580a8 Mon Sep 17 00:00:00 2001 From: manan164 <1897158+manan164@users.noreply.github.com> Date: Fri, 30 Jan 2026 23:08:36 +0530 Subject: [PATCH 36/39] spotless --- .../netflix/conductor/core/reconciliation/WorkflowSweeper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index f650514b81..ca5713cdb2 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; import com.netflix.conductor.annotations.VisibleForTesting; import com.netflix.conductor.common.metadata.tasks.TaskDef; @@ -36,7 +37,6 @@ import com.netflix.conductor.model.TaskModel.Status; import com.netflix.conductor.model.WorkflowModel; import com.netflix.conductor.service.ExecutionLockService; -import org.springframework.stereotype.Component; import static com.netflix.conductor.core.config.SchedulerConfiguration.SWEEPER_EXECUTOR_NAME; import static com.netflix.conductor.core.utils.Utils.DECIDER_QUEUE; From 36b52d86ccd551e51113278b09551d5785fe1822 Mon Sep 17 00:00:00 2001 From: manan164 <1897158+manan164@users.noreply.github.com> Date: Fri, 30 Jan 2026 23:47:02 +0530 Subject: [PATCH 37/39] fix --- .../netflix/conductor/core/reconciliation/WorkflowSweeper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java index ca5713cdb2..2c68f8f36d 100644 --- a/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java +++ b/core/src/main/java/com/netflix/conductor/core/reconciliation/WorkflowSweeper.java @@ -47,7 +47,7 @@ @ConditionalOnProperty( name = "conductor.app.legacy.sweeper.enabled", havingValue = "true", - matchIfMissing = true) + matchIfMissing = false) public class WorkflowSweeper { private static final Logger LOGGER = LoggerFactory.getLogger(WorkflowSweeper.class); From befa98343b10ba2cc5237ab56295beb2bf1b3958 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 30 Jan 2026 20:34:55 -0800 Subject: [PATCH 38/39] removed duplicated classes --- .../AnnotatedMethodParameterMapper.java | 3 +- .../AnnotatedMethodResultMapper.java | 3 +- .../AnnotatedWorkflowSystemTask.java | 2 +- .../tasks/annotated/SampleWorkers.java | 5 ++- .../WorkerTaskAnnotationScanner.java | 2 +- .../sdk/workflow/task/InputParam.java | 26 ------------ .../sdk/workflow/task/OutputParam.java | 24 ----------- .../sdk/workflow/task/WorkerTask.java | 40 ------------------- .../TestAnnotatedMethodParameterMapper.java | 2 +- .../TestAnnotatedMethodResultMapper.java | 2 +- .../TestAnnotatedSystemTaskIntegration.java | 4 +- .../TestAnnotatedWorkflowSystemTask.java | 4 +- 12 files changed, 13 insertions(+), 104 deletions(-) delete mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java delete mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java delete mode 100644 core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java index f920659abf..6a27ee5bfe 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java @@ -22,10 +22,9 @@ import java.util.List; import java.util.Map; -import org.conductoross.conductor.sdk.workflow.task.InputParam; - import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.sdk.workflow.task.InputParam; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java index d6bc90ef3f..f6d9fbb4e6 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodResultMapper.java @@ -16,10 +16,9 @@ import java.util.List; import java.util.Map; -import org.conductoross.conductor.sdk.workflow.task.OutputParam; - import com.netflix.conductor.common.config.ObjectMapperProvider; import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.sdk.workflow.task.OutputParam; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java index 65673c4024..2f534c234a 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedWorkflowSystemTask.java @@ -15,7 +15,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import org.conductoross.conductor.sdk.workflow.task.WorkerTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +22,7 @@ import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; /** * Adapter that wraps a @WorkerTask annotated method as a WorkflowSystemTask. This enables diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java index 9aad45de56..a2446c7d3e 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java @@ -12,10 +12,11 @@ */ package org.conductoross.conductor.core.execution.tasks.annotated; -import org.conductoross.conductor.sdk.workflow.task.InputParam; -import org.conductoross.conductor.sdk.workflow.task.WorkerTask; import org.springframework.stereotype.Component; +import com.netflix.conductor.sdk.workflow.task.InputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; + @Component public class SampleWorkers { diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java index 7815afcd2e..fe9b191ad5 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/WorkerTaskAnnotationScanner.java @@ -18,7 +18,6 @@ import java.util.Set; import org.conductoross.conductor.core.execution.mapper.AnnotatedSystemTaskMapper; -import org.conductoross.conductor.sdk.workflow.task.WorkerTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -34,6 +33,7 @@ import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.ParametersUtils; import com.netflix.conductor.dao.MetadataDAO; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; /** * Spring component that scans for @WorkerTask annotated methods in Spring beans and adds them to diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java deleted file mode 100644 index 15fe99535f..0000000000 --- a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/InputParam.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package org.conductoross.conductor.sdk.workflow.task; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.PARAMETER) -public @interface InputParam { - String value(); - - boolean required() default false; -} diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java deleted file mode 100644 index 2df17faab1..0000000000 --- a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/OutputParam.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package org.conductoross.conductor.sdk.workflow.task; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.TYPE_USE) -public @interface OutputParam { - String value(); -} diff --git a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java b/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java deleted file mode 100644 index 4de239bcf2..0000000000 --- a/core/src/main/java/org/conductoross/conductor/sdk/workflow/task/WorkerTask.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2021 Conductor Authors. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package org.conductoross.conductor.sdk.workflow.task; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** Identifies a simple worker task. */ -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD}) -public @interface WorkerTask { - String value(); - - // No. of threads to use for executing the task - int threadCount() default 1; - - int pollingInterval() default 100; - - String domain() default ""; - - // In millis - int pollTimeout() default 100; - - // number of task pollers - // default is 1 which is good enough for most use cases - // a number higher than 1 will have concurrent pollers doing poll and execute - int pollerCount() default 1; -} diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java index 07cc306aec..1fed1e4d90 100644 --- a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodParameterMapper.java @@ -17,10 +17,10 @@ import java.util.List; import java.util.Map; -import org.conductoross.conductor.sdk.workflow.task.InputParam; import org.junit.Test; import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.sdk.workflow.task.InputParam; import static org.junit.Assert.*; diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java index c57f62c701..2a2d58ac6d 100644 --- a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedMethodResultMapper.java @@ -17,10 +17,10 @@ import java.util.List; import java.util.Map; -import org.conductoross.conductor.sdk.workflow.task.OutputParam; import org.junit.Test; import com.netflix.conductor.model.TaskModel; +import com.netflix.conductor.sdk.workflow.task.OutputParam; import static org.junit.Assert.*; diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java index eb3c1384b4..ff3f360fc8 100644 --- a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedSystemTaskIntegration.java @@ -15,8 +15,6 @@ import java.util.Map; import java.util.Set; -import org.conductoross.conductor.sdk.workflow.task.InputParam; -import org.conductoross.conductor.sdk.workflow.task.WorkerTask; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -30,6 +28,8 @@ import com.netflix.conductor.core.execution.tasks.SystemTaskRegistry; import com.netflix.conductor.core.execution.tasks.WorkflowSystemTask; import com.netflix.conductor.core.utils.ParametersUtils; +import com.netflix.conductor.sdk.workflow.task.InputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; import static org.junit.Assert.*; diff --git a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java index 5f2093c91f..c6a3602fbb 100644 --- a/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java +++ b/core/src/test/java/org/conductoross/conductor/core/execution/tasks/annotated/TestAnnotatedWorkflowSystemTask.java @@ -16,8 +16,6 @@ import java.util.HashMap; import java.util.Map; -import org.conductoross.conductor.sdk.workflow.task.InputParam; -import org.conductoross.conductor.sdk.workflow.task.WorkerTask; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -25,6 +23,8 @@ import com.netflix.conductor.core.execution.WorkflowExecutor; import com.netflix.conductor.model.TaskModel; import com.netflix.conductor.model.WorkflowModel; +import com.netflix.conductor.sdk.workflow.task.InputParam; +import com.netflix.conductor.sdk.workflow.task.WorkerTask; import static org.junit.Assert.*; import static org.mockito.Mockito.mock; From 68794a0566779c05db84de6be1405ecfea8029dc Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Fri, 30 Jan 2026 21:01:03 -0800 Subject: [PATCH 39/39] fixes and clean up --- .../core/execution/DeciderService.java | 35 ++++++++++++++----- .../AnnotatedMethodParameterMapper.java | 22 ------------ .../tasks/annotated/SampleWorkers.java | 3 +- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java index 598f6f4b1f..b54ce7cca6 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/DeciderService.java @@ -87,7 +87,17 @@ public DeciderService( this.systemTaskRegistry = systemTaskRegistry; LOGGER.info("taskMappers: {}", taskMappers.keySet()); LOGGER.info("annotatedTaskMappers: {}", annotatedTaskSystems.keySet()); - this.taskMappers.putAll(annotatedTaskSystems); + // Add annotated mappers only if no existing mapper is registered for that task + // type + // Existing mappers take precedence over annotated ones + annotatedTaskSystems.forEach( + (taskType, mapper) -> { + if (this.taskMappers.putIfAbsent(taskType, mapper) != null) { + LOGGER.info( + "Skipping annotated mapper for '{}' - existing mapper already registered", + taskType); + } + }); } public DeciderOutcome decide(WorkflowModel workflow) throws TerminateWorkflowException { @@ -139,8 +149,10 @@ private DeciderOutcome decide(final WorkflowModel workflow, List preS boolean hasSuccessfulTerminateTask = false; for (TaskModel task : workflow.getTasks()) { - // Filter the list of tasks and include only tasks that are not retried, not executed - // marked to be skipped and not part of System tasks that is DECISION, FORK, JOIN + // Filter the list of tasks and include only tasks that are not retried, not + // executed + // marked to be skipped and not part of System tasks that is DECISION, FORK, + // JOIN // This list will be empty for a new workflow being started if (!task.isRetried() && !task.getStatus().equals(SKIPPED) && !task.isExecuted()) { pendingTasks.add(task); @@ -190,7 +202,8 @@ private DeciderOutcome decide(final WorkflowModel workflow, List preS if (taskDefinition.isPresent()) { checkTaskTimeout(taskDefinition.get(), pendingTask); checkTaskPollTimeout(taskDefinition.get(), pendingTask); - // If the task has not been updated for "responseTimeoutSeconds" then mark task as + // If the task has not been updated for "responseTimeoutSeconds" then mark task + // as // TIMED_OUT if (isResponseTimedOut(taskDefinition.get(), pendingTask)) { timeoutTask(taskDefinition.get(), pendingTask); @@ -452,7 +465,8 @@ public boolean checkForWorkflowCompletion(final WorkflowModel workflow) return false; } - // If there is a TERMINATE task that has been executed successfuly then the workflow + // If there is a TERMINATE task that has been executed successfuly then the + // workflow // should be marked as completed. if (TERMINATE.name().equals(task.getTaskType()) && task.getStatus().isTerminal() @@ -873,7 +887,7 @@ public List getTasksToBeScheduled( String type = taskToSchedule.getType(); - // get tasks already scheduled (in progress/terminal) for this workflow instance + // get tasks already scheduled (in progress/terminal) for this workflow instance List tasksInWorkflow = workflow.getTasks().stream() .filter( @@ -896,10 +910,13 @@ public List getTasksToBeScheduled( .withDeciderService(this) .build(); - // For static forks, each branch of the fork creates a join task upon completion for - // dynamic forks, a join task is created with the fork and also with each branch of the + // For static forks, each branch of the fork creates a join task upon completion + // for + // dynamic forks, a join task is created with the fork and also with each branch + // of the // fork. - // A new task must only be scheduled if a task, with the same reference name is not already + // A new task must only be scheduled if a task, with the same reference name is + // not already // in this workflow instance return taskMappers .getOrDefault(type, taskMappers.get(USER_DEFINED.name())) diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java index 6a27ee5bfe..93d58d2a44 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/AnnotatedMethodParameterMapper.java @@ -103,28 +103,6 @@ private Object[] mapAnnotatedParameters( return values; } - // WorkflowInstanceIdInputParam not available in SDK v3.x - uncomment when - // upgrading - // private boolean containsWorkflowInstanceIdInputParamAnnotation(Annotation[] - // annotations) { - // return Arrays.stream(annotations) - // .map(Annotation::annotationType) - // .anyMatch(WorkflowInstanceIdInputParam.class::equals); - // } - // - // private void validateParameterForWorkflow InstanceId(Parameter parameter) { - // if (!parameter.getType().equals(String.class)) { - // throw new IllegalArgumentException( - // "Parameter " - // + parameter - // + " is annotated with " - // + WorkflowInstanceIdInputParam.class.getSimpleName() - // + " but is not of type " - // + String.class.getSimpleName() - // + "."); - // } - // } - private Object getInputValue( TaskModel task, Class parameterType, Type type, Annotation[] paramAnnotation) { InputParam ip = findInputParamAnnotation(paramAnnotation); diff --git a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java index a2446c7d3e..e3bc25ce62 100644 --- a/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java +++ b/core/src/main/java/org/conductoross/conductor/core/execution/tasks/annotated/SampleWorkers.java @@ -22,6 +22,7 @@ public class SampleWorkers { @WorkerTask("HELLO") public String hello(@InputParam("name") String name) { - return "Hello %s, from the sample worker".formatted(name); + return "Hello %s, from the sample worker, with id: %s" + .formatted(name, TaskContext.get().getTaskId()); } }