Merge branch 'master' into users/sanjuyadav/Node24

Author: sanjuyadav24

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -898,5 +898,20 @@ public class AgentKnobs
             "If true, use structured enhanced logging format with timestamps, components, and operations",
             new EnvironmentKnobSource("AZP_USE_ENHANCED_LOGGING"),
             new BuiltInDefaultKnobSource("false"));
+
+        public static readonly Knob EnableEnhancedContainerDiagnostics = new Knob(
+            nameof(EnableEnhancedContainerDiagnostics),
+            "When enabled, uses ContainerOperationProviderEnhanced with detailed tracing and duration logging for container operations",
+            new PipelineFeatureSource("EnableEnhancedContainerDiagnostics"),
+            new EnvironmentKnobSource("AGENT_ENABLE_ENHANCED_CONTAINER_LOGGING"),
+            new BuiltInDefaultKnobSource("false"));
+
+
+        public static readonly Knob EnableDockerExecDiagnostics = new Knob(
+            nameof(EnableDockerExecDiagnostics),
+            "If true, collect and report comprehensive diagnostics when docker exec commands fail, including container state, resource limits, logs, and platform-specific analysis.",
+            new PipelineFeatureSource("EnableDockerExecDiagnostics"),
+            new EnvironmentKnobSource("AGENT_ENABLE_DOCKER_EXEC_DIAGNOSTICS"),
+            new BuiltInDefaultKnobSource("false"));
     }
 }

src/Agent.Worker/Container/ContainerDiagnosticsManager.cs

@@ -82,7 +82,7 @@ public async Task<DockerVersion> DockerVersion(IExecutionContext context)
             }
 
             Version clientVersion = null;
-            var clientVersionMatchResult = verRegex.Match(serverVersionStr);
+            var clientVersionMatchResult = verRegex.Match(clientVersionStr);
             if (clientVersionMatchResult.Success && !string.IsNullOrEmpty(clientVersionMatchResult.Value))
             {
                 if (!Version.TryParse(clientVersionMatchResult.Value, out clientVersion))
@@ -498,6 +498,7 @@ private static async Task<int> ExecuteDockerCommandAsyncWithRetries(IExecutionCo
                 retryCount++;
             }
 
+            context.Output(StringUtil.Loc("DockerCommandFinalExitCode", command, exitCode));
             return exitCode;
         }
 

src/Agent.Worker/Container/DockerCommandManager.cs

@@ -285,6 +285,8 @@ protected void AddPrependPathToEnvironment()
                     prepend.Add(ExecutionContext.TranslatePathForStepTarget(path));
                 }
                 containerStepHost.PrependPath = string.Join(Path.PathSeparator.ToString(), prepend.Reverse<string>());
+                // Set docker exec diagnostics feature flag from ExecutionContext
+                containerStepHost.EnableDockerExecDiagnostics = AgentKnobs.EnableDockerExecDiagnostics.GetValue(ExecutionContext).AsBoolean();
             }
             else
             {

src/Agent.Worker/ContainerOperationProviderEnhanced.cs

@@ -43,6 +43,7 @@ public interface IContainerStepHost : IStepHost
     {
         ContainerInfo Container { get; set; }
         string PrependPath { get; set; }
+        bool EnableDockerExecDiagnostics { get; set; }
     }
 
     [ServiceLocator(Default = typeof(DefaultStepHost))]
@@ -101,6 +102,7 @@ public sealed class ContainerStepHost : AgentService, IContainerStepHost
     {
         public ContainerInfo Container { get; set; }
         public string PrependPath { get; set; }
+        public bool EnableDockerExecDiagnostics { get; set; }
         public event EventHandler<ProcessDataReceivedEventArgs> OutputDataReceived;
         public event EventHandler<ProcessDataReceivedEventArgs> ErrorDataReceived;
 
@@ -221,20 +223,47 @@ public async Task<int> ExecuteAsync(string workingDirectory,
                 var payloadJson = JsonUtility.ToString(payload);
                 redirectStandardIn.Enqueue(payloadJson);
                 HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Payload: {payloadJson}");
-                return await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work),
-                                                         fileName: containerEnginePath,
-                                                         arguments: containerExecutionArgs,
-                                                         environment: null,
-                                                         requireExitCodeZero: requireExitCodeZero,
-                                                         outputEncoding: outputEncoding,
-                                                         killProcessOnCancel: killProcessOnCancel,
-                                                         redirectStandardIn: redirectStandardIn,
-                                                         inheritConsoleHandler: inheritConsoleHandler,
-                                                         continueAfterCancelProcessTreeKillAttempt: continueAfterCancelProcessTreeKillAttempt,
-                                                         cancellationToken: cancellationToken);
+                
+                int exitCode = 0;
+                try
+                {
+                    exitCode = await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work),
+                                                             fileName: containerEnginePath,
+                                                             arguments: containerExecutionArgs,
+                                                             environment: null,
+                                                             requireExitCodeZero: requireExitCodeZero,
+                                                             outputEncoding: outputEncoding,
+                                                             killProcessOnCancel: killProcessOnCancel,
+                                                             redirectStandardIn: redirectStandardIn,
+                                                             inheritConsoleHandler: inheritConsoleHandler,
+                                                             continueAfterCancelProcessTreeKillAttempt: continueAfterCancelProcessTreeKillAttempt,
+                                                             cancellationToken: cancellationToken);
+                }
+                catch (Exception ex)
+                {
+                    // Log the exception
+                    var trace = HostContext.GetTrace(nameof(ContainerStepHost));
+                    trace.Error($"Docker exec failed: {ex.GetType().Name}: {ex.Message}");
+                    
+                    // Check if docker exec diagnostics feature is enabled
+                    // Value is set by the Handler from ExecutionContext
+                    if (EnableDockerExecDiagnostics)
+                    {
+                        // Collect comprehensive diagnostics when docker exec fails
+                        trace.Info("Docker exec diagnostics enabled, collecting diagnostics");
+                        var diagnosticsManager = HostContext.GetService<Container.IContainerDiagnosticsManager>();
+                        await diagnosticsManager.CollectDockerExecFailureDiagnosticsAsync(ex, containerEnginePath, containerExecutionArgs, Container.ContainerId);
+                    }
+                    else
+                    {
+                        trace.Info("Docker exec diagnostics disabled, skipping diagnostic collection");
+                    }
+                    throw; // Re-throw the original exception
+                }
+                return exitCode;
             }
         }
-
+        
         private class ContainerStandardInPayload
         {
             [JsonProperty("handler")]

src/Agent.Worker/Handlers/Handler.cs

@@ -58,6 +58,7 @@ public abstract class JobExtension : AgentService, IJobExtension
         // download all required tasks.
         // make sure all task's condition inputs are valid.
         // build up three list of steps for jobrunner. (pre-job, job, post-job)
+#pragma warning disable CA1505
         public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
         {
             Trace.Entering();
@@ -313,7 +314,20 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
 
                     if (context.Containers.Count > 0 || context.SidecarContainers.Count > 0)
                     {
-                        var containerProvider = HostContext.GetService<IContainerOperationProvider>();
+                        // Get container provider - uses enhanced version if feature flag is enabled
+                        IContainerOperationProvider containerProvider;
+                        if (AgentKnobs.EnableEnhancedContainerDiagnostics.GetValue(context).AsBoolean())
+                        {
+                            // Create and initialize enhanced provider manually since it's not registered with ServiceLocator
+                            var enhancedProvider = new ContainerOperationProviderEnhanced();
+                            enhancedProvider.Initialize(HostContext);
+                            containerProvider = enhancedProvider;
+                        }
+                        else
+                        {
+                            containerProvider = HostContext.GetService<IContainerOperationProvider>();
+                        }
+                        
                         var containers = new List<ContainerInfo>();
                         containers.AddRange(context.Containers);
                         containers.AddRange(context.SidecarContainers);

src/Agent.Worker/Handlers/StepHost.cs

@@ -277,6 +277,7 @@
   "DirExpireLimit": "Directory expiration limit: {0} days.",
   "DiscoverBuildDir": "Discover stale build directories that haven't been used for more than {0} days.",
   "DiscoverReleaseDir": "Discover stale release directories that haven't been used for more than {0} days.",
+  "DockerCommandFinalExitCode": "Final exit code for {0}: {1}",
   "DownloadAgent": "Downloading {0} agent",
   "DownloadArtifactFinished": "Downloading artifact finished.",
   "DownloadArtifacts": "Download Artifacts",