test: Introduce .unordered in node-integration-tests (#21697)

closes #21626

Inspired by the Cloudflare and Bun integration tests runner:
`.unordered()`

Author: JPeer264

dev-packages/node-integration-tests/suites/tracing/apollo-graphql/test.ts

@@ -37,6 +37,7 @@ describe('GraphQL/Apollo Tests', () => {
           await createTestRunner()
             .expect({ transaction: EXPECTED_START_SERVER_TRANSACTION })
             .expect({ transaction: EXPECTED_TRANSACTION })
+            .unordered()
             .start()
             .completed();
         });
@@ -72,6 +73,7 @@ describe('GraphQL/Apollo Tests', () => {
           await createTestRunner()
             .expect({ transaction: EXPECTED_START_SERVER_TRANSACTION })
             .expect({ transaction: EXPECTED_TRANSACTION })
+            .unordered()
             .start()
             .completed();
         });
@@ -107,6 +109,7 @@ describe('GraphQL/Apollo Tests', () => {
           await createTestRunner()
             .expect({ transaction: EXPECTED_START_SERVER_TRANSACTION })
             .expect({ transaction: EXPECTED_TRANSACTION })
+            .unordered()
             .start()
             .completed();
         });
@@ -142,6 +145,7 @@ describe('GraphQL/Apollo Tests', () => {
           await createTestRunner()
             .expect({ transaction: EXPECTED_START_SERVER_TRANSACTION })
             .expect({ transaction: EXPECTED_TRANSACTION })
+            .unordered()
             .start()
             .completed();
         });

dev-packages/node-integration-tests/utils/runner/createRunner.ts

@@ -131,6 +131,7 @@ export function createRunner(...paths: string[]) {
   const flags: string[] = [];
   // By default, we ignore session & sessions
   const ignored: Set<EnvelopeItemType> = new Set(['session', 'sessions', 'client_report']);
+  let unordered = false;
   let withEnv: Record<string, string> = {};
   let withSentryServer = false;
   let dockerOptions: DockerOptions | undefined;
@@ -211,6 +212,10 @@ export function createRunner(...paths: string[]) {
       }
       return this;
     },
+    unordered: function () {
+      unordered = true;
+      return this;
+    },
     withDockerCompose: function (options: DockerOptions) {
       dockerOptions = options;
       return this;
@@ -284,58 +289,50 @@ export function createRunner(...paths: string[]) {
             return;
           }
 
-          const expected = expectedEnvelopes.shift();
+          if (unordered) {
+            const matchIndex = expectedEnvelopes.findIndex(candidate => {
+              const candidateType = Object.keys(candidate)[0];
+              if (candidateType !== envelopeItemType) {
+                return false;
+              }
+              try {
+                assertExpectedEnvelope(candidate, item);
+                return true;
+              } catch {
+                return false;
+              }
+            });
 
-          // Catch any error or failed assertions and pass them to done to end the test quickly
-          try {
-            if (!expected) {
+            if (matchIndex < 0) {
               return;
             }
 
-            const expectedType = Object.keys(expected)[0];
+            expectedEnvelopes.splice(matchIndex, 1);
+            expectCallbackCalled();
+          } else {
+            const expected = expectedEnvelopes.shift();
 
-            if (expectedType !== envelopeItemType) {
-              throw new Error(
-                `Expected envelope item type '${expectedType}' but got '${envelopeItemType}'. \nItem: ${JSON.stringify(
-                  item,
-                )}`,
-              );
-            }
+            // Catch any error or failed assertions and pass them to done to end the test quickly
+            try {
+              if (!expected) {
+                return;
+              }
 
-            if ('event' in expected) {
-              expectErrorEvent(item[1] as Event, expected.event);
-              expectCallbackCalled();
-            } else if ('transaction' in expected) {
-              expectTransactionEvent(item[1] as TransactionEvent, expected.transaction);
-              expectCallbackCalled();
-            } else if ('session' in expected) {
-              expectSessionEvent(item[1] as SerializedSession, expected.session);
-              expectCallbackCalled();
-            } else if ('sessions' in expected) {
-              expectSessionsEvent(item[1] as SessionAggregates, expected.sessions);
-              expectCallbackCalled();
-            } else if ('check_in' in expected) {
-              expectCheckInEvent(item[1] as SerializedCheckIn, expected.check_in);
-              expectCallbackCalled();
-            } else if ('client_report' in expected) {
-              expectClientReport(item[1] as ClientReport, expected.client_report);
-              expectCallbackCalled();
-            } else if ('log' in expected) {
-              expectLog(item[1] as SerializedLogContainer, expected.log);
-              expectCallbackCalled();
-            } else if ('trace_metric' in expected) {
-              expectMetric(item[1] as SerializedMetricContainer, expected.trace_metric);
-              expectCallbackCalled();
-            } else if ('span' in expected) {
-              expectSpanContainer(item[1] as SerializedStreamedSpanContainer, expected.span);
+              const expectedType = Object.keys(expected)[0];
+
+              if (expectedType !== envelopeItemType) {
+                throw new Error(
+                  `Expected envelope item type '${expectedType}' but got '${envelopeItemType}'. \nItem: ${JSON.stringify(
+                    item,
+                  )}`,
+                );
+              }
+
+              assertExpectedEnvelope(expected, item);
               expectCallbackCalled();
-            } else {
-              throw new Error(
-                `Unhandled expected envelope item type: ${JSON.stringify(expected)}\nItem: ${JSON.stringify(item)}`,
-              );
+            } catch (e) {
+              complete(e as Error);
             }
-          } catch (e) {
-            complete(e as Error);
           }
         }
       }
@@ -641,6 +638,32 @@ function expectErrorEvent(item: Event, expected: ExpectedEvent): void {
   }
 }
 
+function assertExpectedEnvelope(expected: Expected, item: Envelope[1][number]): void {
+  if ('event' in expected) {
+    expectErrorEvent(item[1] as Event, expected.event);
+  } else if ('transaction' in expected) {
+    expectTransactionEvent(item[1] as TransactionEvent, expected.transaction);
+  } else if ('session' in expected) {
+    expectSessionEvent(item[1] as SerializedSession, expected.session);
+  } else if ('sessions' in expected) {
+    expectSessionsEvent(item[1] as SessionAggregates, expected.sessions);
+  } else if ('check_in' in expected) {
+    expectCheckInEvent(item[1] as SerializedCheckIn, expected.check_in);
+  } else if ('client_report' in expected) {
+    expectClientReport(item[1] as ClientReport, expected.client_report);
+  } else if ('log' in expected) {
+    expectLog(item[1] as SerializedLogContainer, expected.log);
+  } else if ('trace_metric' in expected) {
+    expectMetric(item[1] as SerializedMetricContainer, expected.trace_metric);
+  } else if ('span' in expected) {
+    expectSpanContainer(item[1] as SerializedStreamedSpanContainer, expected.span);
+  } else {
+    throw new Error(
+      `Unhandled expected envelope item type: ${JSON.stringify(expected)}\nItem: ${JSON.stringify(item)}`,
+    );
+  }
+}
+
 function expectTransactionEvent(item: TransactionEvent, expected: ExpectedTransaction): void {
   if (typeof expected === 'function') {
     expected(item);