PRO-15775 fix: show failed screenshot tests in summary pack results

Author: uid11

src/actions/getBrowserConsoleMessages.ts

@@ -4,17 +4,21 @@ import {log} from '../utils/log';
 
 import type {ConsoleMessage} from '../types/internal';
 
-type Options = Readonly<{showMessagesInLog?: boolean}>;
+type Options = Readonly<{showMessagesInLog?: boolean; skipLogs?: boolean}>;
 
 const logMessage = 'Get browser console messages';
 
 /**
  * Returns an object that contains messages output to the browser console.
  */
 export const getBrowserConsoleMessages = (options: Options = {}): readonly ConsoleMessage[] => {
-  const {showMessagesInLog = false} = options;
+  const {skipLogs = false, showMessagesInLog = false} = options;
   const consoleMessages = getConsoleMessagesFromContext();
 
+  if (skipLogs) {
+    return consoleMessages;
+  }
+
   if (showMessagesInLog === false) {
     log(logMessage, LogEventType.InternalAction);
   } else {

src/constants/internal.ts

@@ -40,6 +40,8 @@ export {
 export {LogEventStatus, LogEventType} from './log';
 /** @internal */
 export {MESSAGE_BACKGROUND_COLOR_BY_STATUS} from './log';
+/** @internal */
+export {SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE} from './matchScreenshot';
 export {CREATE_PAGE_TOKEN} from './pages';
 /** @internal */
 export {

src/constants/matchScreenshot.ts

@@ -0,0 +1,6 @@
+/**
+ * Error message for "Expected screenshot not specified" error from the `toMatchScreenshot` assert (in `expect`).
+ * @internal
+ */
+export const SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE =
+  'Expected screenshot not specified. If the actual screenshot is correct, specify screenshot from actualScreenshotId field';

src/utils/expect/toMatchScreenshot.ts

@@ -6,6 +6,7 @@ import {isLocalRun} from '../../configurator';
 import {
   EXPECTED_SCREENSHOTS_DIRECTORY_PATH,
   INTERNAL_REPORTS_DIRECTORY_PATH,
+  SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE,
 } from '../../constants/internal';
 import {getOutputDirectoryName} from '../../context/outputDirectoryName';
 
@@ -84,7 +85,7 @@ export const toMatchScreenshot = async (
 
   const message = expectedScreenshotId
     ? `Cannot read expected screenshot ${expectedScreenshotId}`
-    : 'Expected screenshot not specified';
+    : SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE;
 
   if (isLocalRun) {
     if (expectedScreenshotFound) {

src/utils/report/collectReportData.ts

@@ -38,8 +38,8 @@ export const collectReportData = async ({
   const retries = getRetries(fullTestRuns);
   const exitCode = getExitCode(errors.length > 0, retries);
 
-  const failedTestsMainParams = getFailedTestsMainParams(retries.at(-1));
-  const summaryPackResults = getSummaryPackResults(fullTestRuns, retries.at(-1));
+  const failedTestsMainParams = getFailedTestsMainParams(retries);
+  const summaryPackResults = getSummaryPackResults(fullTestRuns, retries);
 
   return {
     apiStatistics,

src/utils/report/getFailedTestsMainParams.ts

@@ -1,16 +1,42 @@
-import {TestRunStatus} from '../../constants/internal';
+import {SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE, TestRunStatus} from '../../constants/internal';
+
+import {assertValueIsFalse} from '../asserts';
 
 import type {Retry} from '../../types/internal';
 
 /**
  * Get array of main parameters of pack's failed tests.
  * @internal
  */
-export const getFailedTestsMainParams = (lastRetry: Retry | undefined): readonly string[] => {
+export const getFailedTestsMainParams = (retries: readonly Retry[]): readonly string[] => {
+  const firstRetry = retries[0];
+  const lastRetry = retries.at(-1);
+
   const failedTests =
     lastRetry?.fullTestRuns.filter((fullTestRun) => fullTestRun.status === TestRunStatus.Failed) ??
     [];
   const failedTestsMainParams = failedTests.map(({mainParams}) => mainParams);
 
+  if (retries.length <= 1) {
+    return failedTestsMainParams;
+  }
+
+  const failedScreenshotTests =
+    firstRetry?.fullTestRuns.filter(
+      (fullTestRun) =>
+        fullTestRun.status === TestRunStatus.Failed &&
+        String(fullTestRun.runError).includes(SCREENSHOT_NOT_SPECIFIED_ERROR_MESSAGE),
+    ) ?? [];
+
+  for (const failedScreenshotTest of failedScreenshotTests) {
+    assertValueIsFalse(
+      failedTestsMainParams.includes(failedScreenshotTest.mainParams),
+      'mainParams of failed screenshot test is unique',
+      {duplicatedTest: failedScreenshotTest, failedScreenshotTests},
+    );
+
+    failedTestsMainParams.push(failedScreenshotTest.mainParams);
+  }
+
   return failedTestsMainParams;
 };

src/utils/report/getSummaryPackResults.ts

@@ -16,9 +16,11 @@ const MAX_FAILED_TESTS_COUNT = 8;
  */
 export const getSummaryPackResults = (
   fullTestRuns: readonly FullTestRun[],
-  lastRetry: Retry | undefined,
+  retries: readonly Retry[],
 ): string => {
-  const allFailedTestsMainParams = getFailedTestsMainParams(lastRetry);
+  const lastRetry = retries.at(-1);
+
+  const allFailedTestsMainParams = getFailedTestsMainParams(retries);
   const failedTestsMainParams = allFailedTestsMainParams.slice(0, MAX_FAILED_TESTS_COUNT);
 
   if (allFailedTestsMainParams.length > MAX_FAILED_TESTS_COUNT) {
@@ -34,7 +36,10 @@ export const getSummaryPackResults = (
       ? fullTestRuns
       : lastRetry?.fullTestRuns;
 
-    const count = source?.filter((fullTestRun) => fullTestRun.status === status)?.length ?? 0;
+    const count =
+      status === TestRunStatus.Failed
+        ? allFailedTestsMainParams.length
+        : (source?.filter((fullTestRun) => fullTestRun.status === status)?.length ?? 0);
 
     if (count > 0) {
       countsOfStatuses.push(`${count} ${status}`);

src/utils/selectors/Selector.ts

@@ -90,10 +90,10 @@ class Selector {
     return result;
   }
 
-  get checked(): Promise<boolean | undefined> {
+  get checked(): Promise<boolean | null> {
     const result = this.getPlaywrightLocator()
       .isChecked()
-      .catch(() => undefined);
+      .catch(() => null);
 
     setRetryData(result, {property: 'checked', selector: this});
 
@@ -160,10 +160,10 @@ class Selector {
     return result;
   }
 
-  get value(): Promise<string | undefined> {
+  get value(): Promise<string | null> {
     const result = this.getPlaywrightLocator()
       .inputValue()
-      .catch(() => undefined);
+      .catch(() => null);
 
     setRetryData(result, {property: 'value', selector: this});