Skip to content

Commit d14c4fa

Browse files
AdamEssenmacherAdamEssenmacher
authored
[codex] Add Firebase E2E runtime drift all mode (#186)
* Add Firebase E2E runtime drift all mode * Scale runtime drift all timeout
1 parent 783d38b commit d14c4fa

5 files changed

Lines changed: 182 additions & 124 deletions

File tree

‎tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseFoundationE2E.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
<EnableNullabilityValidation>false</EnableNullabilityValidation>
1818
<RuntimeDriftCase></RuntimeDriftCase>
1919
<RuntimeDriftCaseMethod></RuntimeDriftCaseMethod>
20+
<RuntimeDriftCases></RuntimeDriftCases>
2021
<RuntimeDriftCasePropsPath></RuntimeDriftCasePropsPath>
2122
<BindingSurfaceCoverageTarget></BindingSurfaceCoverageTarget>
2223
<BindingSurfaceCoveragePropsPath></BindingSurfaceCoveragePropsPath>
@@ -65,6 +66,10 @@
6566
<_Parameter1>RuntimeDriftCaseMethod</_Parameter1>
6667
<_Parameter2>$(RuntimeDriftCaseMethod)</_Parameter2>
6768
</AssemblyAttribute>
69+
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute" Condition="'$(RuntimeDriftCases)' != ''">
70+
<_Parameter1>RuntimeDriftCases</_Parameter1>
71+
<_Parameter2>$(RuntimeDriftCases)</_Parameter2>
72+
</AssemblyAttribute>
6873
</ItemGroup>
6974

7075
<ItemGroup Condition="'$(BindingSurfaceCoverageTarget)' != ''">

‎tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseRuntimeDriftCases.cs‎

Lines changed: 55 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,119 +1,43 @@
11
using System.Reflection;
2-
3-
#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL
4-
using Firebase.Core;
52
using Foundation;
63
using ObjCRuntime;
7-
#endif
84

9-
#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION
10-
using Firebase.Analytics;
11-
using Foundation;
12-
using ObjCRuntime;
5+
#if ENABLE_RUNTIME_DRIFT_CASE_CORE_CONFIGURATION_LOGGERLEVEL
6+
using Firebase.Core;
137
#endif
148

15-
#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION
9+
#if ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_SESSIONIDWITHCOMPLETION || ENABLE_RUNTIME_DRIFT_CASE_ANALYTICS_ONDEVICECONVERSION
1610
using Firebase.Analytics;
17-
using Foundation;
18-
using ObjCRuntime;
1911
#endif
2012

2113
#if ENABLE_RUNTIME_DRIFT_CASE_APPCHECK_LIMITED_USE_TOKENS
2214
using Firebase.AppCheck;
2315
using FirebaseCoreApp = Firebase.Core.App;
24-
using Foundation;
25-
using ObjCRuntime;
2616
#endif
2717

2818
#if ENABLE_RUNTIME_DRIFT_CASE_REMOTECONFIG_REALTIME_CUSTOMSIGNALS
2919
using Firebase.RemoteConfig;
30-
using Foundation;
31-
using ObjCRuntime;
3220
#endif
3321

3422
#if ENABLE_RUNTIME_DRIFT_CASE_DATABASE_SERVERVALUE_INCREMENT || ENABLE_RUNTIME_DRIFT_CASE_DATABASE_QUERY_GETDATA
3523
using Firebase.Database;
3624
using FirebaseCoreOptions = Firebase.Core.Options;
37-
using Foundation;
38-
using ObjCRuntime;
39-
#endif
40-
41-
#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS
42-
using Firebase.ABTesting;
43-
using Foundation;
44-
using ObjCRuntime;
45-
#endif
46-
47-
#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT
48-
using Firebase.ABTesting;
49-
using Foundation;
50-
using ObjCRuntime;
5125
#endif
5226

53-
#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS
27+
#if ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_UPDATEEXPERIMENTS || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_ACTIVATEEXPERIMENT || ENABLE_RUNTIME_DRIFT_CASE_ABTESTING_VALIDATERUNNINGEXPERIMENTS
5428
using Firebase.ABTesting;
55-
using Foundation;
56-
using ObjCRuntime;
57-
#endif
58-
59-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED
60-
using Firebase.CloudFirestore;
61-
using Foundation;
62-
using ObjCRuntime;
63-
#endif
64-
65-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY
66-
using Firebase.CloudFirestore;
67-
using Foundation;
68-
using ObjCRuntime;
69-
#endif
70-
71-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY
72-
using Firebase.CloudFirestore;
73-
using Foundation;
74-
using ObjCRuntime;
75-
#endif
76-
77-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS
78-
using Firebase.CloudFirestore;
79-
using Foundation;
80-
using ObjCRuntime;
81-
#endif
82-
83-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS
84-
using Firebase.CloudFirestore;
85-
using Foundation;
86-
using ObjCRuntime;
87-
#endif
88-
89-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE
90-
using Firebase.CloudFirestore;
91-
using Foundation;
92-
using ObjCRuntime;
9329
#endif
9430

95-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS
96-
using Firebase.CloudFirestore;
97-
using Foundation;
98-
using ObjCRuntime;
99-
#endif
100-
101-
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION
31+
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_GETQUERYNAMED || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_FIELDVALUE_VECTORWITHARRAY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_AGGREGATE_QUERY || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_QUERY_FILTERS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_SNAPSHOT_LISTEN_OPTIONS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_NAMED_DATABASE || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_CACHE_SETTINGS || ENABLE_RUNTIME_DRIFT_CASE_CLOUDFIRESTORE_INDEX_CONFIGURATION
10232
using Firebase.CloudFirestore;
103-
using Foundation;
104-
using ObjCRuntime;
10533
#endif
10634

10735
#if ENABLE_RUNTIME_DRIFT_CASE_CLOUDFUNCTIONS_USEFUNCTIONSEMULATORORIGIN
10836
using Firebase.CloudFunctions;
109-
using Foundation;
110-
using ObjCRuntime;
11137
#endif
11238

11339
#if ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_STACKFRAMEWITHADDRESS || ENABLE_RUNTIME_DRIFT_CASE_CRASHLYTICS_RECORD_ERROR_USER_INFO
11440
using Firebase.Crashlytics;
115-
using Foundation;
116-
using ObjCRuntime;
11741
#endif
11842

11943
namespace FirebaseFoundationE2E;
@@ -122,17 +46,28 @@ static partial class FirebaseRuntimeDriftCases
12246
{
12347
static readonly TimeSpan AsyncTimeout = TimeSpan.FromSeconds(5);
12448

49+
public sealed record RuntimeDriftCase(string Id, string MethodName);
50+
12551
public static string? GetConfiguredCaseId()
12652
{
12753
return GetAssemblyMetadataValue("RuntimeDriftCase");
12854
}
12955

130-
public static async Task<string> ExecuteConfiguredCaseAsync()
56+
public static IReadOnlyList<RuntimeDriftCase> GetConfiguredCases()
13157
{
58+
var configuredCases = GetAssemblyMetadataValue("RuntimeDriftCases");
59+
if (!string.IsNullOrWhiteSpace(configuredCases))
60+
{
61+
return configuredCases
62+
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
63+
.Select(ParseConfiguredCase)
64+
.ToArray();
65+
}
66+
13267
var caseId = GetConfiguredCaseId();
13368
if (string.IsNullOrWhiteSpace(caseId))
13469
{
135-
throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value.");
70+
return Array.Empty<RuntimeDriftCase>();
13671
}
13772

13873
var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod");
@@ -141,20 +76,54 @@ public static async Task<string> ExecuteConfiguredCaseAsync()
14176
throw new InvalidOperationException($"Runtime drift case '{caseId}' is missing RuntimeDriftCaseMethod metadata.");
14277
}
14378

144-
var method = typeof(FirebaseRuntimeDriftCases).GetMethod(methodName, BindingFlags.Static | BindingFlags.NonPublic);
79+
return new[] { new RuntimeDriftCase(caseId, methodName) };
80+
}
81+
82+
public static async Task<string> ExecuteConfiguredCaseAsync()
83+
{
84+
var caseId = GetConfiguredCaseId();
85+
if (string.IsNullOrWhiteSpace(caseId))
86+
{
87+
throw new InvalidOperationException("Runtime drift mode was requested without a RuntimeDriftCase value.");
88+
}
89+
90+
var methodName = GetAssemblyMetadataValue("RuntimeDriftCaseMethod");
91+
var configuredCase = new RuntimeDriftCase(caseId, methodName ?? string.Empty);
92+
return await ExecuteConfiguredCaseAsync(configuredCase);
93+
}
94+
95+
public static async Task<string> ExecuteConfiguredCaseAsync(RuntimeDriftCase configuredCase)
96+
{
97+
if (string.IsNullOrWhiteSpace(configuredCase.MethodName))
98+
{
99+
throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' is missing RuntimeDriftCaseMethod metadata.");
100+
}
101+
102+
var method = typeof(FirebaseRuntimeDriftCases).GetMethod(configuredCase.MethodName, BindingFlags.Static | BindingFlags.NonPublic);
145103
if (method is null)
146104
{
147-
throw new InvalidOperationException($"Runtime drift case '{caseId}' points at missing method '{methodName}'.");
105+
throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' points at missing method '{configuredCase.MethodName}'.");
148106
}
149107

150108
if (method.Invoke(null, null) is not Task<string> task)
151109
{
152-
throw new InvalidOperationException($"Runtime drift case '{caseId}' method '{methodName}' did not return Task<string>.");
110+
throw new InvalidOperationException($"Runtime drift case '{configuredCase.Id}' method '{configuredCase.MethodName}' did not return Task<string>.");
153111
}
154112

155113
return await task;
156114
}
157115

116+
static RuntimeDriftCase ParseConfiguredCase(string value)
117+
{
118+
var separatorIndex = value.IndexOf('=');
119+
if (separatorIndex <= 0 || separatorIndex == value.Length - 1)
120+
{
121+
throw new InvalidOperationException($"Runtime drift case metadata entry '{value}' must use '<id>=<method>' format.");
122+
}
123+
124+
return new RuntimeDriftCase(value[..separatorIndex], value[(separatorIndex + 1)..]);
125+
}
126+
158127
static string? GetAssemblyMetadataValue(string key)
159128
{
160129
return typeof(FirebaseRuntimeDriftCases)

‎tests/E2E/Firebase.Foundation/FirebaseFoundationE2E/FirebaseSelfTestRunner.cs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,14 @@ public static async Task RunAsync(StatusViewController statusViewController)
2929
#if ENABLE_BINDING_SURFACE_COVERAGE
3030
await statusViewController.AppendLineAsync("Firebase binding surface coverage mode enabled.");
3131
#endif
32-
var runtimeDriftCase = FirebaseRuntimeDriftCases.GetConfiguredCaseId();
33-
if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
32+
var runtimeDriftCases = FirebaseRuntimeDriftCases.GetConfiguredCases();
33+
if (runtimeDriftCases.Count > 0)
3434
{
35-
await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {runtimeDriftCase}");
35+
var configuredCaseId = FirebaseRuntimeDriftCases.GetConfiguredCaseId();
36+
var caseDisplay = string.Equals(configuredCaseId, "all", StringComparison.OrdinalIgnoreCase)
37+
? $"all ({runtimeDriftCases.Count} cases)"
38+
: runtimeDriftCases[0].Id;
39+
await statusViewController.AppendLineAsync($"Runtime drift case mode enabled: {caseDisplay}");
3640
}
3741

3842
try
@@ -52,10 +56,13 @@ await ExecuteCaseAsync(result, statusViewController, "ConfigureApp", async () =>
5256
return $"Configured app '{defaultApp.Name}'.";
5357
});
5458

55-
if (!string.IsNullOrWhiteSpace(runtimeDriftCase))
59+
if (runtimeDriftCases.Count > 0)
5660
{
57-
await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase}", () =>
58-
FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync());
61+
foreach (var runtimeDriftCase in runtimeDriftCases)
62+
{
63+
await ExecuteCaseAsync(result, statusViewController, $"RuntimeDrift:{runtimeDriftCase.Id}", () =>
64+
FirebaseRuntimeDriftCases.ExecuteConfiguredCaseAsync(runtimeDriftCase));
65+
}
5966
}
6067
#if ENABLE_BINDING_SURFACE_COVERAGE
6168
else if (!string.IsNullOrWhiteSpace(FirebaseBindingSurfaceCoverage.GetConfiguredTarget()))

‎tests/E2E/Firebase.Foundation/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,16 +81,17 @@ dotnet tool run dotnet-cake -- --target=nuget --names="Firebase.Analytics,Fireba
8181
tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --enable-nullability-validation
8282
```
8383

84-
## Targeted runtime drift mode
84+
## Runtime drift mode
8585

86-
The harness also supports a targeted runtime-drift lane for one binding drift at a time. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff.
86+
The harness also supports a runtime-drift lane for binding-layer checks that need simulator execution. This mode runs only `ConfigureApp` plus the selected drift case, so each remediation PR can prove the unfixed runtime failure locally, then keep only the success-oriented regression test in the final diff. Use `all` to build once and run every checked-in drift case in the same app launch.
8787

8888
The checked-in case manifest lives at [`runtime-drift-cases.json`](./runtime-drift-cases.json), and the backlog/queue is tracked in [`docs/firebase-runtime-failure-backlog.md`](../../docs/firebase-runtime-failure-backlog.md).
8989

9090
Run a specific drift case with:
9191

9292
```sh
9393
tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case cloudfirestore-getquerynamed
94+
tools/e2e/run-firebase-foundation.sh --package-dir output --configuration Debug --runtime-drift-case all
9495
```
9596

9697
## Binding surface coverage mode

0 commit comments

Comments
 (0)