Skip to content

Commit f440a20

Browse files
nicu-danicu-da
authored
recovery from key - store the parties being migrated in the local db for recovery in case of failure (#3201)
Signed-off-by: Nicu Reut <[email protected]>
1 parent 0865984 commit f440a20

File tree

13 files changed

+511
-348
lines changed

13 files changed

+511
-348
lines changed

‎apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ValidatorReonboardingIntegrationTest.scala‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
package org.lfdecentralizedtrust.splice.integration.tests
22

3-
import org.lfdecentralizedtrust.splice.config.ConfigTransforms.bumpUrl
4-
import org.lfdecentralizedtrust.splice.config.{
5-
AuthTokenSourceConfig,
6-
NetworkAppClientConfig,
7-
ParticipantBootstrapDumpConfig,
8-
ParticipantClientConfig,
9-
SpliceConfig,
10-
}
3+
import com.digitalasset.canton.config.CantonRequireTypes.InstanceName
4+
import com.digitalasset.canton.config.RequireTypes.Port
5+
import com.digitalasset.canton.config.{DbConfig, FullClientConfig}
6+
import com.digitalasset.canton.data.CantonTimestamp
117
import com.digitalasset.canton.logging.SuppressionRule
8+
import com.digitalasset.canton.topology.{ForceFlag, ParticipantId, PartyId}
9+
import com.typesafe.config.ConfigValueFactory
10+
import org.apache.pekko.http.scaladsl.model.Uri
11+
import org.lfdecentralizedtrust.splice.config.*
12+
import org.lfdecentralizedtrust.splice.config.ConfigTransforms.bumpUrl
1213
import org.lfdecentralizedtrust.splice.environment.RetryFor
1314
import org.lfdecentralizedtrust.splice.identities.NodeIdentitiesDump
1415
import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
@@ -22,13 +23,8 @@ import org.lfdecentralizedtrust.splice.validator.config.{
2223
MigrateValidatorPartyConfig,
2324
ValidatorCantonIdentifierConfig,
2425
}
25-
import com.digitalasset.canton.config.CantonRequireTypes.InstanceName
26-
import com.digitalasset.canton.config.{DbConfig, FullClientConfig}
27-
import com.digitalasset.canton.config.RequireTypes.Port
28-
import com.digitalasset.canton.data.CantonTimestamp
29-
import com.digitalasset.canton.topology.{ForceFlag, ParticipantId, PartyId}
30-
import com.typesafe.config.ConfigValueFactory
31-
import org.apache.pekko.http.scaladsl.model.Uri
26+
import org.lfdecentralizedtrust.splice.validator.migration.ParticipantPartyMigrator
27+
import org.lfdecentralizedtrust.splice.validator.store.ValidatorConfigProvider
3228
import org.scalatest.time.{Minute, Span}
3329
import org.slf4j.event.Level
3430

@@ -316,7 +312,11 @@ class ValidatorReonboardingIntegrationTest extends ValidatorReonboardingIntegrat
316312
"EXTRA_PARTICIPANT_ADMIN_USER" -> aliceValidatorLocalBackend.config.ledgerApiUser,
317313
"EXTRA_PARTICIPANT_DB" -> newParticipantDb,
318314
) {
319-
loggerFactory.assertLogsSeq(SuppressionRule.LevelAndAbove(Level.WARN))(
315+
loggerFactory.assertLogsSeq(
316+
SuppressionRule.forLogger[ParticipantPartyMigrator] || SuppressionRule
317+
.forLogger[ValidatorConfigProvider] || SuppressionRule
318+
.LevelAndAbove(Level.WARN)
319+
)(
320320
{
321321
aliceValidatorLocalBackend.startSync()
322322
},
@@ -327,6 +327,16 @@ class ValidatorReonboardingIntegrationTest extends ValidatorReonboardingIntegrat
327327
"not be able to migrate due to an unsupported namespace"
328328
)
329329
}
330+
forExactly(1, entries) {
331+
_.message should include(
332+
"Storing all the hosted parties (4) in the database to recover in case of failures"
333+
)
334+
}
335+
forExactly(1, entries) {
336+
_.message should include(
337+
"Clearing parties that were migrated"
338+
)
339+
}
330340
},
331341
)
332342

@@ -418,7 +428,7 @@ class ValidatorReonboardingWithPartiesToMigrateIntegrationTest
418428
// (which will fail unless we migrate all parties) despite migrating the operator party.
419429
// This can only work if there is no collision between the participant admin party and
420430
// the validator operator party.
421-
override val aliceValidatorParticipantNameHint = s"${aliceValidatorPartyHint}-participant";
431+
override val aliceValidatorParticipantNameHint = s"$aliceValidatorPartyHint-participant";
422432

423433
// we bootstrap from dump so we can predict the party IDs
424434
val testDumpDir: Path = Paths.get("apps/app/src/test/resources/dumps")
@@ -459,7 +469,7 @@ class ValidatorReonboardingWithPartiesToMigrateIntegrationTest
459469
testResourcesPath / "standalone-participant-extra-no-auth.conf",
460470
),
461471
Seq.empty,
462-
"alice-participant",
472+
"alice-reonboard-from-key-database",
463473
"EXTRA_PARTICIPANT_ADMIN_USER" -> aliceValidatorLocalBackend.config.ledgerApiUser,
464474
"EXTRA_PARTICIPANT_DB" -> oldParticipantDb,
465475
) {

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/TopologyAdminConnection.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,8 +1384,8 @@ abstract class TopologyAdminConnection(
13841384
runCmd(VaultAdminCommands.ImportKeyPair(ByteString.copyFrom(keyPair), name, password = None))
13851385
}
13861386

1387-
private def listSynchronizerTrustCertificate(synchronizerId: SynchronizerId, member: Member)(
1388-
implicit tc: TraceContext
1387+
def listSynchronizerTrustCertificate(synchronizerId: SynchronizerId, member: Member)(implicit
1388+
tc: TraceContext
13891389
): Future[Seq[TopologyResult[SynchronizerTrustCertificate]]] =
13901390
runCmd(
13911391
TopologyAdminCommands.Read.ListSynchronizerTrustCertificate(

‎apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/FutureUnlessShutdownUtil.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ object FutureUnlessShutdownUtil {
2222
)(implicit ec: ExecutionContext): OptionT[Future, A] = {
2323
OptionT(futureUnlessShutdownToFuture(f.value))
2424
}
25+
2526
implicit class FutureUnlessShutdownOps[A](val f: FutureUnlessShutdown[A]) extends AnyVal {
2627
def toFuture(implicit ec: ExecutionContext): Future[A] =
2728
f.failOnShutdownToAbortException("Splice unsafe shutdown future")

‎apps/common/src/test/scala/org/lfdecentralizedtrust/splice/util/SpliceRateLimiterTest.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import org.apache.pekko.stream.Materializer
1010
import org.apache.pekko.stream.scaladsl.{Sink, Source}
1111
import org.apache.pekko.stream.testkit.StreamSpec
1212
import org.lfdecentralizedtrust.splice.admin.api.client.commands.HttpCommandException
13-
import org.lfdecentralizedtrust.splice.util.{SpliceRateLimitMetrics, SpliceRateLimiter}
1413
import org.lfdecentralizedtrust.splice.util.SpliceRateLimiterTest.runRateLimited
1514

1615
import scala.concurrent.Future

‎apps/validator/src/main/scala/org/lfdecentralizedtrust/splice/validator/ValidatorApp.scala‎

Lines changed: 54 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,36 @@ package org.lfdecentralizedtrust.splice.validator
66
import cats.implicits.{catsSyntaxApplicativeByValue as _, *}
77
import com.daml.grpc.adapter.ExecutionSequencerFactory
88
import com.daml.ledger.javaapi.data.User
9+
import com.digitalasset.canton.SynchronizerAlias
10+
import com.digitalasset.canton.concurrent.FutureSupervisor
11+
import com.digitalasset.canton.config.CantonRequireTypes.InstanceName
12+
import com.digitalasset.canton.config.ProcessingTimeout
13+
import com.digitalasset.canton.config.RequireTypes.NonNegativeLong
14+
import com.digitalasset.canton.data.CantonTimestamp
15+
import com.digitalasset.canton.ledger.api.util.DurationConversion
16+
import com.digitalasset.canton.lifecycle.LifeCycle
17+
import com.digitalasset.canton.logging.{NamedLoggerFactory, TracedLogger}
18+
import com.digitalasset.canton.resource.{DbStorage, Storage}
19+
import com.digitalasset.canton.time.Clock
20+
import com.digitalasset.canton.topology.{PartyId, SynchronizerId}
21+
import com.digitalasset.canton.tracing.{TraceContext, TracerProvider}
22+
import com.digitalasset.canton.util.MonadUtil
23+
import com.google.protobuf.ByteString
24+
import io.grpc.Status
25+
import io.opentelemetry.api.trace.Tracer
26+
import org.apache.pekko.actor.ActorSystem
27+
import org.apache.pekko.http.cors.scaladsl.CorsDirectives.*
28+
import org.apache.pekko.http.cors.scaladsl.settings.CorsSettings
29+
import org.apache.pekko.http.scaladsl.model.HttpMethods
30+
import org.apache.pekko.http.scaladsl.server.Directives.*
31+
import org.apache.pekko.http.scaladsl.server.directives.BasicDirectives
932
import org.lfdecentralizedtrust.splice.admin.api.TraceContextDirectives.withTraceContext
1033
import org.lfdecentralizedtrust.splice.admin.http.{AdminRoutes, HttpErrorHandler}
34+
import org.lfdecentralizedtrust.splice.auth.*
1135
import org.lfdecentralizedtrust.splice.automation.{
1236
DomainParamsAutomationService,
1337
DomainTimeAutomationService,
1438
}
15-
import org.lfdecentralizedtrust.splice.auth.*
1639
import org.lfdecentralizedtrust.splice.config.{NetworkAppClientConfig, SharedSpliceAppParameters}
1740
import org.lfdecentralizedtrust.splice.environment.*
1841
import org.lfdecentralizedtrust.splice.environment.ledger.api.DedupDuration
@@ -31,80 +54,48 @@ import org.lfdecentralizedtrust.splice.migration.{
3154
ParticipantUsersDataRestorer,
3255
}
3356
import org.lfdecentralizedtrust.splice.scan.admin.api.client
57+
import org.lfdecentralizedtrust.splice.scan.admin.api.client.BftScanConnection.BftScanClientConfig
3458
import org.lfdecentralizedtrust.splice.scan.admin.api.client.{
3559
BftScanConnection,
3660
MinimalScanConnection,
3761
SingleScanConnection,
3862
}
39-
import org.lfdecentralizedtrust.splice.scan.admin.api.client.BftScanConnection.BftScanClientConfig
4063
import org.lfdecentralizedtrust.splice.scan.config.ScanAppClientConfig
41-
import org.lfdecentralizedtrust.splice.setup.{
42-
NodeInitializer,
43-
ParticipantInitializer,
44-
ParticipantPartyMigrator,
45-
}
46-
import org.lfdecentralizedtrust.splice.store.{AppStoreWithIngestion, HistoryMetrics, UpdateHistory}
64+
import org.lfdecentralizedtrust.splice.setup.{NodeInitializer, ParticipantInitializer}
65+
import org.lfdecentralizedtrust.splice.store.AppStoreWithIngestion.SpliceLedgerConnectionPriority
4766
import org.lfdecentralizedtrust.splice.store.MultiDomainAcsStore.QueryResult
48-
import org.lfdecentralizedtrust.splice.util.{
49-
AmuletConfigSchedule,
50-
BackupDump,
51-
HasHealth,
52-
PackageVetting,
53-
SpliceCircuitBreaker,
54-
}
67+
import org.lfdecentralizedtrust.splice.store.UpdateHistory.BackfillingRequirement
68+
import org.lfdecentralizedtrust.splice.store.{AppStoreWithIngestion, HistoryMetrics, UpdateHistory}
69+
import org.lfdecentralizedtrust.splice.util.*
5570
import org.lfdecentralizedtrust.splice.validator.admin.http.*
5671
import org.lfdecentralizedtrust.splice.validator.automation.{
5772
ValidatorAutomationService,
5873
ValidatorPackageVettingTrigger,
5974
}
60-
import org.lfdecentralizedtrust.splice.validator.config.{
61-
AppInstance,
62-
MigrateValidatorPartyConfig,
63-
ValidatorAppBackendConfig,
64-
ValidatorCantonIdentifierConfig,
65-
ValidatorOnboardingConfig,
66-
}
75+
import org.lfdecentralizedtrust.splice.validator.config.*
6776
import org.lfdecentralizedtrust.splice.validator.domain.DomainConnector
6877
import org.lfdecentralizedtrust.splice.validator.metrics.ValidatorAppMetrics
69-
import org.lfdecentralizedtrust.splice.validator.migration.DomainMigrationDump
70-
import org.lfdecentralizedtrust.splice.validator.store.ValidatorStore
78+
import org.lfdecentralizedtrust.splice.validator.migration.{
79+
DomainMigrationDump,
80+
ParticipantPartyMigrator,
81+
}
82+
import org.lfdecentralizedtrust.splice.validator.store.{
83+
ValidatorConfigProvider,
84+
ValidatorInternalStore,
85+
ValidatorStore,
86+
}
7187
import org.lfdecentralizedtrust.splice.validator.util.ValidatorUtil
72-
import org.lfdecentralizedtrust.splice.wallet.{ExternalPartyWalletManager, UserWalletManager}
7388
import org.lfdecentralizedtrust.splice.wallet.admin.http.{
7489
HttpExternalWalletHandler,
7590
HttpWalletHandler,
7691
}
7792
import org.lfdecentralizedtrust.splice.wallet.automation.UserWalletAutomationService
7893
import org.lfdecentralizedtrust.splice.wallet.util.ValidatorTopupConfig
79-
import org.lfdecentralizedtrust.tokenstandard.metadata.v1.Resource as TokenStandardMetadataResource
80-
import org.lfdecentralizedtrust.tokenstandard.transferinstruction.v1.Resource as TokenStandardTransferInstructionResource
94+
import org.lfdecentralizedtrust.splice.wallet.{ExternalPartyWalletManager, UserWalletManager}
8195
import org.lfdecentralizedtrust.tokenstandard.allocation.v1.Resource as TokenStandardAllocationResource
8296
import org.lfdecentralizedtrust.tokenstandard.allocationinstruction.v1.Resource as TokenStandardAllocationInstructionResource
83-
import com.digitalasset.canton.concurrent.FutureSupervisor
84-
import com.digitalasset.canton.config.CantonRequireTypes.InstanceName
85-
import com.digitalasset.canton.config.ProcessingTimeout
86-
import com.digitalasset.canton.config.RequireTypes.NonNegativeLong
87-
import com.digitalasset.canton.data.CantonTimestamp
88-
import com.digitalasset.canton.ledger.api.util.DurationConversion
89-
import com.digitalasset.canton.lifecycle.LifeCycle
90-
import com.digitalasset.canton.logging.{NamedLoggerFactory, TracedLogger}
91-
import com.digitalasset.canton.resource.{DbStorage, Storage}
92-
import com.digitalasset.canton.time.Clock
93-
import com.digitalasset.canton.topology.{PartyId, SynchronizerId}
94-
import com.digitalasset.canton.tracing.{TraceContext, TracerProvider}
95-
import com.digitalasset.canton.util.MonadUtil
96-
import com.digitalasset.canton.SynchronizerAlias
97-
import io.grpc.Status
98-
import io.opentelemetry.api.trace.Tracer
99-
import org.apache.pekko.actor.ActorSystem
100-
import org.apache.pekko.http.cors.scaladsl.CorsDirectives.*
101-
import org.apache.pekko.http.cors.scaladsl.settings.CorsSettings
102-
import org.apache.pekko.http.scaladsl.model.HttpMethods
103-
import org.apache.pekko.http.scaladsl.server.Directives.*
104-
import org.apache.pekko.http.scaladsl.server.directives.BasicDirectives
105-
import com.google.protobuf.ByteString
106-
import org.lfdecentralizedtrust.splice.store.AppStoreWithIngestion.SpliceLedgerConnectionPriority
107-
import org.lfdecentralizedtrust.splice.store.UpdateHistory.BackfillingRequirement
97+
import org.lfdecentralizedtrust.tokenstandard.metadata.v1.Resource as TokenStandardMetadataResource
98+
import org.lfdecentralizedtrust.tokenstandard.transferinstruction.v1.Resource as TokenStandardTransferInstructionResource
10899

109100
import scala.concurrent.{ExecutionContextExecutor, Future}
110101
import scala.util.{Failure, Success}
@@ -334,10 +325,15 @@ class ValidatorApp(
334325
.getOrElse(
335326
BaseLedgerConnection.sanitizeUserIdToPartyString(config.ledgerApiUser)
336327
)
328+
val configProvider = new ValidatorConfigProvider(
329+
ValidatorInternalStore(storage, loggerFactory),
330+
loggerFactory,
331+
)
337332
val participantPartyMigrator = new ParticipantPartyMigrator(
338333
connection,
339334
participantAdminConnection,
340335
config.domains.global.alias,
336+
configProvider,
341337
loggerFactory,
342338
)
343339
appInitStep("Migrating party data") {
@@ -823,6 +819,10 @@ class ValidatorApp(
823819
readOnlyLedgerConnection,
824820
loggerFactory,
825821
)
822+
configProvider = new ValidatorConfigProvider(
823+
ValidatorInternalStore(storage, loggerFactory),
824+
loggerFactory,
825+
)
826826
walletManagerOpt =
827827
if (config.enableWallet) {
828828
val externalPartyWalletManager = new ExternalPartyWalletManager(
@@ -1213,6 +1213,7 @@ class ValidatorApp(
12131213
domainTimeAutomationService,
12141214
domainParamsAutomationService,
12151215
store,
1216+
configProvider,
12161217
automation,
12171218
walletManagerOpt,
12181219
timeouts,
@@ -1234,6 +1235,7 @@ object ValidatorApp {
12341235
domainTimeAutomationService: DomainTimeAutomationService,
12351236
domainParamsAutomationService: DomainParamsAutomationService,
12361237
store: ValidatorStore,
1238+
configProvider: ValidatorConfigProvider,
12371239
automation: ValidatorAutomationService,
12381240
walletManager: Option[UserWalletManager],
12391241
timeouts: ProcessingTimeout,

0 commit comments

Comments
 (0)