diff --git a/changelog/10498.txt b/changelog/10498.txt new file mode 100644 index 000000000000..15cd8a8c5dc3 --- /dev/null +++ b/changelog/10498.txt @@ -0,0 +1,4 @@ +```release-note:bug +core: Make all APIs that report init status consistent, and make them report +initialized=true when a Raft join is in progress. +``` \ No newline at end of file diff --git a/command/server.go b/command/server.go index 306822eab21e..a7b652bec409 100644 --- a/command/server.go +++ b/command/server.go @@ -656,7 +656,7 @@ func (c *ServerCommand) runRecoveryMode() int { } if sealConfigError != nil { - init, err := core.Initialized(context.Background()) + init, err := core.InitializedLocally(context.Background()) if err != nil { c.UI.Error(fmt.Sprintf("Error checking if core is initialized: %v", err)) return 1 @@ -1794,7 +1794,7 @@ CLUSTER_SYNTHESIS_COMPLETE: } if sealConfigError != nil { - init, err := core.Initialized(context.Background()) + init, err := core.InitializedLocally(context.Background()) if err != nil { c.UI.Error(fmt.Sprintf("Error checking if core is initialized: %v", err)) return 1 diff --git a/http/sys_seal.go b/http/sys_seal.go index 000f438b22a4..7b86e43645f7 100644 --- a/http/sys_seal.go +++ b/http/sys_seal.go @@ -167,8 +167,13 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req sealed := core.Sealed() + initialized, err := core.Initialized(ctx) + if err != nil { + respondError(w, http.StatusInternalServerError, err) + return + } + var sealConfig *vault.SealConfig - var err error if core.SealAccess().RecoveryKeySupported() { sealConfig, err = core.SealAccess().RecoveryConfig(ctx) } else { @@ -182,7 +187,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req if sealConfig == nil { respondOk(w, &SealStatusResponse{ Type: core.SealAccess().BarrierType(), - Initialized: false, + Initialized: initialized, Sealed: true, RecoverySeal: core.SealAccess().RecoveryKeySupported(), StorageType: core.StorageType(), @@ -211,7 +216,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req respondOk(w, &SealStatusResponse{ Type: sealConfig.Type, - Initialized: true, + Initialized: initialized, Sealed: sealed, T: sealConfig.SecretThreshold, N: sealConfig.SecretShares, diff --git a/vault/external_tests/raft/raft_test.go b/vault/external_tests/raft/raft_test.go index 2014fe455d2c..e0c25d427762 100644 --- a/vault/external_tests/raft/raft_test.go +++ b/vault/external_tests/raft/raft_test.go @@ -875,3 +875,100 @@ func BenchmarkRaft_SingleNode(b *testing.B) { b.Run("256b", func(b *testing.B) { bench(b, 25) }) } + +func TestRaft_Join_InitStatus(t *testing.T) { + t.Parallel() + var conf vault.CoreConfig + var opts = vault.TestClusterOptions{HandlerFunc: vaulthttp.Handler} + teststorage.RaftBackendSetup(&conf, &opts) + opts.SetupFunc = nil + cluster := vault.NewTestCluster(t, &conf, &opts) + cluster.Start() + defer cluster.Cleanup() + + addressProvider := &testhelpers.TestRaftServerAddressProvider{Cluster: cluster} + + leaderCore := cluster.Cores[0] + leaderAPI := leaderCore.Client.Address() + atomic.StoreUint32(&vault.TestingUpdateClusterAddr, 1) + + // Seal the leader so we can install an address provider + { + testhelpers.EnsureCoreSealed(t, leaderCore) + leaderCore.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider) + cluster.UnsealCore(t, leaderCore) + vault.TestWaitActive(t, leaderCore.Core) + } + + joinFunc := func(client *api.Client) { + req := &api.RaftJoinRequest{ + LeaderAPIAddr: leaderAPI, + LeaderCACert: string(cluster.CACertPEM), + } + resp, err := client.Sys().RaftJoin(req) + if err != nil { + t.Fatal(err) + } + if !resp.Joined { + t.Fatalf("failed to join raft cluster") + } + } + + verifyInitStatus := func(coreIdx int, expected bool) { + t.Helper() + client := cluster.Cores[coreIdx].Client + + initialized, err := client.Sys().InitStatus() + if err != nil { + t.Fatal(err) + } + + if initialized != expected { + t.Errorf("core %d: expected init=%v, sys/init returned %v", coreIdx, expected, initialized) + } + + status, err := client.Sys().SealStatus() + if err != nil { + t.Fatal(err) + } + + if status.Initialized != expected { + t.Errorf("core %d: expected init=%v, sys/seal-status returned %v", coreIdx, expected, status.Initialized) + } + + health, err := client.Sys().Health() + if err != nil { + t.Fatal(err) + } + if health.Initialized != expected { + t.Errorf("core %d: expected init=%v, sys/health returned %v", coreIdx, expected, health.Initialized) + } + } + + for i := range cluster.Cores { + verifyInitStatus(i, i < 1) + } + + joinFunc(cluster.Cores[1].Client) + for i, core := range cluster.Cores { + verifyInitStatus(i, i < 2) + if i == 1 { + cluster.UnsealCore(t, core) + verifyInitStatus(i, true) + } + } + + joinFunc(cluster.Cores[2].Client) + for i, core := range cluster.Cores { + verifyInitStatus(i, true) + if i == 2 { + cluster.UnsealCore(t, core) + verifyInitStatus(i, true) + } + } + + testhelpers.WaitForActiveNodeAndStandbys(t, cluster) + for i := range cluster.Cores { + verifyInitStatus(i, true) + } +} diff --git a/vault/init.go b/vault/init.go index 87c0eb63a89b..5fd69876a79b 100644 --- a/vault/init.go +++ b/vault/init.go @@ -69,8 +69,35 @@ func (c *Core) InitializeRecovery(ctx context.Context) error { return nil } -// Initialized checks if the Vault is already initialized +// Initialized checks if the Vault is already initialized. This means one of +// two things: either the barrier has been created (with keyring and master key) +// and the seal config written to storage, or Raft is forming a cluster and a +// join/bootstrap is in progress. func (c *Core) Initialized(ctx context.Context) (bool, error) { + // Check the barrier first + init, err := c.InitializedLocally(ctx) + if err != nil || init { + return init, err + } + + if c.isRaftUnseal() { + return true, nil + } + + rb := c.getRaftBackend() + if rb != nil && rb.Initialized() { + return true, nil + } + + return false, nil +} + +// InitializedLocally checks if the Vault is already initialized from the +// local node's perspective. This is the same thing as Initialized, unless +// using Raft, in which case Initialized may return true (because a peer +// we're joining to has been initialized) while InitializedLocally returns +// false (because we're not done bootstrapping raft on the local node). +func (c *Core) InitializedLocally(ctx context.Context) (bool, error) { // Check the barrier first init, err := c.barrier.Initialized(ctx) if err != nil { diff --git a/vault/raft.go b/vault/raft.go index 0cd3b124fdd1..84cb7fbe9b22 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -716,7 +716,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo return false, errors.New("raft backend not in use") } - init, err := c.Initialized(ctx) + init, err := c.InitializedLocally(ctx) if err != nil { return false, errwrap.Wrapf("failed to check if core is initialized: {{err}}", err) } @@ -794,7 +794,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo return errors.New("raft leader address not provided") } - init, err := c.Initialized(ctx) + init, err := c.InitializedLocally(ctx) if err != nil { return errwrap.Wrapf("failed to check if core is initialized: {{err}}", err) }