Be consistent with how we report init status. (#10498)

Also make half-joined raft peers consider storage to be initialized, whether or not they're sealed.

changelog/10498.txt

@@ -0,0 +1,4 @@
+```release-note:bug
+core: Make all APIs that report init status consistent, and make them report
+initialized=true when a Raft join is in progress.
+```

command/server.go

@@ -656,7 +656,7 @@ func (c *ServerCommand) runRecoveryMode() int {
 	}
 
 	if sealConfigError != nil {
-		init, err := core.Initialized(context.Background())
+		init, err := core.InitializedLocally(context.Background())
 		if err != nil {
 			c.UI.Error(fmt.Sprintf("Error checking if core is initialized: %v", err))
 			return 1
@@ -1794,7 +1794,7 @@ CLUSTER_SYNTHESIS_COMPLETE:
 	}
 
 	if sealConfigError != nil {
-		init, err := core.Initialized(context.Background())
+		init, err := core.InitializedLocally(context.Background())
 		if err != nil {
 			c.UI.Error(fmt.Sprintf("Error checking if core is initialized: %v", err))
 			return 1

http/sys_seal.go

@@ -167,8 +167,13 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req
 
 	sealed := core.Sealed()
 
+	initialized, err := core.Initialized(ctx)
+	if err != nil {
+		respondError(w, http.StatusInternalServerError, err)
+		return
+	}
+
 	var sealConfig *vault.SealConfig
-	var err error
 	if core.SealAccess().RecoveryKeySupported() {
 		sealConfig, err = core.SealAccess().RecoveryConfig(ctx)
 	} else {
@@ -182,7 +187,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req
 	if sealConfig == nil {
 		respondOk(w, &SealStatusResponse{
 			Type:         core.SealAccess().BarrierType(),
-			Initialized:  false,
+			Initialized:  initialized,
 			Sealed:       true,
 			RecoverySeal: core.SealAccess().RecoveryKeySupported(),
 			StorageType:  core.StorageType(),
@@ -211,7 +216,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req
 
 	respondOk(w, &SealStatusResponse{
 		Type:         sealConfig.Type,
-		Initialized:  true,
+		Initialized:  initialized,
 		Sealed:       sealed,
 		T:            sealConfig.SecretThreshold,
 		N:            sealConfig.SecretShares,

vault/external_tests/raft/raft_test.go

@@ -875,3 +875,100 @@ func BenchmarkRaft_SingleNode(b *testing.B) {
 
 	b.Run("256b", func(b *testing.B) { bench(b, 25) })
 }
+
+func TestRaft_Join_InitStatus(t *testing.T) {
+	t.Parallel()
+	var conf vault.CoreConfig
+	var opts = vault.TestClusterOptions{HandlerFunc: vaulthttp.Handler}
+	teststorage.RaftBackendSetup(&conf, &opts)
+	opts.SetupFunc = nil
+	cluster := vault.NewTestCluster(t, &conf, &opts)
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	addressProvider := &testhelpers.TestRaftServerAddressProvider{Cluster: cluster}
+
+	leaderCore := cluster.Cores[0]
+	leaderAPI := leaderCore.Client.Address()
+	atomic.StoreUint32(&vault.TestingUpdateClusterAddr, 1)
+
+	// Seal the leader so we can install an address provider
+	{
+		testhelpers.EnsureCoreSealed(t, leaderCore)
+		leaderCore.UnderlyingRawStorage.(*raft.RaftBackend).SetServerAddressProvider(addressProvider)
+		cluster.UnsealCore(t, leaderCore)
+		vault.TestWaitActive(t, leaderCore.Core)
+	}
+
+	joinFunc := func(client *api.Client) {
+		req := &api.RaftJoinRequest{
+			LeaderAPIAddr: leaderAPI,
+			LeaderCACert:  string(cluster.CACertPEM),
+		}
+		resp, err := client.Sys().RaftJoin(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !resp.Joined {
+			t.Fatalf("failed to join raft cluster")
+		}
+	}
+
+	verifyInitStatus := func(coreIdx int, expected bool) {
+		t.Helper()
+		client := cluster.Cores[coreIdx].Client
+
+		initialized, err := client.Sys().InitStatus()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if initialized != expected {
+			t.Errorf("core %d: expected init=%v, sys/init returned %v", coreIdx, expected, initialized)
+		}
+
+		status, err := client.Sys().SealStatus()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if status.Initialized != expected {
+			t.Errorf("core %d: expected init=%v, sys/seal-status returned %v", coreIdx, expected, status.Initialized)
+		}
+
+		health, err := client.Sys().Health()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if health.Initialized != expected {
+			t.Errorf("core %d: expected init=%v, sys/health returned %v", coreIdx, expected, health.Initialized)
+		}
+	}
+
+	for i := range cluster.Cores {
+		verifyInitStatus(i, i < 1)
+	}
+
+	joinFunc(cluster.Cores[1].Client)
+	for i, core := range cluster.Cores {
+		verifyInitStatus(i, i < 2)
+		if i == 1 {
+			cluster.UnsealCore(t, core)
+			verifyInitStatus(i, true)
+		}
+	}
+
+	joinFunc(cluster.Cores[2].Client)
+	for i, core := range cluster.Cores {
+		verifyInitStatus(i, true)
+		if i == 2 {
+			cluster.UnsealCore(t, core)
+			verifyInitStatus(i, true)
+		}
+	}
+
+	testhelpers.WaitForActiveNodeAndStandbys(t, cluster)
+	for i := range cluster.Cores {
+		verifyInitStatus(i, true)
+	}
+}

vault/init.go

@@ -69,8 +69,35 @@ func (c *Core) InitializeRecovery(ctx context.Context) error {
 	return nil
 }
 
-// Initialized checks if the Vault is already initialized
+// Initialized checks if the Vault is already initialized.  This means one of
+// two things: either the barrier has been created (with keyring and master key)
+// and the seal config written to storage, or Raft is forming a cluster and a
+// join/bootstrap is in progress.
 func (c *Core) Initialized(ctx context.Context) (bool, error) {
+	// Check the barrier first
+	init, err := c.InitializedLocally(ctx)
+	if err != nil || init {
+		return init, err
+	}
+
+	if c.isRaftUnseal() {
+		return true, nil
+	}
+
+	rb := c.getRaftBackend()
+	if rb != nil && rb.Initialized() {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// InitializedLocally checks if the Vault is already initialized from the
+// local node's perspective.  This is the same thing as Initialized, unless
+// using Raft, in which case Initialized may return true (because a peer
+// we're joining to has been initialized) while InitializedLocally returns
+// false (because we're not done bootstrapping raft on the local node).
+func (c *Core) InitializedLocally(ctx context.Context) (bool, error) {
 	// Check the barrier first
 	init, err := c.barrier.Initialized(ctx)
 	if err != nil {

vault/raft.go

@@ -716,7 +716,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 		return false, errors.New("raft backend not in use")
 	}
 
-	init, err := c.Initialized(ctx)
+	init, err := c.InitializedLocally(ctx)
 	if err != nil {
 		return false, errwrap.Wrapf("failed to check if core is initialized: {{err}}", err)
 	}
@@ -794,7 +794,7 @@ func (c *Core) JoinRaftCluster(ctx context.Context, leaderInfos []*raft.LeaderJo
 				return errors.New("raft leader address not provided")
 			}
 
-			init, err := c.Initialized(ctx)
+			init, err := c.InitializedLocally(ctx)
 			if err != nil {
 				return errwrap.Wrapf("failed to check if core is initialized: {{err}}", err)
 			}