Skip to content

Commit

Permalink
Enhancement: Use local ledger instead of postgres for account cache (#…
Browse files Browse the repository at this point in the history 
…1085)

Co-authored-by: shiqi.zheng@algorand.com <shiqi.zheng@algorand.com>
Co-authored-by: Eric Warehime <eric.warehime@gmail.com>
Co-authored-by: AlgoStephenAkiki <85183435+AlgoStephenAkiki@users.noreply.github.com>
Co-authored-by: Zeph Grunschlag <tzaffi@users.noreply.github.com>
  • Loading branch information
@winder @shiqizng
@Eric-Warehime @AlgoStephenAkiki @tzaffi
5 people authored Jul 14, 2022
1 parent 94a0bf1 commit 54d39d7
Show file tree
Hide file tree
Showing 39 changed files with 2,510 additions and 712 deletions.
1 change: 0 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,6 @@ The Indexer data directory is the location where the Indexer can store and/or lo

**It is HIGHLY recommended placing the data directory in a separate, stateful directory for production usage of the Indexer.**


### Auto-Loading Configuration

The Indexer will scan the data directory at startup and load certain configuration files if they are present. The files are as follows:
Expand Down
86 changes: 52 additions & 34 deletions api/handlers_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,27 @@ import (
"testing"
"time"

"github.com/algorand/go-algorand/crypto"
"github.com/labstack/echo/v4"
"github.com/sirupsen/logrus"
test2 "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/algorand/go-algorand-sdk/encoding/json"
"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/crypto/merklesignature"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/data/transactions"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/rpcs"
"github.com/algorand/indexer/processor"

"github.com/algorand/indexer/api/generated/v2"
"github.com/algorand/indexer/idb"
"github.com/algorand/indexer/idb/postgres"
pgtest "github.com/algorand/indexer/idb/postgres/testing"
"github.com/algorand/indexer/processor/blockprocessor"
"github.com/algorand/indexer/util/test"
)

Expand Down Expand Up @@ -56,7 +61,7 @@ func testServerImplementation(db idb.IndexerDb) *ServerImplementation {
return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts}
}

func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*postgres.IndexerDb /*db*/, func() /*shutdownFunc*/) {
func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), processor.Processor, *ledger.Ledger) {
_, connStr, shutdownFunc := pgtest.SetupPostgres(t)

db, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil)
Expand All @@ -70,15 +75,18 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin
err = db.LoadGenesis(genesis)
require.NoError(t, err)

err = db.AddBlock(&genesisBlock)
logger, _ := test2.NewNullLogger()
l, err := test.MakeTestLedger(logger)
require.NoError(t, err)

return db, newShutdownFunc
proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, db.AddBlock)
require.NoError(t, err, "failed to open ledger")
return db, newShutdownFunc, proc, l
}

func TestApplicationHandlers(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing an app call txn with ExtraProgramPages, that the creator and another account have opted into
Expand Down Expand Up @@ -112,8 +120,8 @@ func TestApplicationHandlers(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We query the app
Expand Down Expand Up @@ -218,8 +226,9 @@ func TestApplicationHandlers(t *testing.T) {
}

func TestAccountExcludeParameters(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing a creator of an app, an asset, who also holds and has opted-into those apps.
Expand All @@ -238,8 +247,8 @@ func TestAccountExcludeParameters(t *testing.T) {
&appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We look up the address using various exclude parameters.
Expand Down Expand Up @@ -389,8 +398,9 @@ type accountsErrorResponse struct {
}

func TestAccountMaxResultsLimit(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing an address that has created 10 apps, deleted 5 apps, and created 10 assets,
Expand Down Expand Up @@ -443,8 +453,8 @@ func TestAccountMaxResultsLimit(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We look up the address using a ServerImplementation with a maxAccountsAPIResults limit set,
Expand Down Expand Up @@ -768,8 +778,9 @@ func TestAccountMaxResultsLimit(t *testing.T) {
}

func TestBlockNotFound(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // An empty database.
Expand Down Expand Up @@ -833,8 +844,9 @@ func TestInnerTxn(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -845,8 +857,8 @@ func TestInnerTxn(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -881,8 +893,9 @@ func TestInnerTxn(t *testing.T) {
// transaction group does not allow the root transaction to be returned on both
// pages.
func TestPagingRootTxnDeduplication(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -897,8 +910,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

testcases := []struct {
name string
Expand Down Expand Up @@ -1004,8 +1017,9 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
}

func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing a key reg txn with state proof key
Expand Down Expand Up @@ -1044,8 +1058,8 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

e := echo.New()
{
Expand Down Expand Up @@ -1098,8 +1112,9 @@ func TestVersion(t *testing.T) {
///////////
// Given // An API and context
///////////
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()
api := testServerImplementation(db)

e := echo.New()
Expand Down Expand Up @@ -1127,8 +1142,9 @@ func TestVersion(t *testing.T) {
}

func TestAccountClearsNonUTF8(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1147,8 +1163,8 @@ func TestAccountClearsNonUTF8(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

verify := func(params generated.AssetParams) {
compareB64 := func(expected string, actual *[]byte) {
Expand Down Expand Up @@ -1257,8 +1273,9 @@ func TestLookupInnerLogs(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1268,8 +1285,8 @@ func TestLookupInnerLogs(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -1355,8 +1372,9 @@ func TestLookupMultiInnerLogs(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1366,8 +1384,8 @@ func TestLookupMultiInnerLogs(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down
11 changes: 5 additions & 6 deletions cmd/algorand-indexer/api_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ var apiConfigCmd = &cobra.Command{
Long: "api configuration",
Run: func(cmd *cobra.Command, args []string) {
var err error
config.BindFlags(cmd)
config.BindFlagSet(cmd.Flags())
err = configureLogger()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err)
Expand All @@ -35,22 +35,21 @@ var apiConfigCmd = &cobra.Command{
panic(exit{1})
}

options := makeOptions()
var potentialDisabledMapConfig *api.DisabledMapConfig
if suppliedAPIConfigFile != "" {
potentialDisabledMapConfig, err := api.MakeDisabledMapConfigFromFile(swag, suppliedAPIConfigFile)
potentialDisabledMapConfig, err = api.MakeDisabledMapConfigFromFile(swag, suppliedAPIConfigFile)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to created disabled map config from file: %v", err)
panic(exit{1})
}
options.DisabledMapConfig = potentialDisabledMapConfig
}

var displayDisabledMapConfig *api.DisplayDisabledMap
// Show a limited subset
if !showAllDisabled {
displayDisabledMapConfig = api.MakeDisplayDisabledMapFromConfig(swag, options.DisabledMapConfig, true)
displayDisabledMapConfig = api.MakeDisplayDisabledMapFromConfig(swag, potentialDisabledMapConfig, true)
} else {
displayDisabledMapConfig = api.MakeDisplayDisabledMapFromConfig(swag, options.DisabledMapConfig, false)
displayDisabledMapConfig = api.MakeDisplayDisabledMapFromConfig(swag, potentialDisabledMapConfig, false)
}

output, err := displayDisabledMapConfig.String()
Expand Down
Loading

0 comments on commit 54d39d7

Please sign in to comment.