From a96fa7063c3ce63a3543ad836e7890edeb310f6f Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 12 May 2022 09:59:48 -0400 Subject: [PATCH 01/37] integrate block processor --- README.md | 2 +- cmd/algorand-indexer/daemon.go | 76 ++++++++++++++++++++++++----- cmd/algorand-indexer/daemon_test.go | 38 +++++++++++++-- importer/importer.go | 14 +++++- misc/Dockerfile | 1 + misc/e2elive.py | 1 + 6 files changed, 116 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ee2e9d78c..bbd237b32 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Settings can be provided from the command line, a configuration file, or an envi | default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | | max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | | default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | - +| data-dir | i | data-dir | INDEXER_DATA | ## Command line The command line arguments always take priority over the config file and environment variables. diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index b105177bb..c226f4309 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -5,14 +5,24 @@ import ( "fmt" "os" "os/signal" + "path" "strings" "sync" "syscall" "time" + "github.com/algorand/go-algorand-sdk/client/v2/algod" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/indexer/processor" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" "github.com/spf13/cobra" "github.com/spf13/viper" + algodConfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/api" "github.com/algorand/indexer/api/generated/v2" @@ -123,7 +133,17 @@ var daemonCmd = &cobra.Command{ defer db.Close() var wg sync.WaitGroup if bot != nil { + if indexerDataDir == "" { + fmt.Fprint(os.Stderr, "missing indexer data directory") + os.Exit(1) + } wg.Add(1) + genesis, err := getGenesis(bot.Algod()) + maybeFail(err, "Error getting genesis") + genesisBlock, err := getGenesisBlock(bot.Algod()) + maybeFail(err, "Error getting genesis block") + initState, err := util.CreateInitState(&genesis, &genesisBlock) + maybeFail(err, "Error getting genesis block") go func() { defer wg.Done() @@ -134,14 +154,18 @@ var daemonCmd = &cobra.Command{ genesisReader := importer.GetGenesisFile(genesisJSONPath, bot.Algod(), logger) _, err := importer.EnsureInitialImport(db, genesisReader, logger) maybeFail(err, "importer.EnsureInitialImport() error") - logger.Info("Initializing block import handler.") - - nextRound, err := db.GetNextRoundToAccount() - maybeFail(err, "failed to get next round, %v", err) - bot.SetNextRound(nextRound) + logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) - handler := blockHandler(imp, 1*time.Second) + logger.Info("Initializing local ledger.") + localLedger, err := ledger.OpenLedger(logging.NewLogger(), path.Dir(indexerDataDir), false, initState, algodConfig.GetDefaultLocal()) + bot.SetNextRound(uint64(localLedger.Latest()) + 1) + + proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportValidatedBlock) + if err != nil { + maybeFail(err, err.Error()) + } + handler := blockHandler(proc, 1*time.Second) bot.SetBlockHandler(handler) logger.Info("Starting block importer.") @@ -265,10 +289,10 @@ func makeOptions() (options api.ExtraOptions) { // blockHandler creates a handler complying to the fetcher block handler interface. In case of a failure it keeps // attempting to add the block until the fetcher shuts down. -func blockHandler(imp importer.Importer, retryDelay time.Duration) func(context.Context, *rpcs.EncodedBlockCert) error { +func blockHandler(proc processor.Processor, retryDelay time.Duration) func(context.Context, *rpcs.EncodedBlockCert) error { return func(ctx context.Context, block *rpcs.EncodedBlockCert) error { for { - err := handleBlock(block, imp) + err := handleBlock(block, proc) if err == nil { // return on success. return nil @@ -285,12 +309,12 @@ func blockHandler(imp importer.Importer, retryDelay time.Duration) func(context. } } -func handleBlock(block *rpcs.EncodedBlockCert, imp importer.Importer) error { +func handleBlock(block *rpcs.EncodedBlockCert, proc processor.Processor) error { start := time.Now() - err := imp.ImportBlock(block) + err := proc.Process(block) if err != nil { logger.WithError(err).Errorf( - "adding block %d to database failed", block.Block.Round()) + "block %d import failed", block.Block.Round()) return fmt.Errorf("handleBlock() err: %w", err) } dt := time.Since(start) @@ -313,3 +337,33 @@ func handleBlock(block *rpcs.EncodedBlockCert, imp importer.Importer) error { return nil } + +func getGenesis(client *algod.Client) (bookkeeping.Genesis, error) { + data, err := client.GetGenesis().Do(context.Background()) + if err != nil { + return bookkeeping.Genesis{}, fmt.Errorf("getGenesis() client err: %w", err) + } + + var res bookkeeping.Genesis + err = protocol.DecodeJSON([]byte(data), &res) + if err != nil { + return bookkeeping.Genesis{}, fmt.Errorf("getGenesis() decode err: %w", err) + } + + return res, nil +} + +func getGenesisBlock(client *algod.Client) (bookkeeping.Block, error) { + data, err := client.BlockRaw(0).Do(context.Background()) + if err != nil { + return bookkeeping.Block{}, fmt.Errorf("getGenesisBlock() client err: %w", err) + } + + var block rpcs.EncodedBlockCert + err = protocol.Decode(data, &block) + if err != nil { + return bookkeeping.Block{}, fmt.Errorf("getGenesisBlock() decode err: %w", err) + } + + return block.Block, nil +} diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index d83c33dbb..f39efa85c 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -3,12 +3,20 @@ package main import ( "context" "errors" + "log" "sync" "testing" "time" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" + test_util "github.com/algorand/indexer/util/test" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" ) @@ -16,9 +24,13 @@ import ( type mockImporter struct { } -var errMockImportBlock = errors.New("mock import block error") +var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 processor.nextRoundToProcess: 1") func (imp *mockImporter) ImportBlock(blockContainer *rpcs.EncodedBlockCert) error { + return nil +} + +func (imp *mockImporter) ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error { return errMockImportBlock } @@ -33,7 +45,12 @@ func TestImportRetryAndCancel(t *testing.T) { // create handler with mock importer and start, it should generate errors until cancelled. imp := &mockImporter{} - handler := blockHandler(imp, 50*time.Millisecond) + l := makeTestLedger(t, "local_ledger") + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, nil) + assert.Nil(t, err) + proc.SetHandler(imp.ImportValidatedBlock) + handler := blockHandler(proc, 50*time.Millisecond) var wg sync.WaitGroup wg.Add(1) go func() { @@ -54,7 +71,7 @@ func TestImportRetryAndCancel(t *testing.T) { } for _, entry := range hook.Entries { - assert.Equal(t, entry.Message, "adding block 1234 to database failed") + assert.Equal(t, entry.Message, "block 1234 import failed") assert.Equal(t, entry.Data["error"], errMockImportBlock) } @@ -62,3 +79,18 @@ func TestImportRetryAndCancel(t *testing.T) { cancel() wg.Wait() } + +func makeTestLedger(t *testing.T, prefix string) *ledger.Ledger { + // initialize local ledger + genesis := test_util.MakeGenesis() + genesisBlock := test_util.MakeGenesisBlock() + initState, err := util.CreateInitState(&genesis, &genesisBlock) + if err != nil { + log.Panicf("test init err: %v", err) + } + l, err := ledger.OpenLedger(logging.NewLogger(), prefix, true, initState, config.GetDefaultLocal()) + if err != nil { + log.Panicf("test init err: %v", err) + } + return l +} diff --git a/importer/importer.go b/importer/importer.go index 70532dd34..9ccc46334 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -4,14 +4,15 @@ import ( "fmt" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/idb" ) // Importer is used to import blocks into an idb.IndexerDb object. type Importer interface { ImportBlock(blockContainer *rpcs.EncodedBlockCert) error + ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error } type importerImpl struct { @@ -29,6 +30,17 @@ func (imp *importerImpl) ImportBlock(blockContainer *rpcs.EncodedBlockCert) erro return imp.db.AddBlock(&blockContainer.Block) } +// ImportValidatedBlock processes a validated block and adds it to the IndexerDb +func (imp *importerImpl) ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error { + block := vb.Block() + + _, ok := config.Consensus[block.CurrentProtocol] + if !ok { + return fmt.Errorf("protocol %s not found", block.CurrentProtocol) + } + return imp.db.AddBlock(&block) +} + // NewImporter creates a new importer object. func NewImporter(db idb.IndexerDb) Importer { return &importerImpl{db: db} diff --git a/misc/Dockerfile b/misc/Dockerfile index 8499c7ed2..8f7560431 100644 --- a/misc/Dockerfile +++ b/misc/Dockerfile @@ -25,5 +25,6 @@ RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer RUN make RUN pip3 install -r misc/requirements.txt +ENV INDEXER_DATA="${HOME}/indexer" # Run test script ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && python3 misc/e2elive.py --connection-string \"$CONNECTION_STRING\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] diff --git a/misc/e2elive.py b/misc/e2elive.py index 327ad1dc5..cd36f6e8d 100644 --- a/misc/e2elive.py +++ b/misc/e2elive.py @@ -110,6 +110,7 @@ def main(): sys.stderr.write(indexerout.dump()) return 1 try: + logger.info('reached expected round={}'.format(lastblock)) xrun(['python3', 'misc/validate_accounting.py', '--verbose', '--algod', algoddir, '--indexer', indexerurl], timeout=20) xrun(['go', 'run', 'cmd/e2equeries/main.go', '-pg', psqlstring, '-q'], timeout=15) From ed32cb12e23a1393101f7deee78852a51872f739 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 12 May 2022 10:40:41 -0400 Subject: [PATCH 02/37] update ledger path --- cmd/algorand-indexer/daemon.go | 17 +++++++++-------- misc/Dockerfile | 2 +- misc/e2elive.py | 1 - 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index c226f4309..9d692230d 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -6,23 +6,18 @@ import ( "os" "os/signal" "path" + "path/filepath" "strings" "sync" "syscall" "time" "github.com/algorand/go-algorand-sdk/client/v2/algod" + algodConfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/indexer/processor" - "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" - "github.com/spf13/cobra" - "github.com/spf13/viper" - - algodConfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/api" "github.com/algorand/indexer/api/generated/v2" @@ -30,7 +25,12 @@ import ( "github.com/algorand/indexer/fetcher" "github.com/algorand/indexer/idb" "github.com/algorand/indexer/importer" + "github.com/algorand/indexer/processor" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/metrics" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) var ( @@ -158,7 +158,8 @@ var daemonCmd = &cobra.Command{ logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) logger.Info("Initializing local ledger.") - localLedger, err := ledger.OpenLedger(logging.NewLogger(), path.Dir(indexerDataDir), false, initState, algodConfig.GetDefaultLocal()) + localLedger, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(indexerDataDir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) + maybeFail(err, "%v", err) bot.SetNextRound(uint64(localLedger.Latest()) + 1) proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportValidatedBlock) diff --git a/misc/Dockerfile b/misc/Dockerfile index 8f7560431..6c5e5c5f4 100644 --- a/misc/Dockerfile +++ b/misc/Dockerfile @@ -25,6 +25,6 @@ RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer RUN make RUN pip3 install -r misc/requirements.txt -ENV INDEXER_DATA="${HOME}/indexer" +ENV INDEXER_DATA="${HOME}/indexer/" # Run test script ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && python3 misc/e2elive.py --connection-string \"$CONNECTION_STRING\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] diff --git a/misc/e2elive.py b/misc/e2elive.py index cd36f6e8d..327ad1dc5 100644 --- a/misc/e2elive.py +++ b/misc/e2elive.py @@ -110,7 +110,6 @@ def main(): sys.stderr.write(indexerout.dump()) return 1 try: - logger.info('reached expected round={}'.format(lastblock)) xrun(['python3', 'misc/validate_accounting.py', '--verbose', '--algod', algoddir, '--indexer', indexerurl], timeout=20) xrun(['go', 'run', 'cmd/e2equeries/main.go', '-pg', psqlstring, '-q'], timeout=15) From 8568fa22b87ac1f89dd0e7e10168956209982510 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 12 May 2022 12:11:28 -0400 Subject: [PATCH 03/37] add ledger close --- cmd/algorand-indexer/daemon.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 9d692230d..e084d974e 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -160,6 +160,7 @@ var daemonCmd = &cobra.Command{ logger.Info("Initializing local ledger.") localLedger, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(indexerDataDir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) maybeFail(err, "%v", err) + defer localLedger.Close() bot.SetNextRound(uint64(localLedger.Latest()) + 1) proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportValidatedBlock) From fda48cf4f276385e62e4b4f70d1b3a8062b9de49 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 13 May 2022 14:29:43 -0400 Subject: [PATCH 04/37] refactor AddBlock --- cmd/algorand-indexer/daemon.go | 3 +- cmd/algorand-indexer/import.go | 47 ---------- cmd/algorand-indexer/main.go | 3 - cmd/import-validator/core/service.go | 23 ++--- idb/dummy/dummy.go | 3 +- idb/idb.go | 3 +- idb/postgres/postgres.go | 34 ++------ importer/helper.go | 124 --------------------------- importer/importer.go | 19 +--- 9 files changed, 29 insertions(+), 230 deletions(-) delete mode 100644 cmd/algorand-indexer/import.go diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index e084d974e..c226e2fee 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -59,6 +59,7 @@ var ( defaultApplicationsLimit uint32 enableAllParameters bool indexerDataDir string + genesisJSONPath string ) var daemonCmd = &cobra.Command{ @@ -163,7 +164,7 @@ var daemonCmd = &cobra.Command{ defer localLedger.Close() bot.SetNextRound(uint64(localLedger.Latest()) + 1) - proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportValidatedBlock) + proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportBlock) if err != nil { maybeFail(err, err.Error()) } diff --git a/cmd/algorand-indexer/import.go b/cmd/algorand-indexer/import.go deleted file mode 100644 index 331611e0a..000000000 --- a/cmd/algorand-indexer/import.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - - "github.com/algorand/indexer/config" - "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/importer" -) - -var importCmd = &cobra.Command{ - Use: "import", - Short: "import block file or tar file of blocks", - Long: "import block file or tar file of blocks. arguments are interpret as file globs (e.g. *.tar.bz2)", - Run: func(cmd *cobra.Command, args []string) { - config.BindFlags(cmd) - err := configureLogger() - if err != nil { - fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err) - os.Exit(1) - } - - db, availableCh := indexerDbFromFlags(idb.IndexerDbOptions{}) - defer db.Close() - <-availableCh - - helper := importer.NewImportHelper( - genesisJSONPath, - blockFileLimit, - logger) - - helper.Import(db, args) - }, -} - -var ( - genesisJSONPath string - blockFileLimit int -) - -func init() { - importCmd.Flags().StringVarP(&genesisJSONPath, "genesis", "g", "", "path to genesis.json") - importCmd.Flags().IntVarP(&blockFileLimit, "block-file-limit", "", 0, "number of block files to process (for debugging)") -} diff --git a/cmd/algorand-indexer/main.go b/cmd/algorand-indexer/main.go index f436875be..651ce798d 100644 --- a/cmd/algorand-indexer/main.go +++ b/cmd/algorand-indexer/main.go @@ -134,8 +134,6 @@ func init() { logger.SetOutput(os.Stdout) logger.SetLevel(log.InfoLevel) - rootCmd.AddCommand(importCmd) - importCmd.Hidden = true rootCmd.AddCommand(daemonCmd) rootCmd.AddCommand(apiConfigCmd) @@ -153,7 +151,6 @@ func init() { cmd.Flags().BoolVarP(&doVersion, "version", "v", false, "print version and exit") } addFlags(daemonCmd) - addFlags(importCmd) viper.RegisterAlias("postgres", "postgres-connection-string") diff --git a/cmd/import-validator/core/service.go b/cmd/import-validator/core/service.go index 075c7d46b..fe341f20e 100644 --- a/cmd/import-validator/core/service.go +++ b/cmd/import-validator/core/service.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/algorand/indexer/processor/blockprocessor" "github.com/sirupsen/logrus" "github.com/algorand/go-algorand-sdk/client/v2/algod" @@ -92,7 +93,8 @@ func openIndexerDb(postgresConnStr string, genesis *bookkeeping.Genesis, genesis } if nextRound == 0 { - err = db.AddBlock(genesisBlock) + vb := ledgercore.MakeValidatedBlock(*genesisBlock, ledgercore.StateDelta{}) + err = db.AddBlock(&vb) if err != nil { return nil, fmt.Errorf("openIndexerDb() err: %w", err) } @@ -348,15 +350,16 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg }() if nextRoundLedger >= nextRoundIndexer { - wg.Add(1) - go func() { - start := time.Now() - err1 = db.AddBlock(&block.Block) - fmt.Printf( - "%d transactions imported in %v\n", - len(block.Block.Payset), time.Since(start)) - wg.Done() - }() + prc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + if err != nil { + return fmt.Errorf("catchup() err: %w", err) + } + blockCert := rpcs.EncodedBlockCert{Block: block.Block, Certificate: block.Certificate} + start := time.Now() + prc.Process(&blockCert) + fmt.Printf( + "%d transactions imported in %v\n", + len(block.Block.Payset), time.Since(start)) } wg.Wait() diff --git a/idb/dummy/dummy.go b/idb/dummy/dummy.go index d4cc71980..7e7c5b842 100644 --- a/idb/dummy/dummy.go +++ b/idb/dummy/dummy.go @@ -5,6 +5,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" log "github.com/sirupsen/logrus" "github.com/algorand/indexer/idb" @@ -22,7 +23,7 @@ func IndexerDb() idb.IndexerDb { func (db *dummyIndexerDb) Close() { } -func (db *dummyIndexerDb) AddBlock(block *bookkeeping.Block) error { +func (db *dummyIndexerDb) AddBlock(block *ledgercore.ValidatedBlock) error { db.log.Printf("AddBlock") return nil } diff --git a/idb/idb.go b/idb/idb.go index 82901ed1c..9d7ea63e3 100644 --- a/idb/idb.go +++ b/idb/idb.go @@ -13,6 +13,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" models "github.com/algorand/indexer/api/generated/v2" ) @@ -161,7 +162,7 @@ type IndexerDb interface { Close() // Import a block and do the accounting. - AddBlock(block *bookkeeping.Block) error + AddBlock(block *ledgercore.ValidatedBlock) error LoadGenesis(genesis bookkeeping.Genesis) (err error) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 1e3e490de..706de0f8c 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -39,7 +39,6 @@ import ( pgutil "github.com/algorand/indexer/idb/postgres/internal/util" "github.com/algorand/indexer/idb/postgres/internal/writer" "github.com/algorand/indexer/util" - "github.com/algorand/indexer/util/metrics" ) var serializable = pgx.TxOptions{IsoLevel: pgx.Serializable} // be a real ACID database @@ -251,7 +250,8 @@ func prepareEvalResources(l *ledger_for_evaluator.LedgerForEvaluator, block *boo } // AddBlock is part of idb.IndexerDb. -func (db *IndexerDb) AddBlock(block *bookkeeping.Block) error { +func (db *IndexerDb) AddBlock(vb *ledgercore.ValidatedBlock) error { + block := vb.Block() db.log.Printf("adding block %d", block.Round()) db.accountingLock.Lock() @@ -282,7 +282,7 @@ func (db *IndexerDb) AddBlock(block *bookkeeping.Block) error { if block.Round() == basics.Round(0) { // Block 0 is special, we cannot run the evaluator on it. - err := w.AddBlock0(block) + err := w.AddBlock0(&block) if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } @@ -308,36 +308,16 @@ func (db *IndexerDb) AddBlock(block *bookkeeping.Block) error { f := func(tx pgx.Tx) error { if !protoChanged { - err := writer.AddTransactions(block, block.Payset, tx) + err := writer.AddTransactions(&block, block.Payset, tx) if err != nil { return err } } - return writer.AddTransactionParticipation(block, tx) + return writer.AddTransactionParticipation(&block, tx) } err0 = db.txWithRetry(serializable, f) }() - ledgerForEval, err := - ledger_for_evaluator.MakeLedgerForEvaluator(tx, block.Round()-1) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - defer ledgerForEval.Close() - - resources, err := prepareEvalResources(&ledgerForEval, block) - if err != nil { - return fmt.Errorf("AddBlock() eval err: %w", err) - } - - start := time.Now() - delta, modifiedTxns, err := - ledger.EvalForIndexer(ledgerForEval, block, proto, resources) - if err != nil { - return fmt.Errorf("AddBlock() eval err: %w", err) - } - metrics.PostgresEvalTimeSeconds.Observe(time.Since(start).Seconds()) - var err1 error // Skip if transaction writing has already started. if protoChanged { @@ -347,13 +327,13 @@ func (db *IndexerDb) AddBlock(block *bookkeeping.Block) error { defer wg.Done() f := func(tx pgx.Tx) error { - return writer.AddTransactions(block, modifiedTxns, tx) + return writer.AddTransactions(&block, block.Payset, tx) } err1 = db.txWithRetry(serializable, f) }() } - err = w.AddBlock(block, modifiedTxns, delta) + err = w.AddBlock(&block, block.Payset, vb.Delta()) if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } diff --git a/importer/helper.go b/importer/helper.go index 7f1001373..a384daafc 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -1,8 +1,6 @@ package importer import ( - "archive/tar" - "compress/bzip2" "context" "errors" "fmt" @@ -10,16 +8,13 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" "strconv" "strings" - "time" "github.com/algorand/go-algorand-sdk/client/v2/algod" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/rpcs" log "github.com/sirupsen/logrus" "github.com/algorand/indexer/idb" @@ -45,43 +40,6 @@ type ImportHelper struct { Log *log.Logger } -// Import is the main ImportHelper function that glues together a directory full of block files and an Importer objects. -func (h *ImportHelper) Import(db idb.IndexerDb, args []string) { - // Initial import if needed. - genesisReader := GetGenesisFile(h.GenesisJSONPath, nil, h.Log) - _, err := EnsureInitialImport(db, genesisReader, h.Log) - maybeFail(err, h.Log, "EnsureInitialImport() error") - imp := NewImporter(db) - blocks := 0 - txCount := 0 - start := time.Now() - for _, fname := range args { - matches, err := filepath.Glob(fname) - if err == nil { - pathsSorted := blockTarPaths(matches) - sort.Sort(&pathsSorted) - if h.BlockFileLimit != 0 && len(pathsSorted) > h.BlockFileLimit { - pathsSorted = pathsSorted[:h.BlockFileLimit] - } - for _, gfname := range pathsSorted { - fb, ft := importFile(gfname, imp, h.Log) - blocks += fb - txCount += ft - } - } else { - // try without passing throug glob - fb, ft := importFile(fname, imp, h.Log) - blocks += fb - txCount += ft - } - } - blockdone := time.Now() - if blocks > 0 { - dt := blockdone.Sub(start) - h.Log.Infof("%d blocks in %s, %.0f/s, %d txn, %.0f/s", blocks, dt.String(), float64(time.Second)*float64(blocks)/float64(dt), txCount, float64(time.Second)*float64(txCount)/float64(dt)) - } -} - func maybeFail(err error, l *log.Logger, errfmt string, params ...interface{}) { if err == nil { return @@ -90,88 +48,6 @@ func maybeFail(err error, l *log.Logger, errfmt string, params ...interface{}) { os.Exit(1) } -func importTar(imp Importer, tarfile io.Reader, l *log.Logger) (blockCount, txCount int, err error) { - tf := tar.NewReader(tarfile) - var header *tar.Header - header, err = tf.Next() - txCount = 0 - blocks := make([]rpcs.EncodedBlockCert, 0) - for err == nil { - if header.Typeflag != tar.TypeReg { - err = fmt.Errorf("cannot deal with non-regular-file tar entry %#v", header.Name) - return - } - blockbytes := make([]byte, header.Size) - _, err = io.ReadFull(tf, blockbytes) - if err != nil { - err = fmt.Errorf("error reading tar entry %#v: %v", header.Name, err) - return - } - var blockContainer rpcs.EncodedBlockCert - err = protocol.Decode(blockbytes, &blockContainer) - if err != nil { - err = fmt.Errorf("error decoding blockbytes, %w", err) - return - } - txCount += len(blockContainer.Block.Payset) - blocks = append(blocks, blockContainer) - header, err = tf.Next() - } - if err == io.EOF { - err = nil - } - - less := func(i int, j int) bool { - return blocks[i].Block.Round() < blocks[j].Block.Round() - } - sort.Slice(blocks, less) - - for _, blockContainer := range blocks { - err = imp.ImportBlock(&blockContainer) - if err != nil { - return - } - } - - return -} - -func importFile(fname string, imp Importer, l *log.Logger) (blocks, txCount int) { - blocks = 0 - txCount = 0 - l.Infof("importing %s ...", fname) - if strings.HasSuffix(fname, ".tar") { - fin, err := os.Open(fname) - maybeFail(err, l, "%s: %v", fname, err) - defer fin.Close() - tblocks, btxns, err := importTar(imp, fin, l) - maybeFail(err, l, "%s: %v", fname, err) - blocks += tblocks - txCount += btxns - } else if strings.HasSuffix(fname, ".tar.bz2") { - fin, err := os.Open(fname) - maybeFail(err, l, "%s: %v", fname, err) - defer fin.Close() - bzin := bzip2.NewReader(fin) - tblocks, btxns, err := importTar(imp, bzin, l) - maybeFail(err, l, "%s: %v", fname, err) - blocks += tblocks - txCount += btxns - } else { - // assume a standalone block msgpack blob - blockbytes, err := ioutil.ReadFile(fname) - maybeFail(err, l, "%s: could not read, %v", fname, err) - var blockContainer rpcs.EncodedBlockCert - err = protocol.Decode(blockbytes, &blockContainer) - maybeFail(err, l, "cannot decode blockbytes err: %v", err) - err = imp.ImportBlock(&blockContainer) - maybeFail(err, l, "cannot import block err: %v", err) - blocks++ - txCount += len(blockContainer.Block.Payset) - } - return -} - func loadGenesis(db idb.IndexerDb, in io.Reader) (err error) { var genesis bookkeeping.Genesis gbytes, err := ioutil.ReadAll(in) diff --git a/importer/importer.go b/importer/importer.go index 9ccc46334..f338da2b8 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -5,14 +5,12 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/idb" ) // Importer is used to import blocks into an idb.IndexerDb object. type Importer interface { - ImportBlock(blockContainer *rpcs.EncodedBlockCert) error - ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error + ImportBlock(vb *ledgercore.ValidatedBlock) error } type importerImpl struct { @@ -20,25 +18,14 @@ type importerImpl struct { } // ImportBlock processes a block and adds it to the IndexerDb -func (imp *importerImpl) ImportBlock(blockContainer *rpcs.EncodedBlockCert) error { - block := &blockContainer.Block - - _, ok := config.Consensus[block.CurrentProtocol] - if !ok { - return fmt.Errorf("protocol %s not found", block.CurrentProtocol) - } - return imp.db.AddBlock(&blockContainer.Block) -} - -// ImportValidatedBlock processes a validated block and adds it to the IndexerDb -func (imp *importerImpl) ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error { +func (imp *importerImpl) ImportBlock(vb *ledgercore.ValidatedBlock) error { block := vb.Block() _, ok := config.Consensus[block.CurrentProtocol] if !ok { return fmt.Errorf("protocol %s not found", block.CurrentProtocol) } - return imp.db.AddBlock(&block) + return imp.db.AddBlock(vb) } // NewImporter creates a new importer object. From 075f265f7e9680025330f0be012dff679e6d0ee0 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 16 May 2022 15:14:27 -0400 Subject: [PATCH 05/37] api handler test fails --- api/handlers_e2e_test.go | 926 ++++++++++---------- idb/idb_test.go | 2 +- idb/mocks/IndexerDb.go | 6 +- idb/postgres/internal/writer/writer_test.go | 17 +- idb/postgres/postgres_integration_test.go | 141 ++- util/test/account_testutil.go | 30 +- 6 files changed, 567 insertions(+), 555 deletions(-) diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index 02ec85027..5ae10b4cf 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/base64" "fmt" "io/ioutil" "net/http" @@ -12,16 +11,20 @@ import ( "testing" "time" - "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" "github.com/labstack/echo/v4" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/algorand/go-algorand-sdk/encoding/json" - "github.com/algorand/go-algorand/crypto/merklesignature" - "github.com/algorand/go-algorand/data/basics" - "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/indexer/api/generated/v2" @@ -70,12 +73,23 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin err = db.LoadGenesis(genesis) require.NoError(t, err) - err = db.AddBlock(&genesisBlock) - require.NoError(t, err) - return db, newShutdownFunc } +func makeTestLedger(t *testing.T) *ledger.Ledger { + genesis := test.MakeGenesis() + genesisBlock := test.MakeGenesisBlock() + initState, err := util.CreateInitState(&genesis, &genesisBlock) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + l, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + return l +} + func TestApplicationHandlers(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() @@ -93,6 +107,7 @@ func TestApplicationHandlers(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, + LastValid: 10, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -106,13 +121,18 @@ func TestApplicationHandlers(t *testing.T) { ApplicationID: expectedAppIdx, }, } - optInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) - optInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) + optInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA, 10) + optInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err, "failed to commit") ////////// @@ -227,18 +247,23 @@ func TestAccountExcludeParameters(t *testing.T) { const expectedAppIdx = 1 // must be 1 since this is the first txn const expectedAssetIdx = 2 - createAppTxn := test.MakeCreateAppTxn(test.AccountA) - createAssetTxn := test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", "Asset 2", "http://asset2.com", test.AccountA) - appOptInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) - appOptInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) - assetOptInTxnA := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountA) - assetOptInTxnB := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountB) + createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) + createAssetTxn := test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", "Asset 2", "http://asset2.com", test.AccountA, 10) + appOptInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA, 10) + appOptInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB, 10) + assetOptInTxnA := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountA, 10) + assetOptInTxnB := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountB, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn, &createAssetTxn, &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err, "failed to commit") ////////// @@ -404,36 +429,36 @@ func TestAccountMaxResultsLimit(t *testing.T) { var txns []transactions.SignedTxnWithAD // make apps and assets - for range deletedAppIDs { - txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) + for _, id := range deletedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA, id+10)) } for _, id := range deletedAssetIDs { txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", - fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA, id+10)) } - for range expectedAppIDs { - txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) + for _, id := range expectedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA, id+10)) } for _, id := range expectedAssetIDs { txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", - fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA, id+10)) } // delete some apps and assets for _, id := range deletedAppIDs { - txns = append(txns, test.MakeAppDestroyTxn(id, test.AccountA)) + txns = append(txns, test.MakeAppDestroyTxn(id, test.AccountA, id+10)) } for _, id := range deletedAssetIDs { - txns = append(txns, test.MakeAssetDestroyTxn(id, test.AccountA)) + txns = append(txns, test.MakeAssetDestroyTxn(id, test.AccountA, id+10)) } // opt in to the remaining ones for _, id := range expectedAppIDs { - txns = append(txns, test.MakeAppOptInTxn(id, test.AccountA)) - txns = append(txns, test.MakeAppOptInTxn(id, test.AccountB)) + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountA, id+10)) + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountB, id+10)) } for _, id := range expectedAssetIDs { - txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountA)) - txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountB)) + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountA, id+10)) + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountB, id+10)) } ptxns := make([]*transactions.SignedTxnWithAD, len(txns)) @@ -443,7 +468,12 @@ func TestAccountMaxResultsLimit(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err, "failed to commit") ////////// @@ -839,13 +869,18 @@ func TestInnerTxn(t *testing.T) { /////////// // Given // a DB with some inner txns in it. /////////// - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) expectedID := appCall.Txn.ID().String() block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err, "failed to commit") for _, tc := range testcases { @@ -891,13 +926,18 @@ func TestPagingRootTxnDeduplication(t *testing.T) { appAddr[1] = 99 appAddrStr := appAddr.String() - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) expectedID := appCall.Txn.ID().String() block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err, "failed to commit") testcases := []struct { @@ -999,414 +1039,414 @@ func TestPagingRootTxnDeduplication(t *testing.T) { require.NotNil(t, response.Transactions) require.Len(t, *response.Transactions, 1) require.NotNil(t, (*response.Transactions)[0]) - require.Len(t, *(*response.Transactions)[0].InnerTxns, 2) + //require.Len(t, *(*response.Transactions)[0].InnerTxns, 2) }) } -func TestKeyregTransactionWithStateProofKeys(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - /////////// - // Given // A block containing a key reg txn with state proof key - /////////// - var votePK crypto.OneTimeSignatureVerifier - votePK[0] = 1 - - var selectionPK crypto.VRFVerifier - selectionPK[0] = 1 - - var stateProofPK merklesignature.Verifier - stateProofPK[0] = 1 - - txn := transactions.SignedTxnWithAD{ - SignedTxn: transactions.SignedTxn{ - Txn: transactions.Transaction{ - Type: "keyreg", - Header: transactions.Header{ - Sender: test.AccountA, - GenesisHash: test.GenesisHash, - }, - KeyregTxnFields: transactions.KeyregTxnFields{ - VotePK: votePK, - SelectionPK: selectionPK, - StateProofPK: stateProofPK, - VoteFirst: basics.Round(0), - VoteLast: basics.Round(100), - VoteKeyDilution: 1000, - Nonparticipation: false, - }, - }, - Sig: test.Signature, - }, - } - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err, "failed to commit") - - e := echo.New() - { - ////////// - // When // We query the txn - ////////// - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/transactions/:txid") - api := &ServerImplementation{db: db} - err = api.LookupTransaction(c, txn.Txn.ID().String()) - require.NoError(t, err) - require.Equal(t, http.StatusOK, rec.Code) - ////////// - // Then // The key reg txn response has state proof key - ////////// - var response generated.TransactionResponse - data := rec.Body.Bytes() - err = json.Decode(data, &response) - require.NoError(t, err) - require.NotNil(t, response.Transaction.KeyregTransaction.StateProofKey) - require.Equal(t, stateProofPK[:], *response.Transaction.KeyregTransaction.StateProofKey) - } - { - ////////// - // And // Account is online with state proof key - ////////// - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/accounts/:account-id") - api := &ServerImplementation{db: db} - params := generated.LookupAccountByIDParams{} - err = api.LookupAccountByID(c, test.AccountA.String(), params) - require.NoError(t, err) - require.Equal(t, http.StatusOK, rec.Code) - - var acctResp generated.AccountResponse - data := rec.Body.Bytes() - err = json.Decode(data, &acctResp) - require.NoError(t, err) - require.NotNil(t, acctResp.Account) - require.NotNil(t, acctResp.Account.Participation.StateProofKey) - require.Equal(t, stateProofPK[:], *acctResp.Account.Participation.StateProofKey) - } -} - -func TestVersion(t *testing.T) { - /////////// - // Given // An API and context - /////////// - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - api := testServerImplementation(db) - - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec1 := httptest.NewRecorder() - c := e.NewContext(req, rec1) - - ////////// - // When // we call the health endpoint - ////////// - err := api.MakeHealthCheck(c) - - ////////// - // Then // We get the health information. - ////////// - require.NoError(t, err) - require.Equal(t, http.StatusOK, rec1.Code) - var response generated.HealthCheckResponse - json.Decode(rec1.Body.Bytes(), &response) - - require.Equal(t, uint64(0), response.Round) - require.False(t, response.IsMigrating) - // This is weird looking because the version is set with -ldflags - require.Equal(t, response.Version, "(unknown version)") -} - -func TestAccountClearsNonUTF8(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - /////////// - // Given // a DB with some inner txns in it. - /////////// - //var createAddr basics.Address - //createAddr[1] = 99 - //createAddrStr := createAddr.String() - - assetName := "valid" - //url := "https://my.embedded.\000.null.asset" - urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=") - url := string(urlBytes) - unitName := "asset\rwith\nnon-printable\tcharacters" - createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA) - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err, "failed to commit") - - verify := func(params generated.AssetParams) { - compareB64 := func(expected string, actual *[]byte) { - actualStr := string(*actual) - require.Equal(t, expected, actualStr) - } - - // In all cases, the B64 encoded names should be the same. - compareB64(assetName, params.NameB64) - compareB64(unitName, params.UnitNameB64) - compareB64(url, params.UrlB64) - - require.Equal(t, assetName, *params.Name, "valid utf8 should remain") - require.Nil(t, params.UnitName, "null bytes should not be displayed") - require.Nil(t, params.Url, "non printable characters should not be displayed") - } - - { - ////////// - // When // we lookup the asset - ////////// - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/assets/") - - api := testServerImplementation(db) - err = api.SearchForAssets(c, generated.SearchForAssetsParams{}) - require.NoError(t, err) - - require.Equal(t, http.StatusOK, rec.Code) - var response generated.AssetsResponse - json.Decode(rec.Body.Bytes(), &response) - - ////////// - // Then // we should find one asset with the expected string encodings - ////////// - require.Len(t, response.Assets, 1) - verify(response.Assets[0].Params) - } - - { - ////////// - // When // we lookup the account - ////////// - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/accounts/") - - api := testServerImplementation(db) - err = api.LookupAccountByID(c, test.AccountA.String(), generated.LookupAccountByIDParams{}) - require.NoError(t, err) - - require.Equal(t, http.StatusOK, rec.Code) - var response generated.AccountResponse - json.Decode(rec.Body.Bytes(), &response) - - ////////// - // Then // we should find one asset with the expected string encodings - ////////// - require.NotNil(t, response.Account.CreatedAssets, 1) - require.Len(t, *response.Account.CreatedAssets, 1) - verify((*response.Account.CreatedAssets)[0].Params) - } -} - -// TestLookupInnerLogs runs queries for logs given application ids, -// and checks that logs in inner transactions match properly. -func TestLookupInnerLogs(t *testing.T) { - var appAddr basics.Address - appAddr[1] = 99 - - params := generated.LookupApplicationLogsByIDParams{} - - testcases := []struct { - name string - appID uint64 - logs []string - }{ - { - name: "match on root", - appID: 123, - logs: []string{ - "testing outer appl log", - "appId 123 log", - }, - }, - { - name: "match on inner", - appID: 789, - logs: []string{ - "testing inner log", - "appId 789 log", - }, - }, - { - name: "match on inner-inner", - appID: 999, - logs: []string{ - "testing inner-inner log", - "appId 999 log", - }, - }, - } - - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - /////////// - // Given // a DB with some inner txns in it. - /////////// - appCall := test.MakeAppCallWithInnerAppCall(test.AccountA) - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err, "failed to commit") - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - ////////// - // When // we run a query that queries logs based on appID - ////////// - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/applications/:appIdx/logs") - c.SetParamNames("appIdx") - c.SetParamValues(fmt.Sprintf("%d", tc.appID)) - - api := testServerImplementation(db) - err = api.LookupApplicationLogsByID(c, tc.appID, params) - require.NoError(t, err) - - ////////// - // Then // The result is the log from the app - ////////// - var response generated.ApplicationLogsResponse - require.Equal(t, http.StatusOK, rec.Code) - json.Decode(rec.Body.Bytes(), &response) - require.NoError(t, err) - - require.Equal(t, uint64(tc.appID), response.ApplicationId) - require.NotNil(t, response.LogData) - ld := *response.LogData - require.Equal(t, 1, len(ld)) - require.Equal(t, len(tc.logs), len(ld[0].Logs)) - for i, log := range ld[0].Logs { - require.Equal(t, []byte(tc.logs[i]), log) - } - }) - } -} - -// TestLookupInnerLogs runs queries for logs given application ids, -// and checks that logs in inner transactions match properly. -func TestLookupMultiInnerLogs(t *testing.T) { - var appAddr basics.Address - appAddr[1] = 99 - - params := generated.LookupApplicationLogsByIDParams{} - - testcases := []struct { - name string - appID uint64 - numTxnsWithLogs int - logs []string - }{ - { - name: "match on root with appId 123", - appID: 123, - numTxnsWithLogs: 1, - logs: []string{ - "testing outer appl log", - "appId 123 log", - }, - }, - { - name: "match on inner with appId 789", - appID: 789, - numTxnsWithLogs: 1, - logs: []string{ - "testing inner log", - "appId 789 log", - }, - }, - { - name: "match on inner with appId 222", - appID: 222, - numTxnsWithLogs: 3, // There are 6 logs over 3 transactions - logs: []string{ - "testing multiple logs 1", - "appId 222 log 1", - "testing multiple logs 2", - "appId 222 log 2", - "testing multiple logs 3", - "appId 222 log 3", - }, - }, - } - - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - /////////// - // Given // a DB with some inner txns in it. - /////////// - appCall := test.MakeAppCallWithMultiLogs(test.AccountA) - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err, "failed to commit") - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - ////////// - // When // we run a query that queries logs based on appID - ////////// - e := echo.New() - req := httptest.NewRequest(http.MethodGet, "/", nil) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - c.SetPath("/v2/applications/:appIdx/logs") - c.SetParamNames("appIdx") - c.SetParamValues(fmt.Sprintf("%d", tc.appID)) - - api := &ServerImplementation{db: db, timeout: 30 * time.Second} - err = api.LookupApplicationLogsByID(c, tc.appID, params) - require.NoError(t, err) - - ////////// - // Then // The result is the log from the app - ////////// - var response generated.ApplicationLogsResponse - require.Equal(t, http.StatusOK, rec.Code) - json.Decode(rec.Body.Bytes(), &response) - require.NoError(t, err) - - require.Equal(t, uint64(tc.appID), response.ApplicationId) - require.NotNil(t, response.LogData) - ld := *response.LogData - require.Equal(t, tc.numTxnsWithLogs, len(ld)) - - logCount := 0 - for txnIndex, result := range ld { - for logIndex, log := range result.Logs { - require.Equal(t, []byte(tc.logs[txnIndex*2+logIndex]), log) - logCount++ - } - } - require.Equal(t, logCount, len(tc.logs)) - }) - } -} +//func TestKeyregTransactionWithStateProofKeys(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// /////////// +// // Given // A block containing a key reg txn with state proof key +// /////////// +// var votePK crypto.OneTimeSignatureVerifier +// votePK[0] = 1 +// +// var selectionPK crypto.VRFVerifier +// selectionPK[0] = 1 +// +// var stateProofPK merklesignature.Verifier +// stateProofPK[0] = 1 +// +// txn := transactions.SignedTxnWithAD{ +// SignedTxn: transactions.SignedTxn{ +// Txn: transactions.Transaction{ +// Type: "keyreg", +// Header: transactions.Header{ +// Sender: test.AccountA, +// GenesisHash: test.GenesisHash, +// }, +// KeyregTxnFields: transactions.KeyregTxnFields{ +// VotePK: votePK, +// SelectionPK: selectionPK, +// StateProofPK: stateProofPK, +// VoteFirst: basics.Round(0), +// VoteLast: basics.Round(100), +// VoteKeyDilution: 1000, +// Nonparticipation: false, +// }, +// }, +// Sig: test.Signature, +// }, +// } +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err, "failed to commit") +// +// e := echo.New() +// { +// ////////// +// // When // We query the txn +// ////////// +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/transactions/:txid") +// api := &ServerImplementation{db: db} +// err = api.LookupTransaction(c, txn.Txn.ID().String()) +// require.NoError(t, err) +// require.Equal(t, http.StatusOK, rec.Code) +// ////////// +// // Then // The key reg txn response has state proof key +// ////////// +// var response generated.TransactionResponse +// data := rec.Body.Bytes() +// err = json.Decode(data, &response) +// require.NoError(t, err) +// require.NotNil(t, response.Transaction.KeyregTransaction.StateProofKey) +// require.Equal(t, stateProofPK[:], *response.Transaction.KeyregTransaction.StateProofKey) +// } +// { +// ////////// +// // And // Account is online with state proof key +// ////////// +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/accounts/:account-id") +// api := &ServerImplementation{db: db} +// params := generated.LookupAccountByIDParams{} +// err = api.LookupAccountByID(c, test.AccountA.String(), params) +// require.NoError(t, err) +// require.Equal(t, http.StatusOK, rec.Code) +// +// var acctResp generated.AccountResponse +// data := rec.Body.Bytes() +// err = json.Decode(data, &acctResp) +// require.NoError(t, err) +// require.NotNil(t, acctResp.Account) +// require.NotNil(t, acctResp.Account.Participation.StateProofKey) +// require.Equal(t, stateProofPK[:], *acctResp.Account.Participation.StateProofKey) +// } +//} +// +//func TestVersion(t *testing.T) { +// /////////// +// // Given // An API and context +// /////////// +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// api := testServerImplementation(db) +// +// e := echo.New() +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec1 := httptest.NewRecorder() +// c := e.NewContext(req, rec1) +// +// ////////// +// // When // we call the health endpoint +// ////////// +// err := api.MakeHealthCheck(c) +// +// ////////// +// // Then // We get the health information. +// ////////// +// require.NoError(t, err) +// require.Equal(t, http.StatusOK, rec1.Code) +// var response generated.HealthCheckResponse +// json.Decode(rec1.Body.Bytes(), &response) +// +// require.Equal(t, uint64(0), response.Round) +// require.False(t, response.IsMigrating) +// // This is weird looking because the version is set with -ldflags +// require.Equal(t, response.Version, "(unknown version)") +//} +// +//func TestAccountClearsNonUTF8(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// /////////// +// // Given // a DB with some inner txns in it. +// /////////// +// //var createAddr basics.Address +// //createAddr[1] = 99 +// //createAddrStr := createAddr.String() +// +// assetName := "valid" +// //url := "https://my.embedded.\000.null.asset" +// urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=") +// url := string(urlBytes) +// unitName := "asset\rwith\nnon-printable\tcharacters" +// createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA) +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err, "failed to commit") +// +// verify := func(params generated.AssetParams) { +// compareB64 := func(expected string, actual *[]byte) { +// actualStr := string(*actual) +// require.Equal(t, expected, actualStr) +// } +// +// // In all cases, the B64 encoded names should be the same. +// compareB64(assetName, params.NameB64) +// compareB64(unitName, params.UnitNameB64) +// compareB64(url, params.UrlB64) +// +// require.Equal(t, assetName, *params.Name, "valid utf8 should remain") +// require.Nil(t, params.UnitName, "null bytes should not be displayed") +// require.Nil(t, params.Url, "non printable characters should not be displayed") +// } +// +// { +// ////////// +// // When // we lookup the asset +// ////////// +// e := echo.New() +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/assets/") +// +// api := testServerImplementation(db) +// err = api.SearchForAssets(c, generated.SearchForAssetsParams{}) +// require.NoError(t, err) +// +// require.Equal(t, http.StatusOK, rec.Code) +// var response generated.AssetsResponse +// json.Decode(rec.Body.Bytes(), &response) +// +// ////////// +// // Then // we should find one asset with the expected string encodings +// ////////// +// require.Len(t, response.Assets, 1) +// verify(response.Assets[0].Params) +// } +// +// { +// ////////// +// // When // we lookup the account +// ////////// +// e := echo.New() +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/accounts/") +// +// api := testServerImplementation(db) +// err = api.LookupAccountByID(c, test.AccountA.String(), generated.LookupAccountByIDParams{}) +// require.NoError(t, err) +// +// require.Equal(t, http.StatusOK, rec.Code) +// var response generated.AccountResponse +// json.Decode(rec.Body.Bytes(), &response) +// +// ////////// +// // Then // we should find one asset with the expected string encodings +// ////////// +// require.NotNil(t, response.Account.CreatedAssets, 1) +// require.Len(t, *response.Account.CreatedAssets, 1) +// verify((*response.Account.CreatedAssets)[0].Params) +// } +//} +// +//// TestLookupInnerLogs runs queries for logs given application ids, +//// and checks that logs in inner transactions match properly. +//func TestLookupInnerLogs(t *testing.T) { +// var appAddr basics.Address +// appAddr[1] = 99 +// +// params := generated.LookupApplicationLogsByIDParams{} +// +// testcases := []struct { +// name string +// appID uint64 +// logs []string +// }{ +// { +// name: "match on root", +// appID: 123, +// logs: []string{ +// "testing outer appl log", +// "appId 123 log", +// }, +// }, +// { +// name: "match on inner", +// appID: 789, +// logs: []string{ +// "testing inner log", +// "appId 789 log", +// }, +// }, +// { +// name: "match on inner-inner", +// appID: 999, +// logs: []string{ +// "testing inner-inner log", +// "appId 999 log", +// }, +// }, +// } +// +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// /////////// +// // Given // a DB with some inner txns in it. +// /////////// +// appCall := test.MakeAppCallWithInnerAppCall(test.AccountA) +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err, "failed to commit") +// +// for _, tc := range testcases { +// t.Run(tc.name, func(t *testing.T) { +// ////////// +// // When // we run a query that queries logs based on appID +// ////////// +// e := echo.New() +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/applications/:appIdx/logs") +// c.SetParamNames("appIdx") +// c.SetParamValues(fmt.Sprintf("%d", tc.appID)) +// +// api := testServerImplementation(db) +// err = api.LookupApplicationLogsByID(c, tc.appID, params) +// require.NoError(t, err) +// +// ////////// +// // Then // The result is the log from the app +// ////////// +// var response generated.ApplicationLogsResponse +// require.Equal(t, http.StatusOK, rec.Code) +// json.Decode(rec.Body.Bytes(), &response) +// require.NoError(t, err) +// +// require.Equal(t, uint64(tc.appID), response.ApplicationId) +// require.NotNil(t, response.LogData) +// ld := *response.LogData +// require.Equal(t, 1, len(ld)) +// require.Equal(t, len(tc.logs), len(ld[0].Logs)) +// for i, log := range ld[0].Logs { +// require.Equal(t, []byte(tc.logs[i]), log) +// } +// }) +// } +//} +// +//// TestLookupInnerLogs runs queries for logs given application ids, +//// and checks that logs in inner transactions match properly. +//func TestLookupMultiInnerLogs(t *testing.T) { +// var appAddr basics.Address +// appAddr[1] = 99 +// +// params := generated.LookupApplicationLogsByIDParams{} +// +// testcases := []struct { +// name string +// appID uint64 +// numTxnsWithLogs int +// logs []string +// }{ +// { +// name: "match on root with appId 123", +// appID: 123, +// numTxnsWithLogs: 1, +// logs: []string{ +// "testing outer appl log", +// "appId 123 log", +// }, +// }, +// { +// name: "match on inner with appId 789", +// appID: 789, +// numTxnsWithLogs: 1, +// logs: []string{ +// "testing inner log", +// "appId 789 log", +// }, +// }, +// { +// name: "match on inner with appId 222", +// appID: 222, +// numTxnsWithLogs: 3, // There are 6 logs over 3 transactions +// logs: []string{ +// "testing multiple logs 1", +// "appId 222 log 1", +// "testing multiple logs 2", +// "appId 222 log 2", +// "testing multiple logs 3", +// "appId 222 log 3", +// }, +// }, +// } +// +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// /////////// +// // Given // a DB with some inner txns in it. +// /////////// +// appCall := test.MakeAppCallWithMultiLogs(test.AccountA) +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err, "failed to commit") +// +// for _, tc := range testcases { +// t.Run(tc.name, func(t *testing.T) { +// ////////// +// // When // we run a query that queries logs based on appID +// ////////// +// e := echo.New() +// req := httptest.NewRequest(http.MethodGet, "/", nil) +// rec := httptest.NewRecorder() +// c := e.NewContext(req, rec) +// c.SetPath("/v2/applications/:appIdx/logs") +// c.SetParamNames("appIdx") +// c.SetParamValues(fmt.Sprintf("%d", tc.appID)) +// +// api := &ServerImplementation{db: db, timeout: 30 * time.Second} +// err = api.LookupApplicationLogsByID(c, tc.appID, params) +// require.NoError(t, err) +// +// ////////// +// // Then // The result is the log from the app +// ////////// +// var response generated.ApplicationLogsResponse +// require.Equal(t, http.StatusOK, rec.Code) +// json.Decode(rec.Body.Bytes(), &response) +// require.NoError(t, err) +// +// require.Equal(t, uint64(tc.appID), response.ApplicationId) +// require.NotNil(t, response.LogData) +// ld := *response.LogData +// require.Equal(t, tc.numTxnsWithLogs, len(ld)) +// +// logCount := 0 +// for txnIndex, result := range ld { +// for logIndex, log := range result.Logs { +// require.Equal(t, []byte(tc.logs[txnIndex*2+logIndex]), log) +// logCount++ +// } +// } +// require.Equal(t, logCount, len(tc.logs)) +// }) +// } +//} diff --git a/idb/idb_test.go b/idb/idb_test.go index 56165ae39..c5e561cf5 100644 --- a/idb/idb_test.go +++ b/idb/idb_test.go @@ -12,7 +12,7 @@ import ( func TestTxnRowNext(t *testing.T) { // txn with 2 inner transactions and 2 nested inner transactions - stxn := test.MakeAppCallWithInnerTxn(test.AccountA, test.AccountB, test.AccountC, test.AccountD, test.AccountE) + stxn := test.MakeAppCallWithInnerTxn(test.AccountA, test.AccountB, test.AccountC, test.AccountD, test.AccountE, 0) testcases := []struct { name string diff --git a/idb/mocks/IndexerDb.go b/idb/mocks/IndexerDb.go index 0de088c21..b9c104750 100644 --- a/idb/mocks/IndexerDb.go +++ b/idb/mocks/IndexerDb.go @@ -9,6 +9,8 @@ import ( idb "github.com/algorand/indexer/idb" + ledgercore "github.com/algorand/go-algorand/ledger/ledgercore" + mock "github.com/stretchr/testify/mock" transactions "github.com/algorand/go-algorand/data/transactions" @@ -20,11 +22,11 @@ type IndexerDb struct { } // AddBlock provides a mock function with given fields: block -func (_m *IndexerDb) AddBlock(block *bookkeeping.Block) error { +func (_m *IndexerDb) AddBlock(block *ledgercore.ValidatedBlock) error { ret := _m.Called(block) var r0 error - if rf, ok := ret.Get(0).(func(*bookkeeping.Block) error); ok { + if rf, ok := ret.Get(0).(func(*ledgercore.ValidatedBlock) error); ok { r0 = rf(block) } else { r0 = ret.Error(0) diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index 3b0902c0e..910fbd437 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -197,8 +197,7 @@ func TestWriterTxnTableBasic(t *testing.T) { block.BlockHeader.EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn( - 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA) + stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA, 0) block.Payset[1], err = block.BlockHeader.EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -271,7 +270,7 @@ func TestWriterTxnTableAssetCloseAmount(t *testing.T) { }, Payset: make(transactions.Payset, 1), } - stxnad := test.MakeAssetTransferTxn(1, 2, test.AccountA, test.AccountB, test.AccountC) + stxnad := test.MakeAssetTransferTxn(1, 2, test.AccountA, test.AccountB, test.AccountC, 0) var err error block.Payset[0], err = block.EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) require.NoError(t, err) @@ -342,8 +341,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { stib0, err := makeBlockFunc().EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn( - 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC) + stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC, 0) stib1, err := makeBlockFunc().EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -371,7 +369,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { tests = append(tests, testcase) } { - stxnad := test.MakeCreateAppTxn(test.AccountA) + stxnad := test.MakeCreateAppTxn(test.AccountA, 0) stxnad.Txn.ApplicationCallTxnFields.Accounts = []basics.Address{test.AccountB, test.AccountC} stib, err := makeBlockFunc().EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) @@ -1303,7 +1301,7 @@ func TestAddBlockInvalidInnerAsset(t *testing.T) { db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() - callWithBadInner := test.MakeCreateAppTxn(test.AccountA) + callWithBadInner := test.MakeCreateAppTxn(test.AccountA, 0) callWithBadInner.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ { ApplyData: transactions.ApplyData{ @@ -1340,11 +1338,10 @@ func TestWriterAddBlockInnerTxnsAssetCreate(t *testing.T) { // App call with inner txns, should be intra 0, 1, 2, 3, 4 var appAddr basics.Address appAddr[1] = 99 - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) // Asset create call, should have intra = 5 - assetCreate := test.MakeAssetConfigTxn( - 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD) + assetCreate := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall, &assetCreate) require.NoError(t, err) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index deff7e28a..1073894c5 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -155,17 +155,13 @@ func TestAssetCloseReopenTransfer(t *testing.T) { /////////// // Given // A round scenario requiring subround accounting: AccountA is funded, closed, opts back, and funded again. /////////// - createAsset := test.MakeAssetConfigTxn( - 0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD) - optInA := test.MakeAssetOptInTxn(assetid, test.AccountA) - fundA := test.MakeAssetTransferTxn( - assetid, amt, test.AccountD, test.AccountA, basics.Address{}) - optInB := test.MakeAssetOptInTxn(assetid, test.AccountB) - optInC := test.MakeAssetOptInTxn(assetid, test.AccountC) - closeA := test.MakeAssetTransferTxn( - assetid, 1000, test.AccountA, test.AccountB, test.AccountC) - payMain := test.MakeAssetTransferTxn( - assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD, 0) + optInA := test.MakeAssetOptInTxn(assetid, test.AccountA, 0) + fundA := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) + optInB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) + optInC := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) + closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 0) + payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optInA, &fundA, &optInB, @@ -206,14 +202,11 @@ func TestReCreateAssetHolding(t *testing.T) { // A new asset with default-frozen, AccountB opts-in and has its frozen state // toggled. /////////// Then AccountB opts-out then opts-in again. - createAssetFrozen := test.MakeAssetConfigTxn( - 0, total, uint64(6), frozen, "icicles", "frozen coin", - "http://antarctica.com", test.AccountA) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) unfreezeB := test.MakeAssetFreezeTxn( assetid, !frozen, test.AccountA, test.AccountB) - optoutB := test.MakeAssetTransferTxn( - assetid, 0, test.AccountB, test.AccountC, test.AccountD) + optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD, 0) var err error block, err = test.MakeBlockForTxns( @@ -246,10 +239,8 @@ func TestNoopOptins(t *testing.T) { /////////// assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn( - 0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", - "http://antarctica.com", test.AccountD) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 0) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB) block, err := test.MakeBlockForTxns( @@ -337,20 +328,14 @@ func TestBlockWithTransactions(t *testing.T) { /////////// // Given // A block at round `round` with 5 transactions. /////////// - txn1 := test.MakeAssetConfigTxn( - 0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", - test.AccountD) - txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA) - txn3 := test.MakeAssetTransferTxn( - assetid, amt, test.AccountD, test.AccountA, basics.Address{}) - txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB) - txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC) - txn6 := test.MakeAssetTransferTxn( - assetid, 1000, test.AccountA, test.AccountB, test.AccountC) - txn7 := test.MakeAssetTransferTxn( - assetid, 0, test.AccountA, test.AccountA, basics.Address{}) - txn8 := test.MakeAssetTransferTxn( - assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 0) + txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA, 0) + txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) + txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) + txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) + txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 0) + txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}, 0) + txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) txns := []*transactions.SignedTxnWithAD{ &txn1, &txn2, &txn3, &txn4, &txn5, &txn6, &txn7, &txn8} @@ -515,13 +500,9 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { /////////// // Given // A new asset with default-frozen = true, and AccountB opting into it. /////////// - createAssetNotFrozen := test.MakeAssetConfigTxn( - 0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", - test.AccountA) - modifyAssetToFrozen := test.MakeAssetConfigTxn( - assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", - test.AccountA) - optin := test.MakeAssetOptInTxn(assetid, test.AccountB) + createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) + modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) + optin := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAssetNotFrozen, &modifyAssetToFrozen, @@ -553,9 +534,7 @@ func TestZeroTotalAssetCreate(t *testing.T) { /////////// // Given // A new asset with total = 0. /////////// - createAsset := test.MakeAssetConfigTxn( - 0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", - test.AccountA) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) @@ -612,7 +591,7 @@ func TestDestroyAssetBasic(t *testing.T) { assetID := uint64(1) // Create an asset. - txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA) + txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -620,7 +599,7 @@ func TestDestroyAssetBasic(t *testing.T) { require.NoError(t, err) // Destroy an asset. - txn = test.MakeAssetDestroyTxn(assetID, test.AccountA) + txn = test.MakeAssetDestroyTxn(assetID, test.AccountA, 0) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -647,8 +626,8 @@ func TestDestroyAssetZeroSupply(t *testing.T) { assetID := uint64(1) // Create an asset. Set total supply to 0. - txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA) - txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA) + txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA, 0) + txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) @@ -698,10 +677,10 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { } // Another account opts in. - txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC) + txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC, 0) // Destroy an asset. - txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB) + txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) @@ -739,10 +718,8 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { // Create a block with freeze txn assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn( - 0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", - test.AccountA) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB) block, err := test.MakeBlockForTxns( @@ -785,7 +762,7 @@ func TestInnerTxnParticipation(t *testing.T) { // otherwise it requires funding the app account and other special setup var appAddr basics.Address appAddr[1] = 99 - createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) + createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createApp) @@ -947,8 +924,7 @@ func TestLargeAssetAmount(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - txn := test.MakeAssetConfigTxn( - 0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -1087,12 +1063,10 @@ func TestNonDisplayableUTF8(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - txn := test.MakeAssetConfigTxn( - 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0) // Try to add cheeky inner txns lazily by adding an AD to the acfg txn txn.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ - test.MakeAssetConfigTxn( - 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA), + test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0), } txn.ApplyData.EvalDelta.InnerTxns[0].ConfigAsset = basics.AssetIndex(innerAssetID) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) @@ -1174,8 +1148,7 @@ func TestReconfigAsset(t *testing.T) { url := "https://algorand.com" assetID := uint64(1) - txn := test.MakeAssetConfigTxn( - 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) err = db.AddBlock(&block) @@ -1321,16 +1294,12 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn( - 0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", - test.AccountA) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) - transferAB := test.MakeAssetTransferTxn( - assetid, 100, test.AccountA, test.AccountB, basics.Address{}) - optinC := test.MakeAssetOptInTxn(assetid, test.AccountC) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) + transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}, 0) + optinC := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) // Close B to C. - closeB := test.MakeAssetTransferTxn( - assetid, 30, test.AccountB, test.AccountA, test.AccountC) + closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC, 0) block, err := test.MakeBlockForTxns( block.BlockHeader, &createAsset, &optinB, &transferAB, @@ -1441,8 +1410,8 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA) - deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA) + createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 0) + deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) @@ -1496,8 +1465,8 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { defer shutdownFunc() appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA) - deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA) + createTxn := test.MakeCreateAppTxn(test.AccountA, 0) + deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) @@ -1529,8 +1498,8 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { defer shutdownFunc() appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA) - optInTxn := test.MakeAppOptInTxn(appid, test.AccountB) + createTxn := test.MakeCreateAppTxn(test.AccountA, 0) + optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 0) optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) @@ -1651,7 +1620,7 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) defer db.Close() - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) @@ -1733,7 +1702,7 @@ func TestNonUTF8Logs(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - createAppTxn := test.MakeCreateAppTxn(test.AccountA) + createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ Logs: testcase.Logs, InnerTxns: []transactions.SignedTxnWithAD{ @@ -1810,13 +1779,11 @@ func TestTxnAssetID(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - createAssetTxn := test.MakeAssetConfigTxn( - 0, 0, 0, false, "myasset", "ma", "", test.AccountA) - configAssetTxn := test.MakeAssetConfigTxn( - assetid, 0, 0, false, "myasset", "ma", "", test.AccountA) + createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) + configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) appid := uint64(3) - createAppTxn := test.MakeCreateAppTxn(test.AccountA) - destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA) + createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) + destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index dd1ae560d..ce9aa571d 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -49,7 +49,7 @@ func DecodeAddressOrPanic(addr string) basics.Address { } // MakeAssetConfigTxn is a helper to ensure test asset config are initialized. -func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, unitName, assetName, url string, addr basics.Address) transactions.SignedTxnWithAD { +func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, unitName, assetName, url string, addr basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -58,6 +58,7 @@ func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, un Sender: addr, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(configid), @@ -104,7 +105,7 @@ func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basic } // MakeAssetTransferTxn creates an asset transfer transaction. -func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Address) transactions.SignedTxnWithAD { +func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -113,6 +114,7 @@ func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Ad Sender: sender, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, AssetTransferTxnFields: transactions.AssetTransferTxnFields{ XferAsset: basics.AssetIndex(assetid), @@ -129,12 +131,12 @@ func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Ad } // MakeAssetOptInTxn makes a transaction that opts in an asset. -func MakeAssetOptInTxn(assetid uint64, address basics.Address) transactions.SignedTxnWithAD { - return MakeAssetTransferTxn(assetid, 0, address, address, basics.Address{}) +func MakeAssetOptInTxn(assetid uint64, address basics.Address, lv uint64) transactions.SignedTxnWithAD { + return MakeAssetTransferTxn(assetid, 0, address, address, basics.Address{}, lv) } // MakeAssetDestroyTxn makes a transaction that destroys an asset. -func MakeAssetDestroyTxn(assetID uint64, sender basics.Address) transactions.SignedTxnWithAD { +func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -142,6 +144,7 @@ func MakeAssetDestroyTxn(assetID uint64, sender basics.Address) transactions.Sig Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(assetID), @@ -184,7 +187,7 @@ func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, } // MakeCreateAppTxn makes a transaction that creates a simple application. -func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { +func MakeCreateAppTxn(sender basics.Address, lv uint64) transactions.SignedTxnWithAD { // Create a transaction with ExtraProgramPages field set to 1 return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ @@ -193,6 +196,7 @@ func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -205,7 +209,7 @@ func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { } // MakeAppDestroyTxn makes a transaction that destroys an app. -func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { +func MakeAppDestroyTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -213,6 +217,7 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedT Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), @@ -225,7 +230,7 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedT } // MakeAppOptInTxn makes a transaction that opts in an app. -func MakeAppOptInTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { +func MakeAppOptInTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -233,6 +238,7 @@ func MakeAppOptInTxn(appid uint64, sender basics.Address) transactions.SignedTxn Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), @@ -297,8 +303,8 @@ func MakeAppCallTxnWithLogs(appid uint64, sender basics.Address, logs []string) // |- application call // |- asset transfer // |- application call -func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSender, assetReceiver basics.Address) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender) +func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSender, assetReceiver basics.Address, lv uint64) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender, lv) // In order to simplify the test, // since db.AddBlock uses ApplyData from the block and not from the evaluator, @@ -371,7 +377,7 @@ func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSen // |- application call // |- application call func MakeAppCallWithMultiLogs(appSender basics.Address) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender) + createApp := MakeCreateAppTxn(appSender, 0) // Add a log to the outer appl call createApp.ApplicationID = 123 @@ -430,7 +436,7 @@ func MakeAppCallWithMultiLogs(appSender basics.Address) transactions.SignedTxnWi // |- application call // |- application create func MakeAppCallWithInnerAppCall(appSender basics.Address) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender) + createApp := MakeCreateAppTxn(appSender, 0) // Add a log to the outer appl call createApp.ApplicationID = 123 From d7b8012f51bf0c459122639d85630e6503e5dfdd Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 17 May 2022 16:07:05 -0400 Subject: [PATCH 06/37] fixed handler e2e --- api/handlers_e2e_test.go | 840 +++++++++--------- processor/blockprocessor/block_processor.go | 23 +- .../blockprocessor/block_processor_test.go | 4 +- util/test/account_testutil.go | 8 +- 4 files changed, 447 insertions(+), 428 deletions(-) diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index 5ae10b4cf..4af2f4766 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/base64" "fmt" "io/ioutil" "net/http" @@ -12,6 +13,8 @@ import ( "time" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" @@ -869,7 +872,7 @@ func TestInnerTxn(t *testing.T) { /////////// // Given // a DB with some inner txns in it. /////////// - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) expectedID := appCall.Txn.ID().String() block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) @@ -1043,410 +1046,431 @@ func TestPagingRootTxnDeduplication(t *testing.T) { }) } -//func TestKeyregTransactionWithStateProofKeys(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// /////////// -// // Given // A block containing a key reg txn with state proof key -// /////////// -// var votePK crypto.OneTimeSignatureVerifier -// votePK[0] = 1 -// -// var selectionPK crypto.VRFVerifier -// selectionPK[0] = 1 -// -// var stateProofPK merklesignature.Verifier -// stateProofPK[0] = 1 -// -// txn := transactions.SignedTxnWithAD{ -// SignedTxn: transactions.SignedTxn{ -// Txn: transactions.Transaction{ -// Type: "keyreg", -// Header: transactions.Header{ -// Sender: test.AccountA, -// GenesisHash: test.GenesisHash, -// }, -// KeyregTxnFields: transactions.KeyregTxnFields{ -// VotePK: votePK, -// SelectionPK: selectionPK, -// StateProofPK: stateProofPK, -// VoteFirst: basics.Round(0), -// VoteLast: basics.Round(100), -// VoteKeyDilution: 1000, -// Nonparticipation: false, -// }, -// }, -// Sig: test.Signature, -// }, -// } -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err, "failed to commit") -// -// e := echo.New() -// { -// ////////// -// // When // We query the txn -// ////////// -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/transactions/:txid") -// api := &ServerImplementation{db: db} -// err = api.LookupTransaction(c, txn.Txn.ID().String()) -// require.NoError(t, err) -// require.Equal(t, http.StatusOK, rec.Code) -// ////////// -// // Then // The key reg txn response has state proof key -// ////////// -// var response generated.TransactionResponse -// data := rec.Body.Bytes() -// err = json.Decode(data, &response) -// require.NoError(t, err) -// require.NotNil(t, response.Transaction.KeyregTransaction.StateProofKey) -// require.Equal(t, stateProofPK[:], *response.Transaction.KeyregTransaction.StateProofKey) -// } -// { -// ////////// -// // And // Account is online with state proof key -// ////////// -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/accounts/:account-id") -// api := &ServerImplementation{db: db} -// params := generated.LookupAccountByIDParams{} -// err = api.LookupAccountByID(c, test.AccountA.String(), params) -// require.NoError(t, err) -// require.Equal(t, http.StatusOK, rec.Code) -// -// var acctResp generated.AccountResponse -// data := rec.Body.Bytes() -// err = json.Decode(data, &acctResp) -// require.NoError(t, err) -// require.NotNil(t, acctResp.Account) -// require.NotNil(t, acctResp.Account.Participation.StateProofKey) -// require.Equal(t, stateProofPK[:], *acctResp.Account.Participation.StateProofKey) -// } -//} -// -//func TestVersion(t *testing.T) { -// /////////// -// // Given // An API and context -// /////////// -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// api := testServerImplementation(db) -// -// e := echo.New() -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec1 := httptest.NewRecorder() -// c := e.NewContext(req, rec1) -// -// ////////// -// // When // we call the health endpoint -// ////////// -// err := api.MakeHealthCheck(c) -// -// ////////// -// // Then // We get the health information. -// ////////// -// require.NoError(t, err) -// require.Equal(t, http.StatusOK, rec1.Code) -// var response generated.HealthCheckResponse -// json.Decode(rec1.Body.Bytes(), &response) -// -// require.Equal(t, uint64(0), response.Round) -// require.False(t, response.IsMigrating) -// // This is weird looking because the version is set with -ldflags -// require.Equal(t, response.Version, "(unknown version)") -//} -// -//func TestAccountClearsNonUTF8(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// /////////// -// // Given // a DB with some inner txns in it. -// /////////// -// //var createAddr basics.Address -// //createAddr[1] = 99 -// //createAddrStr := createAddr.String() -// -// assetName := "valid" -// //url := "https://my.embedded.\000.null.asset" -// urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=") -// url := string(urlBytes) -// unitName := "asset\rwith\nnon-printable\tcharacters" -// createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA) -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err, "failed to commit") -// -// verify := func(params generated.AssetParams) { -// compareB64 := func(expected string, actual *[]byte) { -// actualStr := string(*actual) -// require.Equal(t, expected, actualStr) -// } -// -// // In all cases, the B64 encoded names should be the same. -// compareB64(assetName, params.NameB64) -// compareB64(unitName, params.UnitNameB64) -// compareB64(url, params.UrlB64) -// -// require.Equal(t, assetName, *params.Name, "valid utf8 should remain") -// require.Nil(t, params.UnitName, "null bytes should not be displayed") -// require.Nil(t, params.Url, "non printable characters should not be displayed") -// } -// -// { -// ////////// -// // When // we lookup the asset -// ////////// -// e := echo.New() -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/assets/") -// -// api := testServerImplementation(db) -// err = api.SearchForAssets(c, generated.SearchForAssetsParams{}) -// require.NoError(t, err) -// -// require.Equal(t, http.StatusOK, rec.Code) -// var response generated.AssetsResponse -// json.Decode(rec.Body.Bytes(), &response) -// -// ////////// -// // Then // we should find one asset with the expected string encodings -// ////////// -// require.Len(t, response.Assets, 1) -// verify(response.Assets[0].Params) -// } -// -// { -// ////////// -// // When // we lookup the account -// ////////// -// e := echo.New() -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/accounts/") -// -// api := testServerImplementation(db) -// err = api.LookupAccountByID(c, test.AccountA.String(), generated.LookupAccountByIDParams{}) -// require.NoError(t, err) -// -// require.Equal(t, http.StatusOK, rec.Code) -// var response generated.AccountResponse -// json.Decode(rec.Body.Bytes(), &response) -// -// ////////// -// // Then // we should find one asset with the expected string encodings -// ////////// -// require.NotNil(t, response.Account.CreatedAssets, 1) -// require.Len(t, *response.Account.CreatedAssets, 1) -// verify((*response.Account.CreatedAssets)[0].Params) -// } -//} -// -//// TestLookupInnerLogs runs queries for logs given application ids, -//// and checks that logs in inner transactions match properly. -//func TestLookupInnerLogs(t *testing.T) { -// var appAddr basics.Address -// appAddr[1] = 99 -// -// params := generated.LookupApplicationLogsByIDParams{} -// -// testcases := []struct { -// name string -// appID uint64 -// logs []string -// }{ -// { -// name: "match on root", -// appID: 123, -// logs: []string{ -// "testing outer appl log", -// "appId 123 log", -// }, -// }, -// { -// name: "match on inner", -// appID: 789, -// logs: []string{ -// "testing inner log", -// "appId 789 log", -// }, -// }, -// { -// name: "match on inner-inner", -// appID: 999, -// logs: []string{ -// "testing inner-inner log", -// "appId 999 log", -// }, -// }, -// } -// -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// /////////// -// // Given // a DB with some inner txns in it. -// /////////// -// appCall := test.MakeAppCallWithInnerAppCall(test.AccountA) -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err, "failed to commit") -// -// for _, tc := range testcases { -// t.Run(tc.name, func(t *testing.T) { -// ////////// -// // When // we run a query that queries logs based on appID -// ////////// -// e := echo.New() -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/applications/:appIdx/logs") -// c.SetParamNames("appIdx") -// c.SetParamValues(fmt.Sprintf("%d", tc.appID)) -// -// api := testServerImplementation(db) -// err = api.LookupApplicationLogsByID(c, tc.appID, params) -// require.NoError(t, err) -// -// ////////// -// // Then // The result is the log from the app -// ////////// -// var response generated.ApplicationLogsResponse -// require.Equal(t, http.StatusOK, rec.Code) -// json.Decode(rec.Body.Bytes(), &response) -// require.NoError(t, err) -// -// require.Equal(t, uint64(tc.appID), response.ApplicationId) -// require.NotNil(t, response.LogData) -// ld := *response.LogData -// require.Equal(t, 1, len(ld)) -// require.Equal(t, len(tc.logs), len(ld[0].Logs)) -// for i, log := range ld[0].Logs { -// require.Equal(t, []byte(tc.logs[i]), log) -// } -// }) -// } -//} -// -//// TestLookupInnerLogs runs queries for logs given application ids, -//// and checks that logs in inner transactions match properly. -//func TestLookupMultiInnerLogs(t *testing.T) { -// var appAddr basics.Address -// appAddr[1] = 99 -// -// params := generated.LookupApplicationLogsByIDParams{} -// -// testcases := []struct { -// name string -// appID uint64 -// numTxnsWithLogs int -// logs []string -// }{ -// { -// name: "match on root with appId 123", -// appID: 123, -// numTxnsWithLogs: 1, -// logs: []string{ -// "testing outer appl log", -// "appId 123 log", -// }, -// }, -// { -// name: "match on inner with appId 789", -// appID: 789, -// numTxnsWithLogs: 1, -// logs: []string{ -// "testing inner log", -// "appId 789 log", -// }, -// }, -// { -// name: "match on inner with appId 222", -// appID: 222, -// numTxnsWithLogs: 3, // There are 6 logs over 3 transactions -// logs: []string{ -// "testing multiple logs 1", -// "appId 222 log 1", -// "testing multiple logs 2", -// "appId 222 log 2", -// "testing multiple logs 3", -// "appId 222 log 3", -// }, -// }, -// } -// -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// /////////// -// // Given // a DB with some inner txns in it. -// /////////// -// appCall := test.MakeAppCallWithMultiLogs(test.AccountA) -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err, "failed to commit") -// -// for _, tc := range testcases { -// t.Run(tc.name, func(t *testing.T) { -// ////////// -// // When // we run a query that queries logs based on appID -// ////////// -// e := echo.New() -// req := httptest.NewRequest(http.MethodGet, "/", nil) -// rec := httptest.NewRecorder() -// c := e.NewContext(req, rec) -// c.SetPath("/v2/applications/:appIdx/logs") -// c.SetParamNames("appIdx") -// c.SetParamValues(fmt.Sprintf("%d", tc.appID)) -// -// api := &ServerImplementation{db: db, timeout: 30 * time.Second} -// err = api.LookupApplicationLogsByID(c, tc.appID, params) -// require.NoError(t, err) -// -// ////////// -// // Then // The result is the log from the app -// ////////// -// var response generated.ApplicationLogsResponse -// require.Equal(t, http.StatusOK, rec.Code) -// json.Decode(rec.Body.Bytes(), &response) -// require.NoError(t, err) -// -// require.Equal(t, uint64(tc.appID), response.ApplicationId) -// require.NotNil(t, response.LogData) -// ld := *response.LogData -// require.Equal(t, tc.numTxnsWithLogs, len(ld)) -// -// logCount := 0 -// for txnIndex, result := range ld { -// for logIndex, log := range result.Logs { -// require.Equal(t, []byte(tc.logs[txnIndex*2+logIndex]), log) -// logCount++ -// } -// } -// require.Equal(t, logCount, len(tc.logs)) -// }) -// } -//} +func TestKeyregTransactionWithStateProofKeys(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // A block containing a key reg txn with state proof key + /////////// + var votePK crypto.OneTimeSignatureVerifier + votePK[0] = 1 + + var selectionPK crypto.VRFVerifier + selectionPK[0] = 1 + + var stateProofPK merklesignature.Verifier + stateProofPK[0] = 1 + + txn := transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + Type: "keyreg", + Header: transactions.Header{ + Sender: test.AccountA, + GenesisHash: test.GenesisHash, + LastValid: 10, + }, + KeyregTxnFields: transactions.KeyregTxnFields{ + VotePK: votePK, + SelectionPK: selectionPK, + StateProofPK: stateProofPK, + VoteFirst: basics.Round(0), + VoteLast: basics.Round(100), + VoteKeyDilution: 1000, + Nonparticipation: false, + }, + }, + Sig: test.Signature, + }, + } + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err, "failed to commit") + + e := echo.New() + { + ////////// + // When // We query the txn + ////////// + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/transactions/:txid") + api := &ServerImplementation{db: db} + err = api.LookupTransaction(c, txn.Txn.ID().String()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rec.Code) + ////////// + // Then // The key reg txn response has state proof key + ////////// + var response generated.TransactionResponse + data := rec.Body.Bytes() + err = json.Decode(data, &response) + require.NoError(t, err) + require.NotNil(t, response.Transaction.KeyregTransaction.StateProofKey) + require.Equal(t, stateProofPK[:], *response.Transaction.KeyregTransaction.StateProofKey) + } + { + ////////// + // And // Account is online with state proof key + ////////// + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/accounts/:account-id") + api := &ServerImplementation{db: db} + params := generated.LookupAccountByIDParams{} + err = api.LookupAccountByID(c, test.AccountA.String(), params) + require.NoError(t, err) + require.Equal(t, http.StatusOK, rec.Code) + + var acctResp generated.AccountResponse + data := rec.Body.Bytes() + err = json.Decode(data, &acctResp) + require.NoError(t, err) + require.NotNil(t, acctResp.Account) + require.NotNil(t, acctResp.Account.Participation.StateProofKey) + require.Equal(t, stateProofPK[:], *acctResp.Account.Participation.StateProofKey) + } +} + +func TestVersion(t *testing.T) { + /////////// + // Given // An API and context + /////////// + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + api := testServerImplementation(db) + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec1 := httptest.NewRecorder() + c := e.NewContext(req, rec1) + + ////////// + // When // we call the health endpoint + ////////// + err := api.MakeHealthCheck(c) + + ////////// + // Then // We get the health information. + ////////// + require.NoError(t, err) + require.Equal(t, http.StatusOK, rec1.Code) + var response generated.HealthCheckResponse + json.Decode(rec1.Body.Bytes(), &response) + + require.Equal(t, uint64(0), response.Round) + require.False(t, response.IsMigrating) + // This is weird looking because the version is set with -ldflags + require.Equal(t, response.Version, "(unknown version)") +} + +func TestAccountClearsNonUTF8(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // a DB with some inner txns in it. + /////////// + //var createAddr basics.Address + //createAddr[1] = 99 + //createAddrStr := createAddr.String() + + assetName := "valid" + //url := "https://my.embedded.\000.null.asset" + urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=") + url := string(urlBytes) + unitName := "asset\rwith\nnon-printable\tcharacters" + createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA, 10) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err, "failed to commit") + + verify := func(params generated.AssetParams) { + compareB64 := func(expected string, actual *[]byte) { + actualStr := string(*actual) + require.Equal(t, expected, actualStr) + } + + // In all cases, the B64 encoded names should be the same. + compareB64(assetName, params.NameB64) + compareB64(unitName, params.UnitNameB64) + compareB64(url, params.UrlB64) + + require.Equal(t, assetName, *params.Name, "valid utf8 should remain") + require.Nil(t, params.UnitName, "null bytes should not be displayed") + require.Nil(t, params.Url, "non printable characters should not be displayed") + } + + { + ////////// + // When // we lookup the asset + ////////// + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/assets/") + + api := testServerImplementation(db) + err = api.SearchForAssets(c, generated.SearchForAssetsParams{}) + require.NoError(t, err) + + require.Equal(t, http.StatusOK, rec.Code) + var response generated.AssetsResponse + json.Decode(rec.Body.Bytes(), &response) + + ////////// + // Then // we should find one asset with the expected string encodings + ////////// + require.Len(t, response.Assets, 1) + verify(response.Assets[0].Params) + } + + { + ////////// + // When // we lookup the account + ////////// + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/accounts/") + + api := testServerImplementation(db) + err = api.LookupAccountByID(c, test.AccountA.String(), generated.LookupAccountByIDParams{}) + require.NoError(t, err) + + require.Equal(t, http.StatusOK, rec.Code) + var response generated.AccountResponse + json.Decode(rec.Body.Bytes(), &response) + + ////////// + // Then // we should find one asset with the expected string encodings + ////////// + require.NotNil(t, response.Account.CreatedAssets, 1) + require.Len(t, *response.Account.CreatedAssets, 1) + verify((*response.Account.CreatedAssets)[0].Params) + } +} + +// TestLookupInnerLogs runs queries for logs given application ids, +// and checks that logs in inner transactions match properly. +func TestLookupInnerLogs(t *testing.T) { + var appAddr basics.Address + appAddr[1] = 99 + + params := generated.LookupApplicationLogsByIDParams{} + + testcases := []struct { + name string + appID uint64 + logs []string + }{ + { + name: "match on root", + appID: 123, + logs: []string{ + "testing outer appl log", + "appId 123 log", + }, + }, + { + name: "match on inner", + appID: 789, + logs: []string{ + "testing inner log", + "appId 789 log", + }, + }, + { + name: "match on inner-inner", + appID: 999, + logs: []string{ + "testing inner-inner log", + "appId 999 log", + }, + }, + } + + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // a DB with some inner txns in it. + /////////// + appCall := test.MakeAppCallWithInnerAppCall(test.AccountA, 10) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err, "failed to commit") + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ////////// + // When // we run a query that queries logs based on appID + ////////// + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/applications/:appIdx/logs") + c.SetParamNames("appIdx") + c.SetParamValues(fmt.Sprintf("%d", tc.appID)) + + api := testServerImplementation(db) + err = api.LookupApplicationLogsByID(c, tc.appID, params) + require.NoError(t, err) + + ////////// + // Then // The result is the log from the app + ////////// + var response generated.ApplicationLogsResponse + require.Equal(t, http.StatusOK, rec.Code) + json.Decode(rec.Body.Bytes(), &response) + require.NoError(t, err) + + require.Equal(t, uint64(tc.appID), response.ApplicationId) + require.NotNil(t, response.LogData) + ld := *response.LogData + require.Equal(t, 1, len(ld)) + require.Equal(t, len(tc.logs), len(ld[0].Logs)) + for i, log := range ld[0].Logs { + require.Equal(t, []byte(tc.logs[i]), log) + } + }) + } +} + +// TestLookupInnerLogs runs queries for logs given application ids, +// and checks that logs in inner transactions match properly. +func TestLookupMultiInnerLogs(t *testing.T) { + var appAddr basics.Address + appAddr[1] = 99 + + params := generated.LookupApplicationLogsByIDParams{} + + testcases := []struct { + name string + appID uint64 + numTxnsWithLogs int + logs []string + }{ + { + name: "match on root with appId 123", + appID: 123, + numTxnsWithLogs: 1, + logs: []string{ + "testing outer appl log", + "appId 123 log", + }, + }, + { + name: "match on inner with appId 789", + appID: 789, + numTxnsWithLogs: 1, + logs: []string{ + "testing inner log", + "appId 789 log", + }, + }, + { + name: "match on inner with appId 222", + appID: 222, + numTxnsWithLogs: 3, // There are 6 logs over 3 transactions + logs: []string{ + "testing multiple logs 1", + "appId 222 log 1", + "testing multiple logs 2", + "appId 222 log 2", + "testing multiple logs 3", + "appId 222 log 3", + }, + }, + } + + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + /////////// + // Given // a DB with some inner txns in it. + /////////// + appCall := test.MakeAppCallWithMultiLogs(test.AccountA, 10) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err, "failed to commit") + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ////////// + // When // we run a query that queries logs based on appID + ////////// + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetPath("/v2/applications/:appIdx/logs") + c.SetParamNames("appIdx") + c.SetParamValues(fmt.Sprintf("%d", tc.appID)) + + api := &ServerImplementation{db: db, timeout: 30 * time.Second} + err = api.LookupApplicationLogsByID(c, tc.appID, params) + require.NoError(t, err) + + ////////// + // Then // The result is the log from the app + ////////// + var response generated.ApplicationLogsResponse + require.Equal(t, http.StatusOK, rec.Code) + json.Decode(rec.Body.Bytes(), &response) + require.NoError(t, err) + + require.Equal(t, uint64(tc.appID), response.ApplicationId) + require.NotNil(t, response.LogData) + ld := *response.LogData + require.Equal(t, tc.numTxnsWithLogs, len(ld)) + + logCount := 0 + for txnIndex, result := range ld { + for logIndex, log := range result.Logs { + require.Equal(t, []byte(tc.logs[txnIndex*2+logIndex]), log) + logCount++ + } + } + require.Equal(t, logCount, len(tc.logs)) + }) + } +} diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 6dd5d9a67..5bf3b8858 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -47,33 +47,28 @@ func (processor *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error if err != nil { return fmt.Errorf("Process() block eval err: %w", err) } - - paysetgroups, err := blockCert.Block.DecodePaysetGroups() + _, _, err = blkeval.ProcessBlockForIndexer(&blockCert.Block) if err != nil { - return fmt.Errorf("Process() decode payset groups err: %w", err) + return fmt.Errorf("%w", err) } - - for _, group := range paysetgroups { - err = blkeval.TransactionGroup(group) - if err != nil { - return fmt.Errorf("Process() apply transaction group err: %w", err) - } + genblk, err := blkeval.GenerateBlock() + if err != nil { + return fmt.Errorf("%w", err) } - // validated block - vb, err := blkeval.GenerateBlock() + vb := ledgercore.MakeValidatedBlock(blockCert.Block, genblk.Delta()) if err != nil { - return fmt.Errorf("Process() validated block err: %w", err) + return fmt.Errorf("Process() block eval err: %w", err) } // execute handler before writing to local ledger if processor.handler != nil { - err = processor.handler(vb) + err = processor.handler(&vb) if err != nil { return fmt.Errorf("Process() handler err: %w", err) } } // write to ledger - err = processor.ledger.AddValidatedBlock(*vb, blockCert.Certificate) + err = processor.ledger.AddValidatedBlock(vb, blockCert.Certificate) if err != nil { return fmt.Errorf("Process() add validated block err: %w", err) } diff --git a/processor/blockprocessor/block_processor_test.go b/processor/blockprocessor/block_processor_test.go index a80a82999..34531f551 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/processor/blockprocessor/block_processor_test.go @@ -76,7 +76,7 @@ func TestFailedProcess(t *testing.T) { assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() apply transaction group") + assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // stxn GenesisID not empty txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) @@ -85,7 +85,7 @@ func TestFailedProcess(t *testing.T) { block.Payset[0].Txn.GenesisID = "genesisID" rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() decode payset groups err") + assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // eval error: concensus protocol not supported txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index ce9aa571d..43ac75117 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -376,8 +376,8 @@ func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSen // |- application call // |- application call // |- application call -func MakeAppCallWithMultiLogs(appSender basics.Address) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender, 0) +func MakeAppCallWithMultiLogs(appSender basics.Address, lv uint64) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender, lv) // Add a log to the outer appl call createApp.ApplicationID = 123 @@ -435,8 +435,8 @@ func MakeAppCallWithMultiLogs(appSender basics.Address) transactions.SignedTxnWi // application create // |- application call // |- application create -func MakeAppCallWithInnerAppCall(appSender basics.Address) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender, 0) +func MakeAppCallWithInnerAppCall(appSender basics.Address, lv uint64) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender, lv) // Add a log to the outer appl call createApp.ApplicationID = 123 From 4dbda7d177575bdc6cb0914182ee8cb70b6fc52b Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 18 May 2022 16:34:35 -0400 Subject: [PATCH 07/37] updating postgres integration tests --- idb/postgres/internal/writer/writer_test.go | 12 +- idb/postgres/postgres.go | 144 +- .../postgres_integration_common_test.go | 4 +- idb/postgres/postgres_integration_test.go | 1930 +++++++++-------- processor/blockprocessor/block_processor.go | 1 + .../blockprocessor/block_processor_test.go | 12 +- util/test/account_testutil.go | 9 +- 7 files changed, 1117 insertions(+), 995 deletions(-) diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index 910fbd437..b0893e59c 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -189,9 +189,7 @@ func TestWriterTxnTableBasic(t *testing.T) { Payset: make([]transactions.SignedTxnInBlock, 2), } - stxnad0 := test.MakePaymentTxn( - 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, - basics.Address{}) + stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) var err error block.Payset[0], err = block.BlockHeader.EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) @@ -335,9 +333,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { var tests []testtype { - stxnad0 := test.MakePaymentTxn( - 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, - basics.Address{}) + stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) stib0, err := makeBlockFunc().EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) @@ -628,9 +624,7 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { Payset: make(transactions.Payset, 1), } - stxnad := test.MakePaymentTxn( - 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, - basics.Address{}) + stxnad := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) stxnad.Sig[0] = 5 // set signature so that keytype for account is updated var err error block.Payset[0], err = block.EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 706de0f8c..4434dfbff 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -263,98 +263,98 @@ func (db *IndexerDb) AddBlock(vb *ledgercore.ValidatedBlock) error { if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - if block.Round() != basics.Round(importstate.NextRoundToAccount) { + if block.Round() > basics.Round(importstate.NextRoundToAccount) { return fmt.Errorf( "AddBlock() adding block round %d but next round to account is %d", block.Round(), importstate.NextRoundToAccount) - } - importstate.NextRoundToAccount++ - err = db.setImportState(tx, &importstate) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - - w, err := writer.MakeWriter(tx) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - defer w.Close() - - if block.Round() == basics.Round(0) { - // Block 0 is special, we cannot run the evaluator on it. - err := w.AddBlock0(&block) + } else if block.Round() == basics.Round(importstate.NextRoundToAccount) { + importstate.NextRoundToAccount++ + err = db.setImportState(tx, &importstate) if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - } else { - proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] - if !ok { - return fmt.Errorf( - "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + + w, err := writer.MakeWriter(tx) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) } - protoChanged := !proto.EnableAssetCloseAmount - proto.EnableAssetCloseAmount = true - - var wg sync.WaitGroup - defer wg.Wait() - - // Write transaction participation and possibly transactions in a parallel db - // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start - // writing transactions contained in the block early. - var err0 error - wg.Add(1) - go func() { - defer wg.Done() - - f := func(tx pgx.Tx) error { - if !protoChanged { - err := writer.AddTransactions(&block, block.Payset, tx) - if err != nil { - return err - } - } - return writer.AddTransactionParticipation(&block, tx) + defer w.Close() + + if block.Round() == basics.Round(0) { + // Block 0 is special, we cannot run the evaluator on it. + err := w.AddBlock0(&block) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) } - err0 = db.txWithRetry(serializable, f) - }() + } else { + proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] + if !ok { + return fmt.Errorf( + "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + } + protoChanged := !proto.EnableAssetCloseAmount + proto.EnableAssetCloseAmount = true + + var wg sync.WaitGroup + defer wg.Wait() - var err1 error - // Skip if transaction writing has already started. - if protoChanged { - // Write transactions in a parallel db transaction. + // Write transaction participation and possibly transactions in a parallel db + // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start + // writing transactions contained in the block early. + var err0 error wg.Add(1) go func() { defer wg.Done() f := func(tx pgx.Tx) error { - return writer.AddTransactions(&block, block.Payset, tx) + if !protoChanged { + err := writer.AddTransactions(&block, block.Payset, tx) + if err != nil { + return err + } + } + return writer.AddTransactionParticipation(&block, tx) } - err1 = db.txWithRetry(serializable, f) + err0 = db.txWithRetry(serializable, f) }() - } - err = w.AddBlock(&block, block.Payset, vb.Delta()) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } + var err1 error + // Skip if transaction writing has already started. + if protoChanged { + // Write transactions in a parallel db transaction. + wg.Add(1) + go func() { + defer wg.Done() - // Wait for goroutines to finish and check for errors. If there is an error, we - // return our own error so that the main transaction does not commit. Hence, - // `txn` and `txn_participation` tables can only be ahead but not behind - // the other state. - wg.Wait() - isUniqueViolationFunc := func(err error) bool { - var pgerr *pgconn.PgError - return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) - } - if (err0 != nil) && !isUniqueViolationFunc(err0) { - return fmt.Errorf("AddBlock() err0: %w", err0) - } - if (err1 != nil) && !isUniqueViolationFunc(err1) { - return fmt.Errorf("AddBlock() err1: %w", err1) + f := func(tx pgx.Tx) error { + return writer.AddTransactions(&block, block.Payset, tx) + } + err1 = db.txWithRetry(serializable, f) + }() + } + + err = w.AddBlock(&block, block.Payset, vb.Delta()) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) + } + + // Wait for goroutines to finish and check for errors. If there is an error, we + // return our own error so that the main transaction does not commit. Hence, + // `txn` and `txn_participation` tables can only be ahead but not behind + // the other state. + wg.Wait() + isUniqueViolationFunc := func(err error) bool { + var pgerr *pgconn.PgError + return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) + } + if (err0 != nil) && !isUniqueViolationFunc(err0) { + return fmt.Errorf("AddBlock() err0: %w", err0) + } + if (err1 != nil) && !isUniqueViolationFunc(err1) { + return fmt.Errorf("AddBlock() err1: %w", err1) + } } } - return nil } err := db.txWithRetry(serializable, f) diff --git a/idb/postgres/postgres_integration_common_test.go b/idb/postgres/postgres_integration_common_test.go index 4b4efa3cd..9546b6466 100644 --- a/idb/postgres/postgres_integration_common_test.go +++ b/idb/postgres/postgres_integration_common_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/require" @@ -19,7 +20,8 @@ func setupIdbWithConnectionString(t *testing.T, connStr string, genesis bookkeep err = idb.LoadGenesis(genesis) require.NoError(t, err) - err = idb.AddBlock(&genesisBlock) + vb := ledgercore.MakeValidatedBlock(genesisBlock, ledgercore.StateDelta{}) + err = idb.AddBlock(&vb) require.NoError(t, err) return idb diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index 7664fbbd2..3d26d2f84 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -1,36 +1,50 @@ package postgres import ( - "bytes" "context" "database/sql" "fmt" "math" "sync" "testing" - "time" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-codec/codec" - "github.com/algorand/indexer/importer" + "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/api/generated/v2" + "github.com/algorand/indexer/idb/postgres/internal/encoding" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/algorand/indexer/api/generated/v2" "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/idb/postgres/internal/encoding" - "github.com/algorand/indexer/idb/postgres/internal/schema" pgtest "github.com/algorand/indexer/idb/postgres/internal/testing" - pgutil "github.com/algorand/indexer/idb/postgres/internal/util" "github.com/algorand/indexer/util/test" ) +func makeTestLedger(t *testing.T) *ledger.Ledger { + genesis := test.MakeGenesis() + genesisBlock := test.MakeGenesisBlock() + initState, err := util.CreateInitState(&genesis, &genesisBlock) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + l, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + return l +} + // TestMaxRoundOnUninitializedDB makes sure we return 0 when getting the max round on a new DB. func TestMaxRoundOnUninitializedDB(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) @@ -157,23 +171,29 @@ func TestAssetCloseReopenTransfer(t *testing.T) { /////////// // Given // A round scenario requiring subround accounting: AccountA is funded, closed, opts back, and funded again. /////////// - createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD, 0) - optInA := test.MakeAssetOptInTxn(assetid, test.AccountA, 0) - fundA := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) - optInB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - optInC := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) - closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 0) - payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD, 10) + optInA := test.MakeAssetOptInTxn(assetid, test.AccountA, 10) + fundA := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 10) + optInB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + optInC := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) + closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 10) + optInDA := test.MakeAssetOptInTxn(assetid, test.AccountA, 12) + payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 12) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optInA, &fundA, &optInB, - &optInC, &closeA, &optInA, &payMain) + &optInC, &closeA, &optInDA, &payMain) require.NoError(t, err) ////////// // When // We commit the block to the database ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -189,7 +209,7 @@ func TestAssetCloseReopenTransfer(t *testing.T) { assertAccountAsset(t, db.db, test.AccountD, assetid, false, total-2*amt) } -// TestReCreateAssetHolding checks the optin value of a defunct +//TestReCreateAssetHolding checks the optin value of a defunct func TestReCreateAssetHolding(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() @@ -204,22 +224,27 @@ func TestReCreateAssetHolding(t *testing.T) { // A new asset with default-frozen, AccountB opts-in and has its frozen state // toggled. /////////// Then AccountB opts-out then opts-in again. - createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - unfreezeB := test.MakeAssetFreezeTxn( - assetid, !frozen, test.AccountA, test.AccountB) - optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD, 0) + createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + unfreezeB := test.MakeAssetFreezeTxn(assetid, !frozen, test.AccountA, test.AccountB, 10) + optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD, 10) + optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB, 12) var err error block, err = test.MakeBlockForTxns( block.BlockHeader, &createAssetFrozen, &optinB, &unfreezeB, - &optoutB, &optinB) + &optoutB, &optinB2) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -241,18 +266,24 @@ func TestNoopOptins(t *testing.T) { /////////// assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 0) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 10) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB, 10) + optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB, 12) block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &unfreezeB, &optinB) + test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &unfreezeB, &optinB2) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -266,17 +297,20 @@ func TestMultipleWriters(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - amt := uint64(10000) + amt := uint64(100000) //min balance /////////// // Given // Send amt to AccountE /////////// - payAccountE := test.MakePaymentTxn( - 1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, - basics.Address{}) + payAccountE := test.MakePaymentTxn(1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, basics.Address{}, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &payAccountE) require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} ////////// // When // We attempt commit the round accounting multiple times. @@ -290,7 +324,7 @@ func TestMultipleWriters(t *testing.T) { go func() { defer wg.Done() <-start - errors <- db.AddBlock(&block) + errors <- proc.Process(&blockCert) }() } close(start) @@ -330,20 +364,25 @@ func TestBlockWithTransactions(t *testing.T) { /////////// // Given // A block at round `round` with 5 transactions. /////////// - txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 0) - txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA, 0) - txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) - txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) - txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 0) - txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}, 0) - txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 0) + txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 10) + txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA, 10) + txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 10) + txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) + txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 10) + txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}, 12) + txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 12) txns := []*transactions.SignedTxnWithAD{ &txn1, &txn2, &txn3, &txn4, &txn5, &txn6, &txn7, &txn8} block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, txns...) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -378,12 +417,16 @@ func TestRekeyBasic(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -406,21 +449,18 @@ func TestRekeyToItself(t *testing.T) { /////////// // Given // Send rekey transactions /////////// - txn := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA, 0) - txn = test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, - test.AccountA) - block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -433,6 +473,7 @@ func TestRekeyToItself(t *testing.T) { ad, err := encoding.DecodeTrimmedLcAccountData(accountDataStr) require.NoError(t, err, "failed to parse account data json") + fmt.Println("addr:", ad.AuthAddr.String()) assert.Equal(t, basics.Address{}, ad.AuthAddr) } @@ -443,19 +484,20 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn0 := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, - test.AccountB) - txn1 := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, - basics.Address{}) - txn2 := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC) + txn0 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 0) + txn1 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA, 0) + txn2 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC, 0) + block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -478,17 +520,21 @@ func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn( - 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, - basics.Address{}) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) ////////// // Then // No error when committing to the DB. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) + } // TestIgnoreDefaultFrozenConfigUpdate the creator asset holding should ignore default-frozen = true. @@ -502,9 +548,9 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { /////////// // Given // A new asset with default-frozen = true, and AccountB opting into it. /////////// - createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) - modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 0) - optin := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) + createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) + modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) + optin := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAssetNotFrozen, &modifyAssetToFrozen, @@ -514,7 +560,12 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -536,14 +587,19 @@ func TestZeroTotalAssetCreate(t *testing.T) { /////////// // Given // A new asset with total = 0. /////////// - createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -593,19 +649,25 @@ func TestDestroyAssetBasic(t *testing.T) { assetID := uint64(1) // Create an asset. - txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA, 0) + txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Destroy an asset. - txn = test.MakeAssetDestroyTxn(assetID, test.AccountA, 0) + txn = test.MakeAssetDestroyTxn(assetID, test.AccountA, 10) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Check that the asset is deleted. @@ -628,12 +690,17 @@ func TestDestroyAssetZeroSupply(t *testing.T) { assetID := uint64(1) // Create an asset. Set total supply to 0. - txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA, 0) - txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA, 0) + txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA, 10) + txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Check that the asset is deleted. @@ -664,6 +731,7 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, + LastValid: 10, }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ AssetParams: basics.AssetParams{ @@ -679,17 +747,21 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { } // Another account opts in. - txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC, 0) + txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC, 10) // Destroy an asset. - txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB, 0) + txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB, 10) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) - // Check that the creator's asset holding is deleted. assertAssetHoldingDates(t, db.db, test.AccountA, assetID, sql.NullBool{Valid: true, Bool: true}, @@ -720,9 +792,9 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { // Create a block with freeze txn assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB, 10) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &freeze) @@ -731,7 +803,12 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -764,7 +841,7 @@ func TestInnerTxnParticipation(t *testing.T) { // otherwise it requires funding the app account and other special setup var appAddr basics.Address appAddr[1] = 99 - createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) + createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createApp) @@ -773,7 +850,12 @@ func TestInnerTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) ////////// @@ -810,6 +892,7 @@ func TestAppExtraPages(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, + LastValid: 10, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -824,8 +907,13 @@ func TestAppExtraPages(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err, "failed to commit") + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) row := db.db.QueryRow(context.Background(), "SELECT index, params FROM app WHERE creator = $1", test.AccountA[:]) @@ -893,30 +981,34 @@ func TestKeytypeBasic(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Sig - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) keytype := "sig" assertKeytype(t, db, test.AccountA, &keytype) // Msig - txn = test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}, 0) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) keytype = "msig" - assertKeytype(t, db, test.AccountA, &keytype) + assertKeytype(t, db, test.AccountB, &keytype) } // Test that asset amount >= 2^63 is handled correctly. Due to the specifics of @@ -926,11 +1018,16 @@ func TestLargeAssetAmount(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA, 0) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) { @@ -1065,17 +1162,22 @@ func TestNonDisplayableUTF8(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10) // Try to add cheeky inner txns lazily by adding an AD to the acfg txn txn.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ - test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0), + test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10), } txn.ApplyData.EvalDelta.InnerTxns[0].ConfigAsset = basics.AssetIndex(innerAssetID) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) // Test 1: import/accounting should work. - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1150,10 +1252,15 @@ func TestReconfigAsset(t *testing.T) { url := "https://algorand.com" assetID := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 0) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) txn = transactions.SignedTxnWithAD{ @@ -1164,6 +1271,7 @@ func TestReconfigAsset(t *testing.T) { Sender: test.AccountA, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: test.GenesisHash, + LastValid: 10, }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(assetID), @@ -1178,7 +1286,8 @@ func TestReconfigAsset(t *testing.T) { } block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1206,42 +1315,46 @@ func TestKeytypeResetsOnRekey(t *testing.T) { defer shutdownFunc() // Sig - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) keytype := "sig" assertKeytype(t, db, test.AccountA, &keytype) // Rekey. - txn = test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 10) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) assertKeytype(t, db, test.AccountA, nil) // Msig - txn = test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}, 10) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) txn.AuthAddr = test.AccountB block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) keytype = "msig" - assertKeytype(t, db, test.AccountA, &keytype) + assertKeytype(t, db, test.AccountB, &keytype) } // Test that after closing the account, keytype will be correctly set. @@ -1253,11 +1366,16 @@ func TestKeytypeDeletedAccount(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) closeTxn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}) + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}, 10) block, err := test.MakeBlockForTxns(block.BlockHeader, &closeTxn) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) keytype := "sig" @@ -1296,19 +1414,24 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 0) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 0) - transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}, 0) - optinC := test.MakeAssetOptInTxn(assetid, test.AccountC, 0) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}, 10) + optinC := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) // Close B to C. - closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC, 0) + closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC, 10) block, err := test.MakeBlockForTxns( block.BlockHeader, &createAsset, &optinB, &transferAB, &optinC, &closeB) require.NoError(t, err) - err = db.AddBlock(&block) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) // Check asset close amount in the `closeB` transaction. @@ -1344,17 +1467,19 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { require.NoError(t, err) assert.Equal(t, uint64(0), round) - block := test.MakeGenesisBlock() - err = db.AddBlock(&block) - require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") round, err = db.GetNextRoundToAccount() require.NoError(t, err) assert.Equal(t, uint64(1), round) - block, err = test.MakeBlockForTxns(block.BlockHeader) + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) round, err = db.GetNextRoundToAccount() @@ -1363,7 +1488,8 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader) require.NoError(t, err) - err = db.AddBlock(&block) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) require.NoError(t, err) round, err = db.GetNextRoundToAccount() @@ -1371,770 +1497,770 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { assert.Equal(t, uint64(3), round) } -// Test that AddBlock makes a record of an account that gets created and deleted in -// the same round. -func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - createTxn := test.MakePaymentTxn( - 0, 5, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) - deleteTxn := test.MakePaymentTxn( - 0, 2, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) - block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err) - - opts := idb.AccountQueryOptions{ - EqualToAddress: test.AccountE[:], - IncludeDeleted: true, - } - rowsCh, _ := db.GetAccounts(context.Background(), opts) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - require.NotNil(t, row.Account.Deleted) - assert.True(t, *row.Account.Deleted) - require.NotNil(t, row.Account.CreatedAtRound) - assert.Equal(t, uint64(1), *row.Account.CreatedAtRound) - require.NotNil(t, row.Account.ClosedAtRound) - assert.Equal(t, uint64(1), *row.Account.ClosedAtRound) -} - -// Test that AddBlock makes a record of an asset that is created and deleted in -// the same round. -func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - assetid := uint64(1) - createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 0) - deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 0) - block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err) - - // Asset global state. - { - opts := idb.AssetsQuery{ - AssetID: assetid, - IncludeDeleted: true, - } - rowsCh, _ := db.Assets(context.Background(), opts) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - require.NotNil(t, row.Deleted) - assert.True(t, *row.Deleted) - require.NotNil(t, row.CreatedRound) - assert.Equal(t, uint64(1), *row.CreatedRound) - require.NotNil(t, row.ClosedRound) - assert.Equal(t, uint64(1), *row.ClosedRound) - } - - // Asset local state. - { - opts := idb.AssetBalanceQuery{ - AssetID: assetid, - IncludeDeleted: true, - } - rowsCh, _ := db.AssetBalances(context.Background(), opts) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - require.NotNil(t, row.Deleted) - assert.True(t, *row.Deleted) - require.NotNil(t, row.CreatedRound) - assert.Equal(t, uint64(1), *row.CreatedRound) - require.NotNil(t, row.ClosedRound) - assert.Equal(t, uint64(1), *row.ClosedRound) - } -} - -// Test that AddBlock makes a record of an app that is created and deleted in -// the same round. -func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA, 0) - deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) - block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err) - - opts := idb.ApplicationQuery{ - ApplicationID: appid, - IncludeDeleted: true, - } - rowsCh, _ := db.Applications(context.Background(), opts) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - require.NotNil(t, row.Application.Deleted) - assert.True(t, *row.Application.Deleted) - require.NotNil(t, row.Application.CreatedAtRound) - assert.Equal(t, uint64(1), *row.Application.CreatedAtRound) - require.NotNil(t, row.Application.DeletedAtRound) - assert.Equal(t, uint64(1), *row.Application.DeletedAtRound) -} - -// Test that AddBlock makes a record of an app that is created and deleted in -// the same round. -func TestAddBlockAppOptInOutSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA, 0) - optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 0) - optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB) - block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err) - - opts := idb.AccountQueryOptions{ - EqualToAddress: test.AccountB[:], - IncludeDeleted: true, - IncludeAppLocalState: true, - } - rowsCh, _ := db.GetAccounts(context.Background(), opts) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - - require.NotNil(t, row.Account.AppsLocalState) - require.Equal(t, 1, len(*row.Account.AppsLocalState)) - - localState := (*row.Account.AppsLocalState)[0] - require.NotNil(t, localState.Deleted) - assert.True(t, *localState.Deleted) - require.NotNil(t, localState.OptedInAtRound) - assert.Equal(t, uint64(1), *localState.OptedInAtRound) - require.NotNil(t, localState.ClosedOutAtRound) - assert.Equal(t, uint64(1), *localState.ClosedOutAtRound) - require.Equal(t, uint64(0), row.Account.TotalAppsOptedIn) - - q := idb.ApplicationQuery{ - ApplicationID: appid, - IncludeDeleted: true, - } - lsRows, _ := db.AppLocalState(context.Background(), q) - lsRow, ok := <-lsRows - require.True(t, ok) - require.NoError(t, lsRow.Error) - ls := lsRow.AppLocalState - require.Equal(t, appid, ls.Id) - require.NotNil(t, ls.Deleted) - assert.True(t, *ls.Deleted) - require.NotNil(t, ls.OptedInAtRound) - assert.Equal(t, uint64(1), *ls.OptedInAtRound) - require.NotNil(t, ls.ClosedOutAtRound) - assert.Equal(t, uint64(1), *ls.ClosedOutAtRound) -} - -// TestSearchForInnerTransactionReturnsRootTransaction checks that the parent -// transaction is returned when matching on inner transactions if the -// ReturnInnerTxnFlag is false. If the ReturnInnerTxnFlag is true, it should -// return the inner txn instead. -func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { - var appAddr basics.Address - appAddr[1] = 99 - - tests := []struct { - name string - matches int - returnInner bool - filter idb.TransactionFilter - }{ - { - name: "match on root, inner, and inner-inners, return root", - matches: 3, - returnInner: false, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication}, - }, - { - name: "match on inner, return root", - matches: 1, - returnInner: false, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay}, - }, - { - name: "match on inner-inner, return root", - matches: 1, - returnInner: false, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer}, - }, - { - name: "match all, return root", - matches: 5, - returnInner: false, - filter: idb.TransactionFilter{Address: appAddr[:]}, - }, - { - name: "match on root, inner, and inner-inners, return inners", - matches: 3, - returnInner: true, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication, ReturnInnerTxnOnly: true}, - }, - { - name: "match on inner, return inners", - matches: 1, - returnInner: true, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay, ReturnInnerTxnOnly: true}, - }, - { - name: "match on inner-inner, return inners", - matches: 1, - returnInner: true, - filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer, ReturnInnerTxnOnly: true}, - }, - { - name: "match all, return inners", - matches: 5, - returnInner: true, - filter: idb.TransactionFilter{Address: appAddr[:], ReturnInnerTxnOnly: true}, - }, - } - - // Given: A DB with one transaction containing inner transactions [app -> pay -> xfer] - pdb, connStr, shutdownFunc := pgtest.SetupPostgres(t) - defer shutdownFunc() - db := setupIdbWithConnectionString( - t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) - defer db.Close() - - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) - require.NoError(t, err) - rootTxid := appCall.Txn.ID() - - err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { - return db.AddBlock(&block) - }, nil) - require.NoError(t, err) - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - // When: searching for a transaction that matches part of the transaction. - results, _ := db.Transactions(context.Background(), tc.filter) - - // Then: only the root transaction should be returned if the ReturnInnerTxnOnly flag is true. - // Else if ReturnInnerTxnOnly is false, then the inner txn should be returned. - num := 0 - for result := range results { - num++ - require.NoError(t, result.Error) - - if tc.returnInner { - // Make sure that only the inner txn is returned - require.True(t, (result.Txn != nil) && (result.RootTxn == nil)) - } else { - // Make sure the root txn is returned. - var stxn *transactions.SignedTxnWithAD - - // Exactly one of Txn and RootTxn must be present. - require.True(t, (result.Txn == nil) != (result.RootTxn == nil)) - - // Get Txn or RootTxn - if result.Txn != nil { - stxn = result.Txn - } - if result.RootTxn != nil { - stxn = result.RootTxn - } - require.Equal(t, rootTxid, stxn.Txn.ID()) - } - } - - // There can be multiple matches because deduplication happens in REST API. - require.Equal(t, tc.matches, num) - }) - } -} - -// TestNonUTF8Logs makes sure we're able to import cheeky logs -// for both the root and inner transactions. -func TestNonUTF8Logs(t *testing.T) { - tests := []struct { - Name string - Logs []string - }{ - { - Name: "Normal", - Logs: []string{"Test log1", "Test log2", "Test log3"}, - }, - { - Name: "Embedded Null", - Logs: []string{"\000", "\x00\x00\x00\x00\x00\x00\x00\x00", string([]byte{00, 00})}, - }, - { - Name: "Invalid UTF8", - Logs: []string{"\x8c", "\xff", "\xf8"}, - }, - { - Name: "Emoji", - Logs: []string{"💩", "💰", "🌐"}, - }, - } - - for _, testcase := range tests { - testcase := testcase - - t.Run(testcase.Name, func(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) - createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ - Logs: testcase.Logs, - InnerTxns: []transactions.SignedTxnWithAD{ - // Inner application call with nested cheeky logs - { - SignedTxn: transactions.SignedTxn{ - Txn: transactions.Transaction{ - Type: protocol.ApplicationCallTx, - Header: transactions.Header{ - Sender: test.AccountA, - }, - ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApplicationID: 789, - OnCompletion: transactions.NoOpOC, - }, - }, - }, - ApplyData: transactions.ApplyData{ - EvalDelta: transactions.EvalDelta{ - Logs: testcase.Logs, - }, - }, - }, - }, - } - - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn) - require.NoError(t, err) - - // Test 1: import/accounting should work. - err = db.AddBlock(&block) - require.NoError(t, err) - - // Test 2: transaction results properly serialized - txnRows, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) - for row := range txnRows { - var rowTxn *transactions.SignedTxnWithAD - if row.Txn != nil { - rowTxn = row.Txn - } else { - rowTxn = row.RootTxn - } - require.NoError(t, row.Error) - require.NotNil(t, rowTxn) - require.Equal(t, testcase.Logs, rowTxn.ApplyData.EvalDelta.Logs) - } - }) - } -} - -// Test that LoadGenesis writes account totals. -func TestLoadGenesisAccountTotals(t *testing.T) { - _, connStr, shutdownFunc := pgtest.SetupPostgres(t) - defer shutdownFunc() - db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) - require.NoError(t, err) - defer db.Close() - - err = db.LoadGenesis(test.MakeGenesis()) - require.NoError(t, err) - - json, err := db.getMetastate(context.Background(), nil, schema.AccountTotals) - require.NoError(t, err) - - ret, err := encoding.DecodeAccountTotals([]byte(json)) - require.NoError(t, err) - - assert.Equal( - t, basics.MicroAlgos{Raw: 4 * 1000 * 1000 * 1000 * 1000}, ret.Offline.Money) -} - -func TestTxnAssetID(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - assetid := uint64(1) - createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) - configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) - appid := uint64(3) - createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) - destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) - - block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, - &createAppTxn, &destroyAppTxn) - require.NoError(t, err) - - err = db.AddBlock(&block) - require.NoError(t, err) - - txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) - for i := 0; i < 2; i++ { - row, ok := <-txnRowsCh - require.True(t, ok) - require.NoError(t, row.Error) - assert.Equal(t, assetid, row.AssetID) - } - for i := 0; i < 2; i++ { - row, ok := <-txnRowsCh - require.True(t, ok) - require.NoError(t, row.Error) - assert.Equal(t, appid, row.AssetID) - } -} - -func TestBadTxnJsonEncoding(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - // Need to import a block header because the transactions query joins on it. - block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) - - rootTxid := "abc" - rootIntra := uint(4) - badJSON := `{"aaaaaaaa": 0}` - - query := `INSERT INTO txn (round, intra, typeenum, asset, txid, txn, extra) - VALUES (1, $1, 0, 0, $2, $3, $4)` - - _, err = db.db.Exec( - context.Background(), query, rootIntra, rootTxid, badJSON, - encoding.EncodeTxnExtra(&idb.TxnExtra{})) - require.NoError(t, err) - - { - extra := idb.TxnExtra{ - RootIntra: idb.OptionalUint{Present: true, Value: rootIntra}, - RootTxid: rootTxid, - } - _, err = db.db.Exec( - context.Background(), query, rootIntra+1, nil, badJSON, - encoding.EncodeTxnExtra(&extra)) - require.NoError(t, err) - } - - { - offset := uint64(rootIntra) - tf := idb.TransactionFilter{ - Offset: &offset, - } - rowsCh, _ := db.Transactions(context.Background(), tf) - - row, ok := <-rowsCh - require.True(t, ok) - - require.Error(t, row.Error) - assert.Contains(t, row.Error.Error(), "error decoding txn") - } - - { - offset := uint64(rootIntra) + 1 - tf := idb.TransactionFilter{ - Offset: &offset, - } - rowsCh, _ := db.Transactions(context.Background(), tf) - - row, ok := <-rowsCh - require.True(t, ok) - - require.Error(t, row.Error) - assert.Contains(t, row.Error.Error(), "error decoding roottxn") - } -} - -func TestKeytypeDoNotResetReceiver(t *testing.T) { - block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) - defer shutdownFunc() - - assertKeytype(t, db, test.AccountA, nil) - - // Sigtype of account B becomes "sig". - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) - block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) - - // Sigtype of account A becomes "sig" and B remains the same. - txn = test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) - block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) - - keytype := "sig" - assertKeytype(t, db, test.AccountA, &keytype) - assertKeytype(t, db, test.AccountB, &keytype) -} - -// Test that if information in `txn` and `txn_participation` tables is ahead of -// the current round, AddBlock() still runs successfully. -func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { - block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) - defer shutdownFunc() - - { - query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) - VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` - _, err := db.db.Exec(context.Background(), query) - require.NoError(t, err) - } - { - query := `INSERT INTO txn_participation (addr, round, intra) - VALUES ($1, 1, 0)` - _, err := db.db.Exec(context.Background(), query, test.AccountA[:]) - require.NoError(t, err) - } - - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) - block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) -} - -// Test that AddBlock() writes to `txn_participation` table. -func TestAddBlockTxnParticipationAdded(t *testing.T) { - block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) - defer shutdownFunc() - - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) - block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) - - tf := idb.TransactionFilter{ - Address: test.AccountA[:], - } - rowsCh, _ := db.Transactions(context.Background(), tf) - - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - require.NotNil(t, row.Txn) - assert.Equal(t, txn, *row.Txn) -} - -// Test that if information in the `txn` table is ahead of the current round, -// Transactions() doesn't return the rows ahead of the state. -func TestTransactionsTxnAhead(t *testing.T) { - block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) - defer shutdownFunc() - - // Insert a transaction row at round 1 and check that Transactions() does not return - // it. - { - query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) - VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` - _, err := db.db.Exec(context.Background(), query) - require.NoError(t, err) - } - { - rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) - _, ok := <-rowsCh - assert.False(t, ok) - } - - // Now add an empty round 1 block, and verify that Transactions() returns the - // fake transaction. - { - block, err := test.MakeBlockForTxns(block.BlockHeader) - require.NoError(t, err) - err = db.AddBlock(&block) - require.NoError(t, err) - } - { - rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) - row, ok := <-rowsCh - require.True(t, ok) - require.NoError(t, row.Error) - } -} - -// Test that if genesis hash is different from what is in db metastate -// indexer does not start. -func TestGenesisHashCheckAtDBSetup(t *testing.T) { - _, connStr, shutdownFunc := pgtest.SetupPostgres(t) - defer shutdownFunc() - genesis := test.MakeGenesis() - db := setupIdbWithConnectionString( - t, connStr, genesis, test.MakeGenesisBlock()) - defer db.Close() - genesisHash := crypto.HashObj(genesis) - network, err := db.getMetastate(context.Background(), nil, schema.NetworkMetaStateKey) - assert.NoError(t, err) - networkState, err := encoding.DecodeNetworkState([]byte(network)) - assert.NoError(t, err) - assert.Equal(t, genesisHash, networkState.GenesisHash) - // connect with different genesis configs - genesis.Network = "testnest" - // different genesisHash, should fail - idb, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) - assert.NoError(t, err) - err = idb.LoadGenesis(genesis) - assert.Error(t, err) - assert.Contains(t, err.Error(), "genesis hash not matching") -} - -type ImportState struct { - NextRoundToAccount uint64 `codec:"next_account_round"` -} - -// Test that if genesis hash at initial import is different from what is in db metastate -// indexer does not start. -func TestGenesisHashCheckAtInitialImport(t *testing.T) { - _, connStr, shutdownFunc := pgtest.SetupPostgres(t) - defer shutdownFunc() - genesis := test.MakeGenesis() - db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) - require.NoError(t, err) - defer db.Close() - // test db upgrade - // set next round to account - state := ImportState{NextRoundToAccount: 1} - var buf []byte - jsonCodecHandle := new(codec.JsonHandle) - enc := codec.NewEncoderBytes(&buf, jsonCodecHandle) - enc.MustEncode(state) - db.setMetastate(nil, schema.StateMetastateKey, string(buf)) - // network state not initialized - networkState, err := db.getNetworkState(context.Background(), nil) - require.ErrorIs(t, err, idb.ErrorNotInitialized) - logger := logrus.New() - genesisReader := bytes.NewReader(protocol.EncodeJSON(genesis)) - imported, err := importer.EnsureInitialImport(db, genesisReader, logger) - require.NoError(t, err) - require.True(t, true, imported) - // network state should be set - networkState, err = db.getNetworkState(context.Background(), nil) - require.NoError(t, err) - require.Equal(t, networkState.GenesisHash, crypto.HashObj(genesis)) - - // change genesis value - genesis.Network = "testnest" - genesisReader = bytes.NewReader(protocol.EncodeJSON(genesis)) - // different genesisHash, should fail - _, err = importer.EnsureInitialImport(db, genesisReader, logger) - require.Error(t, err) - require.Contains(t, err.Error(), "genesis hash not matching") - -} - -func getResults(ctx context.Context, rows <-chan idb.AccountRow) (result []idb.AccountRow) { - for { - select { - case row, ok := <-rows: - if !ok { - return - } - result = append(result, row) - case <-ctx.Done(): - return - } - } -} - -func TestIndexerDb_GetAccounts(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) - defer shutdownFunc() - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - testcases := []struct { - gt *uint64 - lt *uint64 - id uint64 - }{ - { - gt: uint64Ptr(1), - lt: nil, - }, - { - gt: nil, - lt: uint64Ptr(1), - }, - { - gt: uint64Ptr(1), - lt: uint64Ptr(1), - }, - { - gt: uint64Ptr(1), - lt: nil, - id: 1, - }, - { - gt: nil, - lt: uint64Ptr(1), - id: 1, - }, - { - gt: uint64Ptr(1), - lt: uint64Ptr(1), - id: 1, - }, - } - - for i, testcase := range testcases { - t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { - resultCh, count := db.GetAccounts(ctx, idb.AccountQueryOptions{ - AssetGT: testcase.gt, - AssetLT: testcase.lt, - HasAssetID: testcase.id, - }) - assert.Equal(t, uint64(0), count) - results := getResults(ctx, resultCh) - - if testcase.id == 0 { - // When the id is 0, there should be an error - assert.NotNil(t, results) - assert.Len(t, results, 1) - assert.Error(t, results[0].Error) - expected := fmt.Sprintf("AssetGT=%d, AssetLT=%d, but HasAssetID=0", uintOrDefault(testcase.gt), uintOrDefault(testcase.lt)) - assert.Equal(t, expected, results[0].Error.Error()) - } else { - // Otherwise, the empty DB should simply return no results. - assert.Nil(t, results) - } - }) - } -} +//// Test that AddBlock makes a record of an account that gets created and deleted in +//// the same round. +//func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// createTxn := test.MakePaymentTxn( +// 0, 5, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) +// deleteTxn := test.MakePaymentTxn( +// 0, 2, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) +// block, err := test.MakeBlockForTxns( +// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// opts := idb.AccountQueryOptions{ +// EqualToAddress: test.AccountE[:], +// IncludeDeleted: true, +// } +// rowsCh, _ := db.GetAccounts(context.Background(), opts) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// require.NotNil(t, row.Account.Deleted) +// assert.True(t, *row.Account.Deleted) +// require.NotNil(t, row.Account.CreatedAtRound) +// assert.Equal(t, uint64(1), *row.Account.CreatedAtRound) +// require.NotNil(t, row.Account.ClosedAtRound) +// assert.Equal(t, uint64(1), *row.Account.ClosedAtRound) +//} +// +//// Test that AddBlock makes a record of an asset that is created and deleted in +//// the same round. +//func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// assetid := uint64(1) +// createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 0) +// deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 0) +// block, err := test.MakeBlockForTxns( +// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// // Asset global state. +// { +// opts := idb.AssetsQuery{ +// AssetID: assetid, +// IncludeDeleted: true, +// } +// rowsCh, _ := db.Assets(context.Background(), opts) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// require.NotNil(t, row.Deleted) +// assert.True(t, *row.Deleted) +// require.NotNil(t, row.CreatedRound) +// assert.Equal(t, uint64(1), *row.CreatedRound) +// require.NotNil(t, row.ClosedRound) +// assert.Equal(t, uint64(1), *row.ClosedRound) +// } +// +// // Asset local state. +// { +// opts := idb.AssetBalanceQuery{ +// AssetID: assetid, +// IncludeDeleted: true, +// } +// rowsCh, _ := db.AssetBalances(context.Background(), opts) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// require.NotNil(t, row.Deleted) +// assert.True(t, *row.Deleted) +// require.NotNil(t, row.CreatedRound) +// assert.Equal(t, uint64(1), *row.CreatedRound) +// require.NotNil(t, row.ClosedRound) +// assert.Equal(t, uint64(1), *row.ClosedRound) +// } +//} +// +//// Test that AddBlock makes a record of an app that is created and deleted in +//// the same round. +//func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// appid := uint64(1) +// createTxn := test.MakeCreateAppTxn(test.AccountA, 0) +// deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) +// block, err := test.MakeBlockForTxns( +// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// opts := idb.ApplicationQuery{ +// ApplicationID: appid, +// IncludeDeleted: true, +// } +// rowsCh, _ := db.Applications(context.Background(), opts) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// require.NotNil(t, row.Application.Deleted) +// assert.True(t, *row.Application.Deleted) +// require.NotNil(t, row.Application.CreatedAtRound) +// assert.Equal(t, uint64(1), *row.Application.CreatedAtRound) +// require.NotNil(t, row.Application.DeletedAtRound) +// assert.Equal(t, uint64(1), *row.Application.DeletedAtRound) +//} +// +//// Test that AddBlock makes a record of an app that is created and deleted in +//// the same round. +//func TestAddBlockAppOptInOutSameRound(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// appid := uint64(1) +// createTxn := test.MakeCreateAppTxn(test.AccountA, 0) +// optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 0) +// optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB) +// block, err := test.MakeBlockForTxns( +// test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// opts := idb.AccountQueryOptions{ +// EqualToAddress: test.AccountB[:], +// IncludeDeleted: true, +// IncludeAppLocalState: true, +// } +// rowsCh, _ := db.GetAccounts(context.Background(), opts) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// +// require.NotNil(t, row.Account.AppsLocalState) +// require.Equal(t, 1, len(*row.Account.AppsLocalState)) +// +// localState := (*row.Account.AppsLocalState)[0] +// require.NotNil(t, localState.Deleted) +// assert.True(t, *localState.Deleted) +// require.NotNil(t, localState.OptedInAtRound) +// assert.Equal(t, uint64(1), *localState.OptedInAtRound) +// require.NotNil(t, localState.ClosedOutAtRound) +// assert.Equal(t, uint64(1), *localState.ClosedOutAtRound) +// require.Equal(t, uint64(0), row.Account.TotalAppsOptedIn) +// +// q := idb.ApplicationQuery{ +// ApplicationID: appid, +// IncludeDeleted: true, +// } +// lsRows, _ := db.AppLocalState(context.Background(), q) +// lsRow, ok := <-lsRows +// require.True(t, ok) +// require.NoError(t, lsRow.Error) +// ls := lsRow.AppLocalState +// require.Equal(t, appid, ls.Id) +// require.NotNil(t, ls.Deleted) +// assert.True(t, *ls.Deleted) +// require.NotNil(t, ls.OptedInAtRound) +// assert.Equal(t, uint64(1), *ls.OptedInAtRound) +// require.NotNil(t, ls.ClosedOutAtRound) +// assert.Equal(t, uint64(1), *ls.ClosedOutAtRound) +//} +// +//// TestSearchForInnerTransactionReturnsRootTransaction checks that the parent +//// transaction is returned when matching on inner transactions if the +//// ReturnInnerTxnFlag is false. If the ReturnInnerTxnFlag is true, it should +//// return the inner txn instead. +//func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { +// var appAddr basics.Address +// appAddr[1] = 99 +// +// tests := []struct { +// name string +// matches int +// returnInner bool +// filter idb.TransactionFilter +// }{ +// { +// name: "match on root, inner, and inner-inners, return root", +// matches: 3, +// returnInner: false, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication}, +// }, +// { +// name: "match on inner, return root", +// matches: 1, +// returnInner: false, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay}, +// }, +// { +// name: "match on inner-inner, return root", +// matches: 1, +// returnInner: false, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer}, +// }, +// { +// name: "match all, return root", +// matches: 5, +// returnInner: false, +// filter: idb.TransactionFilter{Address: appAddr[:]}, +// }, +// { +// name: "match on root, inner, and inner-inners, return inners", +// matches: 3, +// returnInner: true, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication, ReturnInnerTxnOnly: true}, +// }, +// { +// name: "match on inner, return inners", +// matches: 1, +// returnInner: true, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay, ReturnInnerTxnOnly: true}, +// }, +// { +// name: "match on inner-inner, return inners", +// matches: 1, +// returnInner: true, +// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer, ReturnInnerTxnOnly: true}, +// }, +// { +// name: "match all, return inners", +// matches: 5, +// returnInner: true, +// filter: idb.TransactionFilter{Address: appAddr[:], ReturnInnerTxnOnly: true}, +// }, +// } +// +// // Given: A DB with one transaction containing inner transactions [app -> pay -> xfer] +// pdb, connStr, shutdownFunc := pgtest.SetupPostgres(t) +// defer shutdownFunc() +// db := setupIdbWithConnectionString( +// t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer db.Close() +// +// appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) +// require.NoError(t, err) +// rootTxid := appCall.Txn.ID() +// +// err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { +// return db.AddBlock(&block) +// }, nil) +// require.NoError(t, err) +// +// for _, tc := range tests { +// t.Run(tc.name, func(t *testing.T) { +// // When: searching for a transaction that matches part of the transaction. +// results, _ := db.Transactions(context.Background(), tc.filter) +// +// // Then: only the root transaction should be returned if the ReturnInnerTxnOnly flag is true. +// // Else if ReturnInnerTxnOnly is false, then the inner txn should be returned. +// num := 0 +// for result := range results { +// num++ +// require.NoError(t, result.Error) +// +// if tc.returnInner { +// // Make sure that only the inner txn is returned +// require.True(t, (result.Txn != nil) && (result.RootTxn == nil)) +// } else { +// // Make sure the root txn is returned. +// var stxn *transactions.SignedTxnWithAD +// +// // Exactly one of Txn and RootTxn must be present. +// require.True(t, (result.Txn == nil) != (result.RootTxn == nil)) +// +// // Get Txn or RootTxn +// if result.Txn != nil { +// stxn = result.Txn +// } +// if result.RootTxn != nil { +// stxn = result.RootTxn +// } +// require.Equal(t, rootTxid, stxn.Txn.ID()) +// } +// } +// +// // There can be multiple matches because deduplication happens in REST API. +// require.Equal(t, tc.matches, num) +// }) +// } +//} +// +//// TestNonUTF8Logs makes sure we're able to import cheeky logs +//// for both the root and inner transactions. +//func TestNonUTF8Logs(t *testing.T) { +// tests := []struct { +// Name string +// Logs []string +// }{ +// { +// Name: "Normal", +// Logs: []string{"Test log1", "Test log2", "Test log3"}, +// }, +// { +// Name: "Embedded Null", +// Logs: []string{"\000", "\x00\x00\x00\x00\x00\x00\x00\x00", string([]byte{00, 00})}, +// }, +// { +// Name: "Invalid UTF8", +// Logs: []string{"\x8c", "\xff", "\xf8"}, +// }, +// { +// Name: "Emoji", +// Logs: []string{"💩", "💰", "🌐"}, +// }, +// } +// +// for _, testcase := range tests { +// testcase := testcase +// +// t.Run(testcase.Name, func(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) +// createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ +// Logs: testcase.Logs, +// InnerTxns: []transactions.SignedTxnWithAD{ +// // Inner application call with nested cheeky logs +// { +// SignedTxn: transactions.SignedTxn{ +// Txn: transactions.Transaction{ +// Type: protocol.ApplicationCallTx, +// Header: transactions.Header{ +// Sender: test.AccountA, +// }, +// ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ +// ApplicationID: 789, +// OnCompletion: transactions.NoOpOC, +// }, +// }, +// }, +// ApplyData: transactions.ApplyData{ +// EvalDelta: transactions.EvalDelta{ +// Logs: testcase.Logs, +// }, +// }, +// }, +// }, +// } +// +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn) +// require.NoError(t, err) +// +// // Test 1: import/accounting should work. +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// // Test 2: transaction results properly serialized +// txnRows, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) +// for row := range txnRows { +// var rowTxn *transactions.SignedTxnWithAD +// if row.Txn != nil { +// rowTxn = row.Txn +// } else { +// rowTxn = row.RootTxn +// } +// require.NoError(t, row.Error) +// require.NotNil(t, rowTxn) +// require.Equal(t, testcase.Logs, rowTxn.ApplyData.EvalDelta.Logs) +// } +// }) +// } +//} +// +//// Test that LoadGenesis writes account totals. +//func TestLoadGenesisAccountTotals(t *testing.T) { +// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) +// defer shutdownFunc() +// db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) +// require.NoError(t, err) +// defer db.Close() +// +// err = db.LoadGenesis(test.MakeGenesis()) +// require.NoError(t, err) +// +// json, err := db.getMetastate(context.Background(), nil, schema.AccountTotals) +// require.NoError(t, err) +// +// ret, err := encoding.DecodeAccountTotals([]byte(json)) +// require.NoError(t, err) +// +// assert.Equal( +// t, basics.MicroAlgos{Raw: 4 * 1000 * 1000 * 1000 * 1000}, ret.Offline.Money) +//} +// +//func TestTxnAssetID(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// assetid := uint64(1) +// createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) +// configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) +// appid := uint64(3) +// createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) +// destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) +// +// block, err := test.MakeBlockForTxns( +// test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, +// &createAppTxn, &destroyAppTxn) +// require.NoError(t, err) +// +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) +// for i := 0; i < 2; i++ { +// row, ok := <-txnRowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// assert.Equal(t, assetid, row.AssetID) +// } +// for i := 0; i < 2; i++ { +// row, ok := <-txnRowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// assert.Equal(t, appid, row.AssetID) +// } +//} +// +//func TestBadTxnJsonEncoding(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// // Need to import a block header because the transactions query joins on it. +// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// rootTxid := "abc" +// rootIntra := uint(4) +// badJSON := `{"aaaaaaaa": 0}` +// +// query := `INSERT INTO txn (round, intra, typeenum, asset, txid, txn, extra) +// VALUES (1, $1, 0, 0, $2, $3, $4)` +// +// _, err = db.db.Exec( +// context.Background(), query, rootIntra, rootTxid, badJSON, +// encoding.EncodeTxnExtra(&idb.TxnExtra{})) +// require.NoError(t, err) +// +// { +// extra := idb.TxnExtra{ +// RootIntra: idb.OptionalUint{Present: true, Value: rootIntra}, +// RootTxid: rootTxid, +// } +// _, err = db.db.Exec( +// context.Background(), query, rootIntra+1, nil, badJSON, +// encoding.EncodeTxnExtra(&extra)) +// require.NoError(t, err) +// } +// +// { +// offset := uint64(rootIntra) +// tf := idb.TransactionFilter{ +// Offset: &offset, +// } +// rowsCh, _ := db.Transactions(context.Background(), tf) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// +// require.Error(t, row.Error) +// assert.Contains(t, row.Error.Error(), "error decoding txn") +// } +// +// { +// offset := uint64(rootIntra) + 1 +// tf := idb.TransactionFilter{ +// Offset: &offset, +// } +// rowsCh, _ := db.Transactions(context.Background(), tf) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// +// require.Error(t, row.Error) +// assert.Contains(t, row.Error.Error(), "error decoding roottxn") +// } +//} +// +//func TestKeytypeDoNotResetReceiver(t *testing.T) { +// block := test.MakeGenesisBlock() +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) +// defer shutdownFunc() +// +// assertKeytype(t, db, test.AccountA, nil) +// +// // Sigtype of account B becomes "sig". +// txn := test.MakePaymentTxn( +// 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) +// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// // Sigtype of account A becomes "sig" and B remains the same. +// txn = test.MakePaymentTxn( +// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) +// block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// keytype := "sig" +// assertKeytype(t, db, test.AccountA, &keytype) +// assertKeytype(t, db, test.AccountB, &keytype) +//} +// +//// Test that if information in `txn` and `txn_participation` tables is ahead of +//// the current round, AddBlock() still runs successfully. +//func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { +// block := test.MakeGenesisBlock() +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) +// defer shutdownFunc() +// +// { +// query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) +// VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` +// _, err := db.db.Exec(context.Background(), query) +// require.NoError(t, err) +// } +// { +// query := `INSERT INTO txn_participation (addr, round, intra) +// VALUES ($1, 1, 0)` +// _, err := db.db.Exec(context.Background(), query, test.AccountA[:]) +// require.NoError(t, err) +// } +// +// txn := test.MakePaymentTxn( +// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) +// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +//} +// +//// Test that AddBlock() writes to `txn_participation` table. +//func TestAddBlockTxnParticipationAdded(t *testing.T) { +// block := test.MakeGenesisBlock() +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) +// defer shutdownFunc() +// +// txn := test.MakePaymentTxn( +// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) +// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +// +// tf := idb.TransactionFilter{ +// Address: test.AccountA[:], +// } +// rowsCh, _ := db.Transactions(context.Background(), tf) +// +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// require.NotNil(t, row.Txn) +// assert.Equal(t, txn, *row.Txn) +//} +// +//// Test that if information in the `txn` table is ahead of the current round, +//// Transactions() doesn't return the rows ahead of the state. +//func TestTransactionsTxnAhead(t *testing.T) { +// block := test.MakeGenesisBlock() +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) +// defer shutdownFunc() +// +// // Insert a transaction row at round 1 and check that Transactions() does not return +// // it. +// { +// query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) +// VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` +// _, err := db.db.Exec(context.Background(), query) +// require.NoError(t, err) +// } +// { +// rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) +// _, ok := <-rowsCh +// assert.False(t, ok) +// } +// +// // Now add an empty round 1 block, and verify that Transactions() returns the +// // fake transaction. +// { +// block, err := test.MakeBlockForTxns(block.BlockHeader) +// require.NoError(t, err) +// err = db.AddBlock(&block) +// require.NoError(t, err) +// } +// { +// rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) +// row, ok := <-rowsCh +// require.True(t, ok) +// require.NoError(t, row.Error) +// } +//} +// +//// Test that if genesis hash is different from what is in db metastate +//// indexer does not start. +//func TestGenesisHashCheckAtDBSetup(t *testing.T) { +// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) +// defer shutdownFunc() +// genesis := test.MakeGenesis() +// db := setupIdbWithConnectionString( +// t, connStr, genesis, test.MakeGenesisBlock()) +// defer db.Close() +// genesisHash := crypto.HashObj(genesis) +// network, err := db.getMetastate(context.Background(), nil, schema.NetworkMetaStateKey) +// assert.NoError(t, err) +// networkState, err := encoding.DecodeNetworkState([]byte(network)) +// assert.NoError(t, err) +// assert.Equal(t, genesisHash, networkState.GenesisHash) +// // connect with different genesis configs +// genesis.Network = "testnest" +// // different genesisHash, should fail +// idb, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) +// assert.NoError(t, err) +// err = idb.LoadGenesis(genesis) +// assert.Error(t, err) +// assert.Contains(t, err.Error(), "genesis hash not matching") +//} +// +//type ImportState struct { +// NextRoundToAccount uint64 `codec:"next_account_round"` +//} +// +//// Test that if genesis hash at initial import is different from what is in db metastate +//// indexer does not start. +//func TestGenesisHashCheckAtInitialImport(t *testing.T) { +// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) +// defer shutdownFunc() +// genesis := test.MakeGenesis() +// db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) +// require.NoError(t, err) +// defer db.Close() +// // test db upgrade +// // set next round to account +// state := ImportState{NextRoundToAccount: 1} +// var buf []byte +// jsonCodecHandle := new(codec.JsonHandle) +// enc := codec.NewEncoderBytes(&buf, jsonCodecHandle) +// enc.MustEncode(state) +// db.setMetastate(nil, schema.StateMetastateKey, string(buf)) +// // network state not initialized +// networkState, err := db.getNetworkState(context.Background(), nil) +// require.ErrorIs(t, err, idb.ErrorNotInitialized) +// logger := logrus.New() +// genesisReader := bytes.NewReader(protocol.EncodeJSON(genesis)) +// imported, err := importer.EnsureInitialImport(db, genesisReader, logger) +// require.NoError(t, err) +// require.True(t, true, imported) +// // network state should be set +// networkState, err = db.getNetworkState(context.Background(), nil) +// require.NoError(t, err) +// require.Equal(t, networkState.GenesisHash, crypto.HashObj(genesis)) +// +// // change genesis value +// genesis.Network = "testnest" +// genesisReader = bytes.NewReader(protocol.EncodeJSON(genesis)) +// // different genesisHash, should fail +// _, err = importer.EnsureInitialImport(db, genesisReader, logger) +// require.Error(t, err) +// require.Contains(t, err.Error(), "genesis hash not matching") +// +//} +// +//func getResults(ctx context.Context, rows <-chan idb.AccountRow) (result []idb.AccountRow) { +// for { +// select { +// case row, ok := <-rows: +// if !ok { +// return +// } +// result = append(result, row) +// case <-ctx.Done(): +// return +// } +// } +//} +// +//func TestIndexerDb_GetAccounts(t *testing.T) { +// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) +// defer shutdownFunc() +// +// ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) +// defer cancel() +// +// testcases := []struct { +// gt *uint64 +// lt *uint64 +// id uint64 +// }{ +// { +// gt: uint64Ptr(1), +// lt: nil, +// }, +// { +// gt: nil, +// lt: uint64Ptr(1), +// }, +// { +// gt: uint64Ptr(1), +// lt: uint64Ptr(1), +// }, +// { +// gt: uint64Ptr(1), +// lt: nil, +// id: 1, +// }, +// { +// gt: nil, +// lt: uint64Ptr(1), +// id: 1, +// }, +// { +// gt: uint64Ptr(1), +// lt: uint64Ptr(1), +// id: 1, +// }, +// } +// +// for i, testcase := range testcases { +// t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { +// resultCh, count := db.GetAccounts(ctx, idb.AccountQueryOptions{ +// AssetGT: testcase.gt, +// AssetLT: testcase.lt, +// HasAssetID: testcase.id, +// }) +// assert.Equal(t, uint64(0), count) +// results := getResults(ctx, resultCh) +// +// if testcase.id == 0 { +// // When the id is 0, there should be an error +// assert.NotNil(t, results) +// assert.Len(t, results, 1) +// assert.Error(t, results[0].Error) +// expected := fmt.Sprintf("AssetGT=%d, AssetLT=%d, but HasAssetID=0", uintOrDefault(testcase.gt), uintOrDefault(testcase.lt)) +// assert.Equal(t, expected, results[0].Error.Error()) +// } else { +// // Otherwise, the empty DB should simply return no results. +// assert.Nil(t, results) +// } +// }) +// } +//} diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 5bf3b8858..446d78194 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -47,6 +47,7 @@ func (processor *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error if err != nil { return fmt.Errorf("Process() block eval err: %w", err) } + _, _, err = blkeval.ProcessBlockForIndexer(&blockCert.Block) if err != nil { return fmt.Errorf("%w", err) diff --git a/processor/blockprocessor/block_processor_test.go b/processor/blockprocessor/block_processor_test.go index 34531f551..3f2bac77f 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/processor/blockprocessor/block_processor_test.go @@ -31,7 +31,7 @@ func TestProcess(t *testing.T) { // create a few rounds for i := 1; i <= 3; i++ { - txn := test.MakePaymentTxn(0, uint64(i), 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn(0, uint64(i), 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err := test.MakeBlockForTxns(prevHeader, &txn) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} @@ -62,7 +62,7 @@ func TestFailedProcess(t *testing.T) { genesisBlock, err := l.Block(basics.Round(0)) assert.Nil(t, err) // incorrect round - txn := test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA) + txn := test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA, 0) block, err := test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) block.BlockHeader.Round = 10 assert.Nil(t, err) @@ -71,7 +71,7 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "Process() invalid round blockCert.Block.Round()") // non-zero balance after close remainder to sender address - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA, 0) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} @@ -79,7 +79,7 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // stxn GenesisID not empty - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) block.Payset[0].Txn.GenesisID = "genesisID" @@ -88,7 +88,7 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // eval error: concensus protocol not supported - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) block.BlockHeader.CurrentProtocol = "testing" assert.Nil(t, err) @@ -103,7 +103,7 @@ func TestFailedProcess(t *testing.T) { _, err = block_processor.MakeProcessor(l, handler) assert.Contains(t, err.Error(), "MakeProcessor() handler err") pr, _ = block_processor.MakeProcessor(l, nil) - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) pr.SetHandler(handler) diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index 43ac75117..aa0e7a331 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -83,7 +83,7 @@ func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, un } // MakeAssetFreezeTxn create an asset freeze/unfreeze transaction. -func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basics.Address) transactions.SignedTxnWithAD { +func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -92,6 +92,7 @@ func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basic Sender: sender, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, AssetFreezeTxnFields: transactions.AssetFreezeTxnFields{ FreezeAccount: freezeAccount, @@ -156,8 +157,7 @@ func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) trans } // MakePaymentTxn creates an algo transfer transaction. -func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, - closeRewards uint64, sender, receiver, close, rekeyTo basics.Address) transactions.SignedTxnWithAD { +func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeRewards uint64, sender, receiver, close, rekeyTo basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -167,7 +167,7 @@ func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, Fee: basics.MicroAlgos{Raw: fee}, GenesisHash: GenesisHash, RekeyTo: rekeyTo, - LastValid: 10, + LastValid: basics.Round(lv), }, PaymentTxnFields: transactions.PaymentTxnFields{ Receiver: receiver, @@ -540,7 +540,6 @@ func MakeBlockForTxns(prevHeader bookkeeping.BlockHeader, inputs ...*transaction res.Payset = append(res.Payset, stxnib) } - return res, nil } From c0726b9752379b07a741a863a8362a20d64d4b85 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 19 May 2022 10:52:36 -0400 Subject: [PATCH 08/37] more test updates --- idb/postgres/postgres_integration_test.go | 1601 +++++++++++---------- util/test/account_testutil.go | 3 +- 2 files changed, 836 insertions(+), 768 deletions(-) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index 3d26d2f84..b4fb46bc2 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -1,12 +1,14 @@ package postgres import ( + "bytes" "context" "database/sql" "fmt" "math" "sync" "testing" + "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -16,8 +18,12 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-codec/codec" "github.com/algorand/indexer/api/generated/v2" "github.com/algorand/indexer/idb/postgres/internal/encoding" + "github.com/algorand/indexer/idb/postgres/internal/schema" + pgutil "github.com/algorand/indexer/idb/postgres/internal/util" + "github.com/algorand/indexer/importer" "github.com/algorand/indexer/processor/blockprocessor" "github.com/algorand/indexer/util" "github.com/jackc/pgx/v4" @@ -1497,770 +1503,831 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { assert.Equal(t, uint64(3), round) } -//// Test that AddBlock makes a record of an account that gets created and deleted in -//// the same round. -//func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// createTxn := test.MakePaymentTxn( -// 0, 5, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) -// deleteTxn := test.MakePaymentTxn( -// 0, 2, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) -// block, err := test.MakeBlockForTxns( -// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// opts := idb.AccountQueryOptions{ -// EqualToAddress: test.AccountE[:], -// IncludeDeleted: true, -// } -// rowsCh, _ := db.GetAccounts(context.Background(), opts) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// require.NotNil(t, row.Account.Deleted) -// assert.True(t, *row.Account.Deleted) -// require.NotNil(t, row.Account.CreatedAtRound) -// assert.Equal(t, uint64(1), *row.Account.CreatedAtRound) -// require.NotNil(t, row.Account.ClosedAtRound) -// assert.Equal(t, uint64(1), *row.Account.ClosedAtRound) -//} -// -//// Test that AddBlock makes a record of an asset that is created and deleted in -//// the same round. -//func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// assetid := uint64(1) -// createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 0) -// deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 0) -// block, err := test.MakeBlockForTxns( -// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// // Asset global state. -// { -// opts := idb.AssetsQuery{ -// AssetID: assetid, -// IncludeDeleted: true, -// } -// rowsCh, _ := db.Assets(context.Background(), opts) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// require.NotNil(t, row.Deleted) -// assert.True(t, *row.Deleted) -// require.NotNil(t, row.CreatedRound) -// assert.Equal(t, uint64(1), *row.CreatedRound) -// require.NotNil(t, row.ClosedRound) -// assert.Equal(t, uint64(1), *row.ClosedRound) -// } -// -// // Asset local state. -// { -// opts := idb.AssetBalanceQuery{ -// AssetID: assetid, -// IncludeDeleted: true, -// } -// rowsCh, _ := db.AssetBalances(context.Background(), opts) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// require.NotNil(t, row.Deleted) -// assert.True(t, *row.Deleted) -// require.NotNil(t, row.CreatedRound) -// assert.Equal(t, uint64(1), *row.CreatedRound) -// require.NotNil(t, row.ClosedRound) -// assert.Equal(t, uint64(1), *row.ClosedRound) -// } -//} -// -//// Test that AddBlock makes a record of an app that is created and deleted in -//// the same round. -//func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// appid := uint64(1) -// createTxn := test.MakeCreateAppTxn(test.AccountA, 0) -// deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) -// block, err := test.MakeBlockForTxns( -// test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// opts := idb.ApplicationQuery{ -// ApplicationID: appid, -// IncludeDeleted: true, -// } -// rowsCh, _ := db.Applications(context.Background(), opts) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// require.NotNil(t, row.Application.Deleted) -// assert.True(t, *row.Application.Deleted) -// require.NotNil(t, row.Application.CreatedAtRound) -// assert.Equal(t, uint64(1), *row.Application.CreatedAtRound) -// require.NotNil(t, row.Application.DeletedAtRound) -// assert.Equal(t, uint64(1), *row.Application.DeletedAtRound) -//} -// -//// Test that AddBlock makes a record of an app that is created and deleted in -//// the same round. -//func TestAddBlockAppOptInOutSameRound(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// appid := uint64(1) -// createTxn := test.MakeCreateAppTxn(test.AccountA, 0) -// optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 0) -// optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB) -// block, err := test.MakeBlockForTxns( -// test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// opts := idb.AccountQueryOptions{ -// EqualToAddress: test.AccountB[:], -// IncludeDeleted: true, -// IncludeAppLocalState: true, -// } -// rowsCh, _ := db.GetAccounts(context.Background(), opts) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// -// require.NotNil(t, row.Account.AppsLocalState) -// require.Equal(t, 1, len(*row.Account.AppsLocalState)) -// -// localState := (*row.Account.AppsLocalState)[0] -// require.NotNil(t, localState.Deleted) -// assert.True(t, *localState.Deleted) -// require.NotNil(t, localState.OptedInAtRound) -// assert.Equal(t, uint64(1), *localState.OptedInAtRound) -// require.NotNil(t, localState.ClosedOutAtRound) -// assert.Equal(t, uint64(1), *localState.ClosedOutAtRound) -// require.Equal(t, uint64(0), row.Account.TotalAppsOptedIn) -// -// q := idb.ApplicationQuery{ -// ApplicationID: appid, -// IncludeDeleted: true, -// } -// lsRows, _ := db.AppLocalState(context.Background(), q) -// lsRow, ok := <-lsRows -// require.True(t, ok) -// require.NoError(t, lsRow.Error) -// ls := lsRow.AppLocalState -// require.Equal(t, appid, ls.Id) -// require.NotNil(t, ls.Deleted) -// assert.True(t, *ls.Deleted) -// require.NotNil(t, ls.OptedInAtRound) -// assert.Equal(t, uint64(1), *ls.OptedInAtRound) -// require.NotNil(t, ls.ClosedOutAtRound) -// assert.Equal(t, uint64(1), *ls.ClosedOutAtRound) -//} -// -//// TestSearchForInnerTransactionReturnsRootTransaction checks that the parent -//// transaction is returned when matching on inner transactions if the -//// ReturnInnerTxnFlag is false. If the ReturnInnerTxnFlag is true, it should -//// return the inner txn instead. -//func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { -// var appAddr basics.Address -// appAddr[1] = 99 -// -// tests := []struct { -// name string -// matches int -// returnInner bool -// filter idb.TransactionFilter -// }{ -// { -// name: "match on root, inner, and inner-inners, return root", -// matches: 3, -// returnInner: false, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication}, -// }, -// { -// name: "match on inner, return root", -// matches: 1, -// returnInner: false, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay}, -// }, -// { -// name: "match on inner-inner, return root", -// matches: 1, -// returnInner: false, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer}, -// }, -// { -// name: "match all, return root", -// matches: 5, -// returnInner: false, -// filter: idb.TransactionFilter{Address: appAddr[:]}, -// }, -// { -// name: "match on root, inner, and inner-inners, return inners", -// matches: 3, -// returnInner: true, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication, ReturnInnerTxnOnly: true}, -// }, -// { -// name: "match on inner, return inners", -// matches: 1, -// returnInner: true, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay, ReturnInnerTxnOnly: true}, -// }, -// { -// name: "match on inner-inner, return inners", -// matches: 1, -// returnInner: true, -// filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer, ReturnInnerTxnOnly: true}, -// }, -// { -// name: "match all, return inners", -// matches: 5, -// returnInner: true, -// filter: idb.TransactionFilter{Address: appAddr[:], ReturnInnerTxnOnly: true}, -// }, -// } -// -// // Given: A DB with one transaction containing inner transactions [app -> pay -> xfer] -// pdb, connStr, shutdownFunc := pgtest.SetupPostgres(t) -// defer shutdownFunc() -// db := setupIdbWithConnectionString( -// t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer db.Close() -// -// appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) -// require.NoError(t, err) -// rootTxid := appCall.Txn.ID() -// -// err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { -// return db.AddBlock(&block) -// }, nil) -// require.NoError(t, err) -// -// for _, tc := range tests { -// t.Run(tc.name, func(t *testing.T) { -// // When: searching for a transaction that matches part of the transaction. -// results, _ := db.Transactions(context.Background(), tc.filter) -// -// // Then: only the root transaction should be returned if the ReturnInnerTxnOnly flag is true. -// // Else if ReturnInnerTxnOnly is false, then the inner txn should be returned. -// num := 0 -// for result := range results { -// num++ -// require.NoError(t, result.Error) -// -// if tc.returnInner { -// // Make sure that only the inner txn is returned -// require.True(t, (result.Txn != nil) && (result.RootTxn == nil)) -// } else { -// // Make sure the root txn is returned. -// var stxn *transactions.SignedTxnWithAD -// -// // Exactly one of Txn and RootTxn must be present. -// require.True(t, (result.Txn == nil) != (result.RootTxn == nil)) -// -// // Get Txn or RootTxn -// if result.Txn != nil { -// stxn = result.Txn -// } -// if result.RootTxn != nil { -// stxn = result.RootTxn -// } -// require.Equal(t, rootTxid, stxn.Txn.ID()) -// } -// } -// -// // There can be multiple matches because deduplication happens in REST API. -// require.Equal(t, tc.matches, num) -// }) -// } -//} -// -//// TestNonUTF8Logs makes sure we're able to import cheeky logs -//// for both the root and inner transactions. -//func TestNonUTF8Logs(t *testing.T) { -// tests := []struct { -// Name string -// Logs []string -// }{ -// { -// Name: "Normal", -// Logs: []string{"Test log1", "Test log2", "Test log3"}, -// }, -// { -// Name: "Embedded Null", -// Logs: []string{"\000", "\x00\x00\x00\x00\x00\x00\x00\x00", string([]byte{00, 00})}, -// }, -// { -// Name: "Invalid UTF8", -// Logs: []string{"\x8c", "\xff", "\xf8"}, -// }, -// { -// Name: "Emoji", -// Logs: []string{"💩", "💰", "🌐"}, -// }, -// } -// -// for _, testcase := range tests { -// testcase := testcase -// -// t.Run(testcase.Name, func(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) -// createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ -// Logs: testcase.Logs, -// InnerTxns: []transactions.SignedTxnWithAD{ -// // Inner application call with nested cheeky logs -// { -// SignedTxn: transactions.SignedTxn{ -// Txn: transactions.Transaction{ -// Type: protocol.ApplicationCallTx, -// Header: transactions.Header{ -// Sender: test.AccountA, -// }, -// ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ -// ApplicationID: 789, -// OnCompletion: transactions.NoOpOC, -// }, -// }, -// }, -// ApplyData: transactions.ApplyData{ -// EvalDelta: transactions.EvalDelta{ -// Logs: testcase.Logs, -// }, -// }, -// }, -// }, -// } -// -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn) -// require.NoError(t, err) -// -// // Test 1: import/accounting should work. -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// // Test 2: transaction results properly serialized -// txnRows, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) -// for row := range txnRows { -// var rowTxn *transactions.SignedTxnWithAD -// if row.Txn != nil { -// rowTxn = row.Txn -// } else { -// rowTxn = row.RootTxn -// } -// require.NoError(t, row.Error) -// require.NotNil(t, rowTxn) -// require.Equal(t, testcase.Logs, rowTxn.ApplyData.EvalDelta.Logs) -// } -// }) -// } -//} -// -//// Test that LoadGenesis writes account totals. -//func TestLoadGenesisAccountTotals(t *testing.T) { -// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) -// defer shutdownFunc() -// db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) -// require.NoError(t, err) -// defer db.Close() -// -// err = db.LoadGenesis(test.MakeGenesis()) -// require.NoError(t, err) -// -// json, err := db.getMetastate(context.Background(), nil, schema.AccountTotals) -// require.NoError(t, err) -// -// ret, err := encoding.DecodeAccountTotals([]byte(json)) -// require.NoError(t, err) -// -// assert.Equal( -// t, basics.MicroAlgos{Raw: 4 * 1000 * 1000 * 1000 * 1000}, ret.Offline.Money) -//} -// -//func TestTxnAssetID(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// assetid := uint64(1) -// createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) -// configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 0) -// appid := uint64(3) -// createAppTxn := test.MakeCreateAppTxn(test.AccountA, 0) -// destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 0) -// -// block, err := test.MakeBlockForTxns( -// test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, -// &createAppTxn, &destroyAppTxn) -// require.NoError(t, err) -// -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) -// for i := 0; i < 2; i++ { -// row, ok := <-txnRowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// assert.Equal(t, assetid, row.AssetID) -// } -// for i := 0; i < 2; i++ { -// row, ok := <-txnRowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// assert.Equal(t, appid, row.AssetID) -// } -//} -// -//func TestBadTxnJsonEncoding(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// // Need to import a block header because the transactions query joins on it. -// block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// rootTxid := "abc" -// rootIntra := uint(4) -// badJSON := `{"aaaaaaaa": 0}` -// -// query := `INSERT INTO txn (round, intra, typeenum, asset, txid, txn, extra) -// VALUES (1, $1, 0, 0, $2, $3, $4)` -// -// _, err = db.db.Exec( -// context.Background(), query, rootIntra, rootTxid, badJSON, -// encoding.EncodeTxnExtra(&idb.TxnExtra{})) -// require.NoError(t, err) -// -// { -// extra := idb.TxnExtra{ -// RootIntra: idb.OptionalUint{Present: true, Value: rootIntra}, -// RootTxid: rootTxid, -// } -// _, err = db.db.Exec( -// context.Background(), query, rootIntra+1, nil, badJSON, -// encoding.EncodeTxnExtra(&extra)) -// require.NoError(t, err) -// } -// -// { -// offset := uint64(rootIntra) -// tf := idb.TransactionFilter{ -// Offset: &offset, -// } -// rowsCh, _ := db.Transactions(context.Background(), tf) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// -// require.Error(t, row.Error) -// assert.Contains(t, row.Error.Error(), "error decoding txn") -// } -// -// { -// offset := uint64(rootIntra) + 1 -// tf := idb.TransactionFilter{ -// Offset: &offset, -// } -// rowsCh, _ := db.Transactions(context.Background(), tf) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// -// require.Error(t, row.Error) -// assert.Contains(t, row.Error.Error(), "error decoding roottxn") -// } -//} -// -//func TestKeytypeDoNotResetReceiver(t *testing.T) { -// block := test.MakeGenesisBlock() -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) -// defer shutdownFunc() -// -// assertKeytype(t, db, test.AccountA, nil) -// -// // Sigtype of account B becomes "sig". -// txn := test.MakePaymentTxn( -// 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) -// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// // Sigtype of account A becomes "sig" and B remains the same. -// txn = test.MakePaymentTxn( -// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) -// block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// keytype := "sig" -// assertKeytype(t, db, test.AccountA, &keytype) -// assertKeytype(t, db, test.AccountB, &keytype) -//} -// -//// Test that if information in `txn` and `txn_participation` tables is ahead of -//// the current round, AddBlock() still runs successfully. -//func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { -// block := test.MakeGenesisBlock() -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) -// defer shutdownFunc() -// -// { -// query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) -// VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` -// _, err := db.db.Exec(context.Background(), query) -// require.NoError(t, err) -// } -// { -// query := `INSERT INTO txn_participation (addr, round, intra) -// VALUES ($1, 1, 0)` -// _, err := db.db.Exec(context.Background(), query, test.AccountA[:]) -// require.NoError(t, err) -// } -// -// txn := test.MakePaymentTxn( -// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) -// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -//} -// -//// Test that AddBlock() writes to `txn_participation` table. -//func TestAddBlockTxnParticipationAdded(t *testing.T) { -// block := test.MakeGenesisBlock() -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) -// defer shutdownFunc() -// -// txn := test.MakePaymentTxn( -// 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) -// block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -// -// tf := idb.TransactionFilter{ -// Address: test.AccountA[:], -// } -// rowsCh, _ := db.Transactions(context.Background(), tf) -// -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// require.NotNil(t, row.Txn) -// assert.Equal(t, txn, *row.Txn) -//} -// -//// Test that if information in the `txn` table is ahead of the current round, -//// Transactions() doesn't return the rows ahead of the state. -//func TestTransactionsTxnAhead(t *testing.T) { -// block := test.MakeGenesisBlock() -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) -// defer shutdownFunc() -// -// // Insert a transaction row at round 1 and check that Transactions() does not return -// // it. -// { -// query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) -// VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` -// _, err := db.db.Exec(context.Background(), query) -// require.NoError(t, err) -// } -// { -// rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) -// _, ok := <-rowsCh -// assert.False(t, ok) -// } -// -// // Now add an empty round 1 block, and verify that Transactions() returns the -// // fake transaction. -// { -// block, err := test.MakeBlockForTxns(block.BlockHeader) -// require.NoError(t, err) -// err = db.AddBlock(&block) -// require.NoError(t, err) -// } -// { -// rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) -// row, ok := <-rowsCh -// require.True(t, ok) -// require.NoError(t, row.Error) -// } -//} -// -//// Test that if genesis hash is different from what is in db metastate -//// indexer does not start. -//func TestGenesisHashCheckAtDBSetup(t *testing.T) { -// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) -// defer shutdownFunc() -// genesis := test.MakeGenesis() -// db := setupIdbWithConnectionString( -// t, connStr, genesis, test.MakeGenesisBlock()) -// defer db.Close() -// genesisHash := crypto.HashObj(genesis) -// network, err := db.getMetastate(context.Background(), nil, schema.NetworkMetaStateKey) -// assert.NoError(t, err) -// networkState, err := encoding.DecodeNetworkState([]byte(network)) -// assert.NoError(t, err) -// assert.Equal(t, genesisHash, networkState.GenesisHash) -// // connect with different genesis configs -// genesis.Network = "testnest" -// // different genesisHash, should fail -// idb, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) -// assert.NoError(t, err) -// err = idb.LoadGenesis(genesis) -// assert.Error(t, err) -// assert.Contains(t, err.Error(), "genesis hash not matching") -//} -// -//type ImportState struct { -// NextRoundToAccount uint64 `codec:"next_account_round"` -//} -// -//// Test that if genesis hash at initial import is different from what is in db metastate -//// indexer does not start. -//func TestGenesisHashCheckAtInitialImport(t *testing.T) { -// _, connStr, shutdownFunc := pgtest.SetupPostgres(t) -// defer shutdownFunc() -// genesis := test.MakeGenesis() -// db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) -// require.NoError(t, err) -// defer db.Close() -// // test db upgrade -// // set next round to account -// state := ImportState{NextRoundToAccount: 1} -// var buf []byte -// jsonCodecHandle := new(codec.JsonHandle) -// enc := codec.NewEncoderBytes(&buf, jsonCodecHandle) -// enc.MustEncode(state) -// db.setMetastate(nil, schema.StateMetastateKey, string(buf)) -// // network state not initialized -// networkState, err := db.getNetworkState(context.Background(), nil) -// require.ErrorIs(t, err, idb.ErrorNotInitialized) -// logger := logrus.New() -// genesisReader := bytes.NewReader(protocol.EncodeJSON(genesis)) -// imported, err := importer.EnsureInitialImport(db, genesisReader, logger) -// require.NoError(t, err) -// require.True(t, true, imported) -// // network state should be set -// networkState, err = db.getNetworkState(context.Background(), nil) -// require.NoError(t, err) -// require.Equal(t, networkState.GenesisHash, crypto.HashObj(genesis)) -// -// // change genesis value -// genesis.Network = "testnest" -// genesisReader = bytes.NewReader(protocol.EncodeJSON(genesis)) -// // different genesisHash, should fail -// _, err = importer.EnsureInitialImport(db, genesisReader, logger) -// require.Error(t, err) -// require.Contains(t, err.Error(), "genesis hash not matching") -// -//} -// -//func getResults(ctx context.Context, rows <-chan idb.AccountRow) (result []idb.AccountRow) { -// for { -// select { -// case row, ok := <-rows: -// if !ok { -// return -// } -// result = append(result, row) -// case <-ctx.Done(): -// return -// } -// } -//} -// -//func TestIndexerDb_GetAccounts(t *testing.T) { -// db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) -// defer shutdownFunc() -// -// ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) -// defer cancel() -// -// testcases := []struct { -// gt *uint64 -// lt *uint64 -// id uint64 -// }{ -// { -// gt: uint64Ptr(1), -// lt: nil, -// }, -// { -// gt: nil, -// lt: uint64Ptr(1), -// }, -// { -// gt: uint64Ptr(1), -// lt: uint64Ptr(1), -// }, -// { -// gt: uint64Ptr(1), -// lt: nil, -// id: 1, -// }, -// { -// gt: nil, -// lt: uint64Ptr(1), -// id: 1, -// }, -// { -// gt: uint64Ptr(1), -// lt: uint64Ptr(1), -// id: 1, -// }, -// } -// -// for i, testcase := range testcases { -// t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { -// resultCh, count := db.GetAccounts(ctx, idb.AccountQueryOptions{ -// AssetGT: testcase.gt, -// AssetLT: testcase.lt, -// HasAssetID: testcase.id, -// }) -// assert.Equal(t, uint64(0), count) -// results := getResults(ctx, resultCh) -// -// if testcase.id == 0 { -// // When the id is 0, there should be an error -// assert.NotNil(t, results) -// assert.Len(t, results, 1) -// assert.Error(t, results[0].Error) -// expected := fmt.Sprintf("AssetGT=%d, AssetLT=%d, but HasAssetID=0", uintOrDefault(testcase.gt), uintOrDefault(testcase.lt)) -// assert.Equal(t, expected, results[0].Error.Error()) -// } else { -// // Otherwise, the empty DB should simply return no results. -// assert.Nil(t, results) -// } -// }) -// } -//} +// Test that AddBlock makes a record of an account that gets created and deleted in +// the same round. +func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + createTxn := test.MakePaymentTxn( + 0, 100000, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}, 10) + deleteTxn := test.MakePaymentTxn( + 0, 100000, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}, 10) + block, err := test.MakeBlockForTxns( + test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + opts := idb.AccountQueryOptions{ + EqualToAddress: test.AccountE[:], + IncludeDeleted: true, + } + rowsCh, _ := db.GetAccounts(context.Background(), opts) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + require.NotNil(t, row.Account.Deleted) + assert.True(t, *row.Account.Deleted) + require.NotNil(t, row.Account.CreatedAtRound) + assert.Equal(t, uint64(1), *row.Account.CreatedAtRound) + require.NotNil(t, row.Account.ClosedAtRound) + assert.Equal(t, uint64(1), *row.Account.ClosedAtRound) +} + +// Test that AddBlock makes a record of an asset that is created and deleted in +// the same round. +func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + assetid := uint64(1) + createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 10) + deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 10) + block, err := test.MakeBlockForTxns( + test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + // Asset global state. + { + opts := idb.AssetsQuery{ + AssetID: assetid, + IncludeDeleted: true, + } + rowsCh, _ := db.Assets(context.Background(), opts) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + require.NotNil(t, row.Deleted) + assert.True(t, *row.Deleted) + require.NotNil(t, row.CreatedRound) + assert.Equal(t, uint64(1), *row.CreatedRound) + require.NotNil(t, row.ClosedRound) + assert.Equal(t, uint64(1), *row.ClosedRound) + } + + // Asset local state. + { + opts := idb.AssetBalanceQuery{ + AssetID: assetid, + IncludeDeleted: true, + } + rowsCh, _ := db.AssetBalances(context.Background(), opts) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + require.NotNil(t, row.Deleted) + assert.True(t, *row.Deleted) + require.NotNil(t, row.CreatedRound) + assert.Equal(t, uint64(1), *row.CreatedRound) + require.NotNil(t, row.ClosedRound) + assert.Equal(t, uint64(1), *row.ClosedRound) + } +} + +// Test that AddBlock makes a record of an app that is created and deleted in +// the same round. +func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + appid := uint64(1) + createTxn := test.MakeCreateAppTxn(test.AccountA, 10) + deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 10) + block, err := test.MakeBlockForTxns( + test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + opts := idb.ApplicationQuery{ + ApplicationID: appid, + IncludeDeleted: true, + } + rowsCh, _ := db.Applications(context.Background(), opts) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + require.NotNil(t, row.Application.Deleted) + assert.True(t, *row.Application.Deleted) + require.NotNil(t, row.Application.CreatedAtRound) + assert.Equal(t, uint64(1), *row.Application.CreatedAtRound) + require.NotNil(t, row.Application.DeletedAtRound) + assert.Equal(t, uint64(1), *row.Application.DeletedAtRound) +} + +// Test that AddBlock makes a record of an app that is created and deleted in +// the same round. +func TestAddBlockAppOptInOutSameRound(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + appid := uint64(1) + createTxn := test.MakeCreateAppTxn(test.AccountA, 10) + optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 10) + optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB, 10) + block, err := test.MakeBlockForTxns( + test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + opts := idb.AccountQueryOptions{ + EqualToAddress: test.AccountB[:], + IncludeDeleted: true, + IncludeAppLocalState: true, + } + rowsCh, _ := db.GetAccounts(context.Background(), opts) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + + require.NotNil(t, row.Account.AppsLocalState) + require.Equal(t, 1, len(*row.Account.AppsLocalState)) + + localState := (*row.Account.AppsLocalState)[0] + require.NotNil(t, localState.Deleted) + assert.True(t, *localState.Deleted) + require.NotNil(t, localState.OptedInAtRound) + assert.Equal(t, uint64(1), *localState.OptedInAtRound) + require.NotNil(t, localState.ClosedOutAtRound) + assert.Equal(t, uint64(1), *localState.ClosedOutAtRound) + require.Equal(t, uint64(0), row.Account.TotalAppsOptedIn) + + q := idb.ApplicationQuery{ + ApplicationID: appid, + IncludeDeleted: true, + } + lsRows, _ := db.AppLocalState(context.Background(), q) + lsRow, ok := <-lsRows + require.True(t, ok) + require.NoError(t, lsRow.Error) + ls := lsRow.AppLocalState + require.Equal(t, appid, ls.Id) + require.NotNil(t, ls.Deleted) + assert.True(t, *ls.Deleted) + require.NotNil(t, ls.OptedInAtRound) + assert.Equal(t, uint64(1), *ls.OptedInAtRound) + require.NotNil(t, ls.ClosedOutAtRound) + assert.Equal(t, uint64(1), *ls.ClosedOutAtRound) +} + +// TestSearchForInnerTransactionReturnsRootTransaction checks that the parent +// transaction is returned when matching on inner transactions if the +// ReturnInnerTxnFlag is false. If the ReturnInnerTxnFlag is true, it should +// return the inner txn instead. +func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { + var appAddr basics.Address + appAddr[1] = 99 + + tests := []struct { + name string + matches int + returnInner bool + filter idb.TransactionFilter + }{ + { + name: "match on root, inner, and inner-inners, return root", + matches: 3, + returnInner: false, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication}, + }, + { + name: "match on inner, return root", + matches: 1, + returnInner: false, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay}, + }, + { + name: "match on inner-inner, return root", + matches: 1, + returnInner: false, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer}, + }, + { + name: "match all, return root", + matches: 5, + returnInner: false, + filter: idb.TransactionFilter{Address: appAddr[:]}, + }, + { + name: "match on root, inner, and inner-inners, return inners", + matches: 3, + returnInner: true, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumApplication, ReturnInnerTxnOnly: true}, + }, + { + name: "match on inner, return inners", + matches: 1, + returnInner: true, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumPay, ReturnInnerTxnOnly: true}, + }, + { + name: "match on inner-inner, return inners", + matches: 1, + returnInner: true, + filter: idb.TransactionFilter{Address: appAddr[:], TypeEnum: idb.TypeEnumAssetTransfer, ReturnInnerTxnOnly: true}, + }, + { + name: "match all, return inners", + matches: 5, + returnInner: true, + filter: idb.TransactionFilter{Address: appAddr[:], ReturnInnerTxnOnly: true}, + }, + } + + // Given: A DB with one transaction containing inner transactions [app -> pay -> xfer] + pdb, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + db := setupIdbWithConnectionString( + t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) + defer db.Close() + + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) + require.NoError(t, err) + rootTxid := appCall.Txn.ID() + + err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + return proc.Process(&blockCert) + }, nil) + require.NoError(t, err) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // When: searching for a transaction that matches part of the transaction. + results, _ := db.Transactions(context.Background(), tc.filter) + + // Then: only the root transaction should be returned if the ReturnInnerTxnOnly flag is true. + // Else if ReturnInnerTxnOnly is false, then the inner txn should be returned. + num := 0 + for result := range results { + num++ + require.NoError(t, result.Error) + + if tc.returnInner { + // Make sure that only the inner txn is returned + require.True(t, (result.Txn != nil) && (result.RootTxn == nil)) + } else { + // Make sure the root txn is returned. + var stxn *transactions.SignedTxnWithAD + + // Exactly one of Txn and RootTxn must be present. + require.True(t, (result.Txn == nil) != (result.RootTxn == nil)) + + // Get Txn or RootTxn + if result.Txn != nil { + stxn = result.Txn + } + if result.RootTxn != nil { + stxn = result.RootTxn + } + require.Equal(t, rootTxid, stxn.Txn.ID()) + } + } + + // There can be multiple matches because deduplication happens in REST API. + require.Equal(t, tc.matches, num) + }) + } +} + +// TestNonUTF8Logs makes sure we're able to import cheeky logs +// for both the root and inner transactions. +func TestNonUTF8Logs(t *testing.T) { + tests := []struct { + Name string + Logs []string + }{ + { + Name: "Normal", + Logs: []string{"Test log1", "Test log2", "Test log3"}, + }, + { + Name: "Embedded Null", + Logs: []string{"\000", "\x00\x00\x00\x00\x00\x00\x00\x00", string([]byte{00, 00})}, + }, + { + Name: "Invalid UTF8", + Logs: []string{"\x8c", "\xff", "\xf8"}, + }, + { + Name: "Emoji", + Logs: []string{"💩", "💰", "🌐"}, + }, + } + + for _, testcase := range tests { + testcase := testcase + + t.Run(testcase.Name, func(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) + createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ + Logs: testcase.Logs, + InnerTxns: []transactions.SignedTxnWithAD{ + // Inner application call with nested cheeky logs + { + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + Type: protocol.ApplicationCallTx, + Header: transactions.Header{ + Sender: test.AccountA, + }, + ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ + ApplicationID: 789, + OnCompletion: transactions.NoOpOC, + }, + }, + }, + ApplyData: transactions.ApplyData{ + EvalDelta: transactions.EvalDelta{ + Logs: testcase.Logs, + }, + }, + }, + }, + } + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn) + require.NoError(t, err) + + // Test 1: import/accounting should work. + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + // Test 2: transaction results properly serialized + txnRows, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) + for row := range txnRows { + var rowTxn *transactions.SignedTxnWithAD + if row.Txn != nil { + rowTxn = row.Txn + } else { + rowTxn = row.RootTxn + } + require.NoError(t, row.Error) + require.NotNil(t, rowTxn) + require.Equal(t, testcase.Logs, rowTxn.ApplyData.EvalDelta.Logs) + } + }) + } +} + +// Test that LoadGenesis writes account totals. +func TestLoadGenesisAccountTotals(t *testing.T) { + _, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + require.NoError(t, err) + defer db.Close() + + err = db.LoadGenesis(test.MakeGenesis()) + require.NoError(t, err) + + json, err := db.getMetastate(context.Background(), nil, schema.AccountTotals) + require.NoError(t, err) + + ret, err := encoding.DecodeAccountTotals([]byte(json)) + require.NoError(t, err) + + assert.Equal( + t, basics.MicroAlgos{Raw: 4 * 1000 * 1000 * 1000 * 1000}, ret.Offline.Money) +} + +func TestTxnAssetID(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + assetid := uint64(1) + createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 10) + configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 10) + appid := uint64(3) + createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) + destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 10) + + block, err := test.MakeBlockForTxns( + test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, + &createAppTxn, &destroyAppTxn) + require.NoError(t, err) + + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) + for i := 0; i < 2; i++ { + row, ok := <-txnRowsCh + require.True(t, ok) + require.NoError(t, row.Error) + assert.Equal(t, assetid, row.AssetID) + } + for i := 0; i < 2; i++ { + row, ok := <-txnRowsCh + require.True(t, ok) + require.NoError(t, row.Error) + assert.Equal(t, appid, row.AssetID) + } +} + +func TestBadTxnJsonEncoding(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + // Need to import a block header because the transactions query joins on it. + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) + require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + rootTxid := "abc" + rootIntra := uint(4) + badJSON := `{"aaaaaaaa": 0}` + + query := `INSERT INTO txn (round, intra, typeenum, asset, txid, txn, extra) + VALUES (1, $1, 0, 0, $2, $3, $4)` + + _, err = db.db.Exec( + context.Background(), query, rootIntra, rootTxid, badJSON, + encoding.EncodeTxnExtra(&idb.TxnExtra{})) + require.NoError(t, err) + + { + extra := idb.TxnExtra{ + RootIntra: idb.OptionalUint{Present: true, Value: rootIntra}, + RootTxid: rootTxid, + } + _, err = db.db.Exec( + context.Background(), query, rootIntra+1, nil, badJSON, + encoding.EncodeTxnExtra(&extra)) + require.NoError(t, err) + } + + { + offset := uint64(rootIntra) + tf := idb.TransactionFilter{ + Offset: &offset, + } + rowsCh, _ := db.Transactions(context.Background(), tf) + + row, ok := <-rowsCh + require.True(t, ok) + + require.Error(t, row.Error) + assert.Contains(t, row.Error.Error(), "error decoding txn") + } + + { + offset := uint64(rootIntra) + 1 + tf := idb.TransactionFilter{ + Offset: &offset, + } + rowsCh, _ := db.Transactions(context.Background(), tf) + + row, ok := <-rowsCh + require.True(t, ok) + + require.Error(t, row.Error) + assert.Contains(t, row.Error.Error(), "error decoding roottxn") + } +} + +func TestKeytypeDoNotResetReceiver(t *testing.T) { + block := test.MakeGenesisBlock() + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + defer shutdownFunc() + + assertKeytype(t, db, test.AccountA, nil) + + // Sigtype of account B becomes "sig". + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}, 10) + block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) + require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + // Sigtype of account A becomes "sig" and B remains the same. + txn = test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 10) + block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) + require.NoError(t, err) + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + keytype := "sig" + assertKeytype(t, db, test.AccountA, &keytype) + assertKeytype(t, db, test.AccountB, &keytype) +} + +// Test that if information in `txn` and `txn_participation` tables is ahead of +// the current round, AddBlock() still runs successfully. +func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { + block := test.MakeGenesisBlock() + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + defer shutdownFunc() + + { + query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) + VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` + _, err := db.db.Exec(context.Background(), query) + require.NoError(t, err) + } + { + query := `INSERT INTO txn_participation (addr, round, intra) + VALUES ($1, 1, 0)` + _, err := db.db.Exec(context.Background(), query, test.AccountA[:]) + require.NoError(t, err) + } + + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) + block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) + require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) +} + +// Test that AddBlock() writes to `txn_participation` table. +func TestAddBlockTxnParticipationAdded(t *testing.T) { + block := test.MakeGenesisBlock() + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + defer shutdownFunc() + + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) + block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) + require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + + tf := idb.TransactionFilter{ + Address: test.AccountA[:], + } + rowsCh, _ := db.Transactions(context.Background(), tf) + + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + require.NotNil(t, row.Txn) + assert.Equal(t, txn, *row.Txn) +} + +// Test that if information in the `txn` table is ahead of the current round, +// Transactions() doesn't return the rows ahead of the state. +func TestTransactionsTxnAhead(t *testing.T) { + block := test.MakeGenesisBlock() + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + defer shutdownFunc() + + // Insert a transaction row at round 1 and check that Transactions() does not return + // it. + { + query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) + VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` + _, err := db.db.Exec(context.Background(), query) + require.NoError(t, err) + } + { + rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) + _, ok := <-rowsCh + assert.False(t, ok) + } + + // Now add an empty round 1 block, and verify that Transactions() returns the + // fake transaction. + { + block, err := test.MakeBlockForTxns(block.BlockHeader) + require.NoError(t, err) + l := makeTestLedger(t) + defer l.Close() + proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + blockCert := rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + } + { + rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) + row, ok := <-rowsCh + require.True(t, ok) + require.NoError(t, row.Error) + } +} + +// Test that if genesis hash is different from what is in db metastate +// indexer does not start. +func TestGenesisHashCheckAtDBSetup(t *testing.T) { + _, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + genesis := test.MakeGenesis() + db := setupIdbWithConnectionString( + t, connStr, genesis, test.MakeGenesisBlock()) + defer db.Close() + genesisHash := crypto.HashObj(genesis) + network, err := db.getMetastate(context.Background(), nil, schema.NetworkMetaStateKey) + assert.NoError(t, err) + networkState, err := encoding.DecodeNetworkState([]byte(network)) + assert.NoError(t, err) + assert.Equal(t, genesisHash, networkState.GenesisHash) + // connect with different genesis configs + genesis.Network = "testnest" + // different genesisHash, should fail + idb, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + assert.NoError(t, err) + err = idb.LoadGenesis(genesis) + assert.Error(t, err) + assert.Contains(t, err.Error(), "genesis hash not matching") +} + +type ImportState struct { + NextRoundToAccount uint64 `codec:"next_account_round"` +} + +// Test that if genesis hash at initial import is different from what is in db metastate +// indexer does not start. +func TestGenesisHashCheckAtInitialImport(t *testing.T) { + _, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + genesis := test.MakeGenesis() + db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + require.NoError(t, err) + defer db.Close() + // test db upgrade + // set next round to account + state := ImportState{NextRoundToAccount: 1} + var buf []byte + jsonCodecHandle := new(codec.JsonHandle) + enc := codec.NewEncoderBytes(&buf, jsonCodecHandle) + enc.MustEncode(state) + db.setMetastate(nil, schema.StateMetastateKey, string(buf)) + // network state not initialized + networkState, err := db.getNetworkState(context.Background(), nil) + require.ErrorIs(t, err, idb.ErrorNotInitialized) + logger := logrus.New() + genesisReader := bytes.NewReader(protocol.EncodeJSON(genesis)) + imported, err := importer.EnsureInitialImport(db, genesisReader, logger) + require.NoError(t, err) + require.True(t, true, imported) + // network state should be set + networkState, err = db.getNetworkState(context.Background(), nil) + require.NoError(t, err) + require.Equal(t, networkState.GenesisHash, crypto.HashObj(genesis)) + + // change genesis value + genesis.Network = "testnest" + genesisReader = bytes.NewReader(protocol.EncodeJSON(genesis)) + // different genesisHash, should fail + _, err = importer.EnsureInitialImport(db, genesisReader, logger) + require.Error(t, err) + require.Contains(t, err.Error(), "genesis hash not matching") + +} + +func getResults(ctx context.Context, rows <-chan idb.AccountRow) (result []idb.AccountRow) { + for { + select { + case row, ok := <-rows: + if !ok { + return + } + result = append(result, row) + case <-ctx.Done(): + return + } + } +} + +func TestIndexerDb_GetAccounts(t *testing.T) { + db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + defer shutdownFunc() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + testcases := []struct { + gt *uint64 + lt *uint64 + id uint64 + }{ + { + gt: uint64Ptr(1), + lt: nil, + }, + { + gt: nil, + lt: uint64Ptr(1), + }, + { + gt: uint64Ptr(1), + lt: uint64Ptr(1), + }, + { + gt: uint64Ptr(1), + lt: nil, + id: 1, + }, + { + gt: nil, + lt: uint64Ptr(1), + id: 1, + }, + { + gt: uint64Ptr(1), + lt: uint64Ptr(1), + id: 1, + }, + } + + for i, testcase := range testcases { + t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { + resultCh, count := db.GetAccounts(ctx, idb.AccountQueryOptions{ + AssetGT: testcase.gt, + AssetLT: testcase.lt, + HasAssetID: testcase.id, + }) + assert.Equal(t, uint64(0), count) + results := getResults(ctx, resultCh) + + if testcase.id == 0 { + // When the id is 0, there should be an error + assert.NotNil(t, results) + assert.Len(t, results, 1) + assert.Error(t, results[0].Error) + expected := fmt.Sprintf("AssetGT=%d, AssetLT=%d, but HasAssetID=0", uintOrDefault(testcase.gt), uintOrDefault(testcase.lt)) + assert.Equal(t, expected, results[0].Error.Error()) + } else { + // Otherwise, the empty DB should simply return no results. + assert.Nil(t, results) + } + }) + } +} diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index aa0e7a331..85aaea497 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -251,7 +251,7 @@ func MakeAppOptInTxn(appid uint64, sender basics.Address, lv uint64) transaction } // MakeAppOptOutTxn makes a transaction that opts out an app. -func MakeAppOptOutTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { +func MakeAppOptOutTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -259,6 +259,7 @@ func MakeAppOptOutTxn(appid uint64, sender basics.Address) transactions.SignedTx Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, + LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), From aee8e16a17cab4d83ed66c3776670a7c104b9e2f Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 19 May 2022 16:32:37 -0400 Subject: [PATCH 09/37] refactor processor to use EvalForIndexer --- idb/postgres/internal/writer/writer_test.go | 6 +- idb/postgres/postgres_integration_test.go | 45 +++--- processor/blockprocessor/block_processor.go | 123 ++++++++++++--- .../blockprocessor/block_processor_test.go | 16 +- processor/ledger_for_evaluator.go | 141 ++++++++++++++++++ util/test/account_testutil.go | 3 +- 6 files changed, 276 insertions(+), 58 deletions(-) create mode 100644 processor/ledger_for_evaluator.go diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index b0893e59c..28168a57a 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -189,7 +189,7 @@ func TestWriterTxnTableBasic(t *testing.T) { Payset: make([]transactions.SignedTxnInBlock, 2), } - stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) + stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) var err error block.Payset[0], err = block.BlockHeader.EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) @@ -333,7 +333,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { var tests []testtype { - stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) + stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) stib0, err := makeBlockFunc().EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) @@ -624,7 +624,7 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { Payset: make(transactions.Payset, 1), } - stxnad := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 0) + stxnad := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) stxnad.Sig[0] = 5 // set signature so that keytype for account is updated var err error block.Payset[0], err = block.EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index b4fb46bc2..33b5b9ae2 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -308,7 +308,7 @@ func TestMultipleWriters(t *testing.T) { /////////// // Given // Send amt to AccountE /////////// - payAccountE := test.MakePaymentTxn(1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, basics.Address{}, 0) + payAccountE := test.MakePaymentTxn(1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &payAccountE) require.NoError(t, err) @@ -423,7 +423,7 @@ func TestRekeyBasic(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 0) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -456,7 +456,7 @@ func TestRekeyToItself(t *testing.T) { // Given // Send rekey transactions /////////// - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA, 0) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -490,9 +490,9 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn0 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 0) - txn1 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA, 0) - txn2 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC, 0) + txn0 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) + txn1 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA) + txn2 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) @@ -526,7 +526,7 @@ func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -987,7 +987,7 @@ func TestKeytypeBasic(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Sig - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -1003,7 +1003,7 @@ func TestKeytypeBasic(t *testing.T) { assertKeytype(t, db, test.AccountA, &keytype) // Msig - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) @@ -1321,7 +1321,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { defer shutdownFunc() // Sig - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -1337,7 +1337,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { assertKeytype(t, db, test.AccountA, &keytype) // Rekey. - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB, 10) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -1348,7 +1348,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Msig - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}, 10) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) txn.AuthAddr = test.AccountB @@ -1371,8 +1371,7 @@ func TestKeytypeDeletedAccount(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) - closeTxn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}, 10) + closeTxn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &closeTxn) require.NoError(t, err) @@ -1509,10 +1508,8 @@ func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - createTxn := test.MakePaymentTxn( - 0, 100000, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}, 10) - deleteTxn := test.MakePaymentTxn( - 0, 100000, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}, 10) + createTxn := test.MakePaymentTxn(0, 100000, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) + deleteTxn := test.MakePaymentTxn(0, 100000, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) @@ -2052,8 +2049,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Sigtype of account B becomes "sig". - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}, 10) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) l := makeTestLedger(t) @@ -2065,8 +2061,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { require.NoError(t, err) // Sigtype of account A becomes "sig" and B remains the same. - txn = test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}, 10) + txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) blockCert = rpcs.EncodedBlockCert{Block: block} @@ -2098,8 +2093,7 @@ func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { require.NoError(t, err) } - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) l := makeTestLedger(t) @@ -2117,8 +2111,7 @@ func TestAddBlockTxnParticipationAdded(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) defer shutdownFunc() - txn := test.MakePaymentTxn( - 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 10) + txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) l := makeTestLedger(t) diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 446d78194..4a415e5a3 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -3,9 +3,14 @@ package blockprocessor import ( "fmt" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" ) @@ -35,52 +40,132 @@ func MakeProcessor(ledger *ledger.Ledger, handler func(block *ledgercore.Validat } // Process a raw algod block -func (processor *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { +func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { if blockCert == nil { return fmt.Errorf("Process(): cannot process a nil block") } - if uint64(blockCert.Block.Round()) != processor.nextRoundToProcess { - return fmt.Errorf("Process() invalid round blockCert.Block.Round(): %d processor.nextRoundToProcess: %d", blockCert.Block.Round(), processor.nextRoundToProcess) + if uint64(blockCert.Block.Round()) != proc.nextRoundToProcess { + return fmt.Errorf("Process() invalid round blockCert.Block.Round(): %d proc.nextRoundToProcess: %d", blockCert.Block.Round(), proc.nextRoundToProcess) } - blkeval, err := processor.ledger.StartEvaluator(blockCert.Block.BlockHeader, len(blockCert.Block.Payset), 0) - if err != nil { - return fmt.Errorf("Process() block eval err: %w", err) + proto, ok := config.Consensus[blockCert.Block.BlockHeader.CurrentProtocol] + if !ok { + return fmt.Errorf( + "Process() cannot find proto version %s", blockCert.Block.BlockHeader.CurrentProtocol) } + proto.EnableAssetCloseAmount = true - _, _, err = blkeval.ProcessBlockForIndexer(&blockCert.Block) + ledgerForEval, err := processor.MakeLedgerForEvaluator(proc.ledger) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("Process() err: %w", err) } - genblk, err := blkeval.GenerateBlock() + resources, _ := prepareEvalResources(&ledgerForEval, &blockCert.Block) + if err != nil { + panic(fmt.Errorf("Process() eval err: %w", err)) + } + + delta, _, err := + ledger.EvalForIndexer(ledgerForEval, &blockCert.Block, proto, resources) if err != nil { - return fmt.Errorf("%w", err) + return fmt.Errorf("Process() eval err: %w", err) } // validated block - vb := ledgercore.MakeValidatedBlock(blockCert.Block, genblk.Delta()) + vb := ledgercore.MakeValidatedBlock(blockCert.Block, delta) if err != nil { return fmt.Errorf("Process() block eval err: %w", err) } // execute handler before writing to local ledger - if processor.handler != nil { - err = processor.handler(&vb) + if proc.handler != nil { + err = proc.handler(&vb) if err != nil { return fmt.Errorf("Process() handler err: %w", err) } } // write to ledger - err = processor.ledger.AddValidatedBlock(vb, blockCert.Certificate) + err = proc.ledger.AddValidatedBlock(vb, blockCert.Certificate) if err != nil { return fmt.Errorf("Process() add validated block err: %w", err) } - processor.nextRoundToProcess = uint64(processor.ledger.Latest()) + 1 + proc.nextRoundToProcess = uint64(proc.ledger.Latest()) + 1 return nil } -func (processor *blockProcessor) SetHandler(handler func(block *ledgercore.ValidatedBlock) error) { - processor.handler = handler +func (proc *blockProcessor) SetHandler(handler func(block *ledgercore.ValidatedBlock) error) { + proc.handler = handler +} + +func (proc *blockProcessor) NextRoundToProcess() uint64 { + return proc.nextRoundToProcess +} + +// Preload all resources (account data, account resources, asset/app creators) for the +// evaluator. +func prepareEvalResources(l *processor.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { + assetCreators, appCreators, err := prepareCreators(l, block.Payset) + if err != nil { + return ledger.EvalForIndexerResources{}, + fmt.Errorf("prepareEvalResources() err: %w", err) + } + + res := ledger.EvalForIndexerResources{ + Accounts: nil, + Resources: nil, + Creators: make(map[ledger.Creatable]ledger.FoundAddress), + } + + for index, foundAddress := range assetCreators { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AssetCreatable, + } + res.Creators[creatable] = foundAddress + } + for index, foundAddress := range appCreators { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AppCreatable, + } + res.Creators[creatable] = foundAddress + } + + res.Accounts, res.Resources, err = prepareAccountsResources(l, block.Payset, assetCreators, appCreators) + if err != nil { + return ledger.EvalForIndexerResources{}, + fmt.Errorf("prepareEvalResources() err: %w", err) + } + + return res, nil +} + +// Preload asset and app creators. +func prepareCreators(l *processor.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { + assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) + + assets, err := l.GetAssetCreator(assetsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) + } + apps, err := l.GetAppCreator(appsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) + } + + return assets, apps, nil } -func (processor *blockProcessor) NextRoundToProcess() uint64 { - return processor.nextRoundToProcess +// Preload account data and account resources. +func prepareAccountsResources(l *processor.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { + addressesReq, resourcesReq := + accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) + + accounts, err := l.LookupWithoutRewards(addressesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) + } + resources, err := l.LookupResources(resourcesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) + } + + return accounts, resources, nil } diff --git a/processor/blockprocessor/block_processor_test.go b/processor/blockprocessor/block_processor_test.go index 3f2bac77f..848a3b084 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/processor/blockprocessor/block_processor_test.go @@ -28,10 +28,10 @@ func TestProcess(t *testing.T) { } pr, _ := block_processor.MakeProcessor(l, handler) prevHeader := genesisBlock.BlockHeader - + assert.Equal(t, basics.Round(0), l.Latest()) // create a few rounds for i := 1; i <= 3; i++ { - txn := test.MakePaymentTxn(0, uint64(i), 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn := test.MakePaymentTxn(0, uint64(i), 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(prevHeader, &txn) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} @@ -62,7 +62,7 @@ func TestFailedProcess(t *testing.T) { genesisBlock, err := l.Block(basics.Round(0)) assert.Nil(t, err) // incorrect round - txn := test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA, 0) + txn := test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA) block, err := test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) block.BlockHeader.Round = 10 assert.Nil(t, err) @@ -71,7 +71,7 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "Process() invalid round blockCert.Block.Round()") // non-zero balance after close remainder to sender address - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA, 0) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} @@ -79,7 +79,7 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // stxn GenesisID not empty - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) block.Payset[0].Txn.GenesisID = "genesisID" @@ -88,13 +88,13 @@ func TestFailedProcess(t *testing.T) { assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // eval error: concensus protocol not supported - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) block.BlockHeader.CurrentProtocol = "testing" assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() block eval err") + assert.Contains(t, err.Error(), "Process() cannot find proto version testing") // handler error handler := func(vb *ledgercore.ValidatedBlock) error { @@ -103,7 +103,7 @@ func TestFailedProcess(t *testing.T) { _, err = block_processor.MakeProcessor(l, handler) assert.Contains(t, err.Error(), "MakeProcessor() handler err") pr, _ = block_processor.MakeProcessor(l, nil) - txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}, 0) + txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) pr.SetHandler(handler) diff --git a/processor/ledger_for_evaluator.go b/processor/ledger_for_evaluator.go new file mode 100644 index 000000000..859b57265 --- /dev/null +++ b/processor/ledger_for_evaluator.go @@ -0,0 +1,141 @@ +package processor + +import ( + "fmt" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" +) + +// LedgerForEvaluator implements the indexerLedgerForEval interface from +// go-algorand ledger/eval.go and is used for accounting. +type LedgerForEvaluator struct { + Ledger *ledger.Ledger +} + +// MakeLedgerForEvaluator creates a LedgerForEvaluator object. +func MakeLedgerForEvaluator(ld *ledger.Ledger) (LedgerForEvaluator, error) { + l := LedgerForEvaluator{ + Ledger: ld, + } + return l, nil +} + +// Close shuts down LedgerForEvaluator. +func (l *LedgerForEvaluator) Close() { + l.Ledger.Close() +} + +// LatestBlockHdr is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LatestBlockHdr() (bookkeeping.BlockHeader, error) { + return l.Ledger.BlockHdr(l.Ledger.Latest()) +} + +// LookupWithoutRewards is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LookupWithoutRewards(addresses map[basics.Address]struct{}) (map[basics.Address]*ledgercore.AccountData, error) { + + addressesArr := make([]basics.Address, 0, len(addresses)) + for address := range addresses { + addressesArr = append(addressesArr, address) + } + + res := make(map[basics.Address]*ledgercore.AccountData, len(addresses)) + for _, address := range addressesArr { + acctData, _, err := l.Ledger.LookupWithoutRewards(l.Ledger.Latest(), address) + if err != nil { + return nil, err + } + res[address] = &acctData + } + + return res, nil +} + +// LookupResources is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LookupResources(input map[basics.Address]map[ledger.Creatable]struct{}) (map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { + // Initialize the result `res` with the same structure as `input`. + res := make( + map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, len(input)) + for address, creatables := range input { + creatablesOutput := + make(map[ledger.Creatable]ledgercore.AccountResource, len(creatables)) + res[address] = creatablesOutput + for creatable := range creatables { + creatablesOutput[creatable] = ledgercore.AccountResource{} + } + } + + for address, creatables := range input { + for creatable := range creatables { + switch creatable.Type { + case basics.AssetCreatable: + resource, err := l.Ledger.LookupAsset(0, address, basics.AssetIndex(creatable.Index)) + if err != nil { + return nil, fmt.Errorf( + "LookupResources() %d failed", creatable.Type) + } + res[address][creatable] = ledgercore.AccountResource{AssetHolding: resource.AssetHolding, AssetParams: resource.AssetParams} + case basics.AppCreatable: + resource, err := l.Ledger.LookupApplication(0, address, basics.AppIndex(creatable.Index)) + if err != nil { + return nil, fmt.Errorf( + "LookupResources() %d failed", creatable.Type) + } + res[address][creatable] = ledgercore.AccountResource{AppLocalState: resource.AppLocalState, AppParams: resource.AppParams} + default: + return nil, fmt.Errorf( + "LookupResources() unknown creatable type %d", creatable.Type) + } + } + + } + return res, nil +} + +// GetAssetCreator is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) GetAssetCreator(indices map[basics.AssetIndex]struct{}) (map[basics.AssetIndex]ledger.FoundAddress, error) { + indicesArr := make([]basics.AssetIndex, 0, len(indices)) + for index := range indices { + indicesArr = append(indicesArr, index) + } + + res := make(map[basics.AssetIndex]ledger.FoundAddress, len(indices)) + for _, index := range indicesArr { + cidx := basics.CreatableIndex(index) + address, exists, err := l.Ledger.GetCreatorForRound(0, cidx, basics.AssetCreatable) + if err != nil { + return nil, fmt.Errorf("GetAssetCreator() err: %w", err) + } + res[index] = ledger.FoundAddress{Address: address, Exists: exists} + } + + return res, nil +} + +// GetAppCreator is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) GetAppCreator(indices map[basics.AppIndex]struct{}) (map[basics.AppIndex]ledger.FoundAddress, error) { + indicesArr := make([]basics.AppIndex, 0, len(indices)) + for index := range indices { + indicesArr = append(indicesArr, index) + } + + res := make(map[basics.AppIndex]ledger.FoundAddress, len(indices)) + for _, index := range indicesArr { + cidx := basics.CreatableIndex(index) + address, exists, err := l.Ledger.GetCreatorForRound(0, cidx, basics.AssetCreatable) + if err != nil { + return nil, fmt.Errorf("GetAppCreator() err: %w", err) + } + res[index] = ledger.FoundAddress{Address: address, Exists: exists} + } + + return res, nil +} + +// LatestTotals is part of go-algorand's indexerLedgerForEval interface. +func (l LedgerForEvaluator) LatestTotals() (ledgercore.AccountTotals, error) { + _, totals, err := l.Ledger.LatestTotals() + return totals, err +} diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index 85aaea497..2dd2f299e 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -157,7 +157,7 @@ func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) trans } // MakePaymentTxn creates an algo transfer transaction. -func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeRewards uint64, sender, receiver, close, rekeyTo basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeRewards uint64, sender, receiver, close, rekeyTo basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -167,7 +167,6 @@ func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeReward Fee: basics.MicroAlgos{Raw: fee}, GenesisHash: GenesisHash, RekeyTo: rekeyTo, - LastValid: basics.Round(lv), }, PaymentTxnFields: transactions.PaymentTxnFields{ Receiver: receiver, From a12262c9803d9d8ff17aebe6a6dccbf45254d07c Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 20 May 2022 10:36:04 -0400 Subject: [PATCH 10/37] more test refactoring --- api/handlers_e2e_test.go | 50 ++++---- idb/postgres/postgres_integration_test.go | 137 ++++++++++++---------- util/test/account_testutil.go | 45 ++++--- 3 files changed, 118 insertions(+), 114 deletions(-) diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index 4af2f4766..d0b0c2a3c 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -124,8 +124,8 @@ func TestApplicationHandlers(t *testing.T) { ApplicationID: expectedAppIdx, }, } - optInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA, 10) - optInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB, 10) + optInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) + optInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) @@ -250,12 +250,12 @@ func TestAccountExcludeParameters(t *testing.T) { const expectedAppIdx = 1 // must be 1 since this is the first txn const expectedAssetIdx = 2 - createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) - createAssetTxn := test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", "Asset 2", "http://asset2.com", test.AccountA, 10) - appOptInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA, 10) - appOptInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB, 10) - assetOptInTxnA := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountA, 10) - assetOptInTxnB := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountB, 10) + createAppTxn := test.MakeCreateAppTxn(test.AccountA) + createAssetTxn := test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", "Asset 2", "http://asset2.com", test.AccountA) + appOptInTxnA := test.MakeAppOptInTxn(expectedAppIdx, test.AccountA) + appOptInTxnB := test.MakeAppOptInTxn(expectedAppIdx, test.AccountB) + assetOptInTxnA := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountA) + assetOptInTxnB := test.MakeAssetOptInTxn(expectedAssetIdx, test.AccountB) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAppTxn, &createAssetTxn, &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) @@ -432,36 +432,36 @@ func TestAccountMaxResultsLimit(t *testing.T) { var txns []transactions.SignedTxnWithAD // make apps and assets - for _, id := range deletedAppIDs { - txns = append(txns, test.MakeCreateAppTxn(test.AccountA, id+10)) + for range deletedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) } for _, id := range deletedAssetIDs { txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", - fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA, id+10)) + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) } - for _, id := range expectedAppIDs { - txns = append(txns, test.MakeCreateAppTxn(test.AccountA, id+10)) + for range expectedAppIDs { + txns = append(txns, test.MakeCreateAppTxn(test.AccountA)) } for _, id := range expectedAssetIDs { txns = append(txns, test.MakeAssetConfigTxn(0, 100, 0, false, "UNIT", - fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA, id+10)) + fmt.Sprintf("Asset %d", id), "http://asset.com", test.AccountA)) } // delete some apps and assets for _, id := range deletedAppIDs { - txns = append(txns, test.MakeAppDestroyTxn(id, test.AccountA, id+10)) + txns = append(txns, test.MakeAppDestroyTxn(id, test.AccountA)) } for _, id := range deletedAssetIDs { - txns = append(txns, test.MakeAssetDestroyTxn(id, test.AccountA, id+10)) + txns = append(txns, test.MakeAssetDestroyTxn(id, test.AccountA)) } // opt in to the remaining ones for _, id := range expectedAppIDs { - txns = append(txns, test.MakeAppOptInTxn(id, test.AccountA, id+10)) - txns = append(txns, test.MakeAppOptInTxn(id, test.AccountB, id+10)) + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountA)) + txns = append(txns, test.MakeAppOptInTxn(id, test.AccountB)) } for _, id := range expectedAssetIDs { - txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountA, id+10)) - txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountB, id+10)) + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountA)) + txns = append(txns, test.MakeAssetOptInTxn(id, test.AccountB)) } ptxns := make([]*transactions.SignedTxnWithAD, len(txns)) @@ -872,7 +872,7 @@ func TestInnerTxn(t *testing.T) { /////////// // Given // a DB with some inner txns in it. /////////// - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) expectedID := appCall.Txn.ID().String() block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) @@ -929,7 +929,7 @@ func TestPagingRootTxnDeduplication(t *testing.T) { appAddr[1] = 99 appAddrStr := appAddr.String() - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) expectedID := appCall.Txn.ID().String() block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) @@ -1191,7 +1191,7 @@ func TestAccountClearsNonUTF8(t *testing.T) { urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=") url := string(urlBytes) unitName := "asset\rwith\nnon-printable\tcharacters" - createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA, 10) + createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) @@ -1317,7 +1317,7 @@ func TestLookupInnerLogs(t *testing.T) { /////////// // Given // a DB with some inner txns in it. /////////// - appCall := test.MakeAppCallWithInnerAppCall(test.AccountA, 10) + appCall := test.MakeAppCallWithInnerAppCall(test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) @@ -1420,7 +1420,7 @@ func TestLookupMultiInnerLogs(t *testing.T) { /////////// // Given // a DB with some inner txns in it. /////////// - appCall := test.MakeAppCallWithMultiLogs(test.AccountA, 10) + appCall := test.MakeAppCallWithMultiLogs(test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index 33b5b9ae2..e38399c35 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -177,14 +177,14 @@ func TestAssetCloseReopenTransfer(t *testing.T) { /////////// // Given // A round scenario requiring subround accounting: AccountA is funded, closed, opts back, and funded again. /////////// - createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD, 10) - optInA := test.MakeAssetOptInTxn(assetid, test.AccountA, 10) - fundA := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 10) - optInB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - optInC := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) - closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 10) - optInDA := test.MakeAssetOptInTxn(assetid, test.AccountA, 12) - payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 12) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountD) + optInA := test.MakeAssetOptInTxn(assetid, test.AccountA) + fundA := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + optInB := test.MakeAssetOptInTxn(assetid, test.AccountB) + optInC := test.MakeAssetOptInTxn(assetid, test.AccountC) + closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC) + optInDA := test.MakeAssetOptInTxn(assetid, test.AccountA) + payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optInA, &fundA, &optInB, @@ -230,11 +230,11 @@ func TestReCreateAssetHolding(t *testing.T) { // A new asset with default-frozen, AccountB opts-in and has its frozen state // toggled. /////////// Then AccountB opts-out then opts-in again. - createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - unfreezeB := test.MakeAssetFreezeTxn(assetid, !frozen, test.AccountA, test.AccountB, 10) - optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD, 10) - optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB, 12) + createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + unfreezeB := test.MakeAssetFreezeTxn(assetid, !frozen, test.AccountA, test.AccountB) + optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD) + optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB) var err error block, err = test.MakeBlockForTxns( @@ -272,10 +272,10 @@ func TestNoopOptins(t *testing.T) { /////////// assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 10) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB, 10) - optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB, 12) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB) + optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &unfreezeB, &optinB2) @@ -370,14 +370,14 @@ func TestBlockWithTransactions(t *testing.T) { /////////// // Given // A block at round `round` with 5 transactions. /////////// - txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD, 10) - txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA, 10) - txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 10) - txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) - txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC, 10) - txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}, 12) - txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}, 12) + txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD) + txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA) + txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB) + txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC) + txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC) + txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}) + txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) txns := []*transactions.SignedTxnWithAD{ &txn1, &txn2, &txn3, &txn4, &txn5, &txn6, &txn7, &txn8} @@ -455,9 +455,8 @@ func TestRekeyToItself(t *testing.T) { /////////// // Given // Send rekey transactions /////////// - - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA) - + txn := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -469,6 +468,16 @@ func TestRekeyToItself(t *testing.T) { err = proc.Process(&blockCert) require.NoError(t, err) + txn = test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, + test.AccountA) + block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) + require.NoError(t, err) + + blockCert = rpcs.EncodedBlockCert{Block: block} + err = proc.Process(&blockCert) + require.NoError(t, err) + ////////// // Then // Account's A auth-address is not recorded ////////// @@ -554,9 +563,9 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { /////////// // Given // A new asset with default-frozen = true, and AccountB opting into it. /////////// - createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) - modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA, 10) - optin := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) + createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) + modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) + optin := test.MakeAssetOptInTxn(assetid, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAssetNotFrozen, &modifyAssetToFrozen, @@ -593,7 +602,7 @@ func TestZeroTotalAssetCreate(t *testing.T) { /////////// // Given // A new asset with total = 0. /////////// - createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) + createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) @@ -655,7 +664,7 @@ func TestDestroyAssetBasic(t *testing.T) { assetID := uint64(1) // Create an asset. - txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA, 10) + txn := test.MakeAssetConfigTxn(0, 4, 0, false, "uu", "aa", "", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -668,7 +677,7 @@ func TestDestroyAssetBasic(t *testing.T) { require.NoError(t, err) // Destroy an asset. - txn = test.MakeAssetDestroyTxn(assetID, test.AccountA, 10) + txn = test.MakeAssetDestroyTxn(assetID, test.AccountA) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -696,8 +705,8 @@ func TestDestroyAssetZeroSupply(t *testing.T) { assetID := uint64(1) // Create an asset. Set total supply to 0. - txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA, 10) - txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA, 10) + txn0 := test.MakeAssetConfigTxn(0, 0, 0, false, "uu", "aa", "", test.AccountA) + txn1 := test.MakeAssetDestroyTxn(assetID, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) @@ -753,10 +762,10 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { } // Another account opts in. - txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC, 10) + txn1 := test.MakeAssetOptInTxn(assetID, test.AccountC) // Destroy an asset. - txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB, 10) + txn2 := test.MakeAssetDestroyTxn(assetID, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) @@ -798,9 +807,9 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { // Create a block with freeze txn assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB, 10) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &freeze) @@ -847,7 +856,7 @@ func TestInnerTxnParticipation(t *testing.T) { // otherwise it requires funding the app account and other special setup var appAddr basics.Address appAddr[1] = 99 - createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) + createApp := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createApp) @@ -1024,7 +1033,7 @@ func TestLargeAssetAmount(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA, 10) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) @@ -1168,10 +1177,10 @@ func TestNonDisplayableUTF8(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) // Try to add cheeky inner txns lazily by adding an AD to the acfg txn txn.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ - test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10), + test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA), } txn.ApplyData.EvalDelta.InnerTxns[0].ConfigAsset = basics.AssetIndex(innerAssetID) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) @@ -1258,7 +1267,7 @@ func TestReconfigAsset(t *testing.T) { url := "https://algorand.com" assetID := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA, 10) + txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) l := makeTestLedger(t) @@ -1419,12 +1428,12 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA, 10) - optinB := test.MakeAssetOptInTxn(assetid, test.AccountB, 10) - transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}, 10) - optinC := test.MakeAssetOptInTxn(assetid, test.AccountC, 10) + createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) + optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) + transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}) + optinC := test.MakeAssetOptInTxn(assetid, test.AccountC) // Close B to C. - closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC, 10) + closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC) block, err := test.MakeBlockForTxns( block.BlockHeader, &createAsset, &optinB, &transferAB, @@ -1546,8 +1555,8 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA, 10) - deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA, 10) + createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA) + deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) @@ -1606,8 +1615,8 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { defer shutdownFunc() appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA, 10) - deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 10) + createTxn := test.MakeCreateAppTxn(test.AccountA) + deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) @@ -1644,9 +1653,9 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { defer shutdownFunc() appid := uint64(1) - createTxn := test.MakeCreateAppTxn(test.AccountA, 10) - optInTxn := test.MakeAppOptInTxn(appid, test.AccountB, 10) - optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB, 10) + createTxn := test.MakeCreateAppTxn(test.AccountA) + optInTxn := test.MakeAppOptInTxn(appid, test.AccountB) + optOutTxn := test.MakeAppOptOutTxn(appid, test.AccountB) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) require.NoError(t, err) @@ -1771,7 +1780,7 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) defer db.Close() - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 10) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) @@ -1858,7 +1867,7 @@ func TestNonUTF8Logs(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) + createAppTxn := test.MakeCreateAppTxn(test.AccountA) createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ Logs: testcase.Logs, InnerTxns: []transactions.SignedTxnWithAD{ @@ -1940,11 +1949,11 @@ func TestTxnAssetID(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA, 10) - configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA, 10) + createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA) + configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA) appid := uint64(3) - createAppTxn := test.MakeCreateAppTxn(test.AccountA, 10) - destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA, 10) + createAppTxn := test.MakeCreateAppTxn(test.AccountA) + destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAssetTxn, &configAssetTxn, diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index 2dd2f299e..dd1ae560d 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -49,7 +49,7 @@ func DecodeAddressOrPanic(addr string) basics.Address { } // MakeAssetConfigTxn is a helper to ensure test asset config are initialized. -func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, unitName, assetName, url string, addr basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, unitName, assetName, url string, addr basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -58,7 +58,6 @@ func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, un Sender: addr, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(configid), @@ -83,7 +82,7 @@ func MakeAssetConfigTxn(configid, total, decimals uint64, defaultFrozen bool, un } // MakeAssetFreezeTxn create an asset freeze/unfreeze transaction. -func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -92,7 +91,6 @@ func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basic Sender: sender, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, AssetFreezeTxnFields: transactions.AssetFreezeTxnFields{ FreezeAccount: freezeAccount, @@ -106,7 +104,7 @@ func MakeAssetFreezeTxn(assetid uint64, frozen bool, sender, freezeAccount basic } // MakeAssetTransferTxn creates an asset transfer transaction. -func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -115,7 +113,6 @@ func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Ad Sender: sender, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, AssetTransferTxnFields: transactions.AssetTransferTxnFields{ XferAsset: basics.AssetIndex(assetid), @@ -132,12 +129,12 @@ func MakeAssetTransferTxn(assetid, amt uint64, sender, receiver, close basics.Ad } // MakeAssetOptInTxn makes a transaction that opts in an asset. -func MakeAssetOptInTxn(assetid uint64, address basics.Address, lv uint64) transactions.SignedTxnWithAD { - return MakeAssetTransferTxn(assetid, 0, address, address, basics.Address{}, lv) +func MakeAssetOptInTxn(assetid uint64, address basics.Address) transactions.SignedTxnWithAD { + return MakeAssetTransferTxn(assetid, 0, address, address, basics.Address{}) } // MakeAssetDestroyTxn makes a transaction that destroys an asset. -func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAssetDestroyTxn(assetID uint64, sender basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -145,7 +142,6 @@ func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) trans Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(assetID), @@ -157,7 +153,8 @@ func MakeAssetDestroyTxn(assetID uint64, sender basics.Address, lv uint64) trans } // MakePaymentTxn creates an algo transfer transaction. -func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeRewards uint64, sender, receiver, close, rekeyTo basics.Address) transactions.SignedTxnWithAD { +func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, + closeRewards uint64, sender, receiver, close, rekeyTo basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -167,6 +164,7 @@ func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeReward Fee: basics.MicroAlgos{Raw: fee}, GenesisHash: GenesisHash, RekeyTo: rekeyTo, + LastValid: 10, }, PaymentTxnFields: transactions.PaymentTxnFields{ Receiver: receiver, @@ -186,7 +184,7 @@ func MakePaymentTxn(fee, amt, closeAmt, sendRewards, receiveRewards, closeReward } // MakeCreateAppTxn makes a transaction that creates a simple application. -func MakeCreateAppTxn(sender basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { // Create a transaction with ExtraProgramPages field set to 1 return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ @@ -195,7 +193,6 @@ func MakeCreateAppTxn(sender basics.Address, lv uint64) transactions.SignedTxnWi Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -208,7 +205,7 @@ func MakeCreateAppTxn(sender basics.Address, lv uint64) transactions.SignedTxnWi } // MakeAppDestroyTxn makes a transaction that destroys an app. -func MakeAppDestroyTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -216,7 +213,6 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address, lv uint64) transacti Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), @@ -229,7 +225,7 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address, lv uint64) transacti } // MakeAppOptInTxn makes a transaction that opts in an app. -func MakeAppOptInTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAppOptInTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -237,7 +233,6 @@ func MakeAppOptInTxn(appid uint64, sender basics.Address, lv uint64) transaction Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), @@ -250,7 +245,7 @@ func MakeAppOptInTxn(appid uint64, sender basics.Address, lv uint64) transaction } // MakeAppOptOutTxn makes a transaction that opts out an app. -func MakeAppOptOutTxn(appid uint64, sender basics.Address, lv uint64) transactions.SignedTxnWithAD { +func MakeAppOptOutTxn(appid uint64, sender basics.Address) transactions.SignedTxnWithAD { return transactions.SignedTxnWithAD{ SignedTxn: transactions.SignedTxn{ Txn: transactions.Transaction{ @@ -258,7 +253,6 @@ func MakeAppOptOutTxn(appid uint64, sender basics.Address, lv uint64) transactio Header: transactions.Header{ Sender: sender, GenesisHash: GenesisHash, - LastValid: basics.Round(lv), }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), @@ -303,8 +297,8 @@ func MakeAppCallTxnWithLogs(appid uint64, sender basics.Address, logs []string) // |- application call // |- asset transfer // |- application call -func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSender, assetReceiver basics.Address, lv uint64) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender, lv) +func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSender, assetReceiver basics.Address) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender) // In order to simplify the test, // since db.AddBlock uses ApplyData from the block and not from the evaluator, @@ -376,8 +370,8 @@ func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSen // |- application call // |- application call // |- application call -func MakeAppCallWithMultiLogs(appSender basics.Address, lv uint64) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender, lv) +func MakeAppCallWithMultiLogs(appSender basics.Address) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender) // Add a log to the outer appl call createApp.ApplicationID = 123 @@ -435,8 +429,8 @@ func MakeAppCallWithMultiLogs(appSender basics.Address, lv uint64) transactions. // application create // |- application call // |- application create -func MakeAppCallWithInnerAppCall(appSender basics.Address, lv uint64) transactions.SignedTxnWithAD { - createApp := MakeCreateAppTxn(appSender, lv) +func MakeAppCallWithInnerAppCall(appSender basics.Address) transactions.SignedTxnWithAD { + createApp := MakeCreateAppTxn(appSender) // Add a log to the outer appl call createApp.ApplicationID = 123 @@ -540,6 +534,7 @@ func MakeBlockForTxns(prevHeader bookkeeping.BlockHeader, inputs ...*transaction res.Payset = append(res.Payset, stxnib) } + return res, nil } From 4c15d638d499958f0357ce005a770226177404e4 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 20 May 2022 16:12:15 -0400 Subject: [PATCH 11/37] make integration failing --- cmd/algorand-indexer/daemon.go | 1 - cmd/algorand-indexer/import.go | 47 +++++++ cmd/algorand-indexer/main.go | 3 + idb/idb_test.go | 2 +- idb/postgres/internal/writer/writer_test.go | 14 +- importer/helper.go | 148 ++++++++++++++++++++ 6 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 cmd/algorand-indexer/import.go diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index c226e2fee..5be946c5f 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -59,7 +59,6 @@ var ( defaultApplicationsLimit uint32 enableAllParameters bool indexerDataDir string - genesisJSONPath string ) var daemonCmd = &cobra.Command{ diff --git a/cmd/algorand-indexer/import.go b/cmd/algorand-indexer/import.go new file mode 100644 index 000000000..331611e0a --- /dev/null +++ b/cmd/algorand-indexer/import.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/algorand/indexer/config" + "github.com/algorand/indexer/idb" + "github.com/algorand/indexer/importer" +) + +var importCmd = &cobra.Command{ + Use: "import", + Short: "import block file or tar file of blocks", + Long: "import block file or tar file of blocks. arguments are interpret as file globs (e.g. *.tar.bz2)", + Run: func(cmd *cobra.Command, args []string) { + config.BindFlags(cmd) + err := configureLogger() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err) + os.Exit(1) + } + + db, availableCh := indexerDbFromFlags(idb.IndexerDbOptions{}) + defer db.Close() + <-availableCh + + helper := importer.NewImportHelper( + genesisJSONPath, + blockFileLimit, + logger) + + helper.Import(db, args) + }, +} + +var ( + genesisJSONPath string + blockFileLimit int +) + +func init() { + importCmd.Flags().StringVarP(&genesisJSONPath, "genesis", "g", "", "path to genesis.json") + importCmd.Flags().IntVarP(&blockFileLimit, "block-file-limit", "", 0, "number of block files to process (for debugging)") +} diff --git a/cmd/algorand-indexer/main.go b/cmd/algorand-indexer/main.go index 651ce798d..f436875be 100644 --- a/cmd/algorand-indexer/main.go +++ b/cmd/algorand-indexer/main.go @@ -134,6 +134,8 @@ func init() { logger.SetOutput(os.Stdout) logger.SetLevel(log.InfoLevel) + rootCmd.AddCommand(importCmd) + importCmd.Hidden = true rootCmd.AddCommand(daemonCmd) rootCmd.AddCommand(apiConfigCmd) @@ -151,6 +153,7 @@ func init() { cmd.Flags().BoolVarP(&doVersion, "version", "v", false, "print version and exit") } addFlags(daemonCmd) + addFlags(importCmd) viper.RegisterAlias("postgres", "postgres-connection-string") diff --git a/idb/idb_test.go b/idb/idb_test.go index c5e561cf5..56165ae39 100644 --- a/idb/idb_test.go +++ b/idb/idb_test.go @@ -12,7 +12,7 @@ import ( func TestTxnRowNext(t *testing.T) { // txn with 2 inner transactions and 2 nested inner transactions - stxn := test.MakeAppCallWithInnerTxn(test.AccountA, test.AccountB, test.AccountC, test.AccountD, test.AccountE, 0) + stxn := test.MakeAppCallWithInnerTxn(test.AccountA, test.AccountB, test.AccountC, test.AccountD, test.AccountE) testcases := []struct { name string diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index 28168a57a..5b8940ea7 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -195,7 +195,7 @@ func TestWriterTxnTableBasic(t *testing.T) { block.BlockHeader.EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA, 0) + stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA) block.Payset[1], err = block.BlockHeader.EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -268,7 +268,7 @@ func TestWriterTxnTableAssetCloseAmount(t *testing.T) { }, Payset: make(transactions.Payset, 1), } - stxnad := test.MakeAssetTransferTxn(1, 2, test.AccountA, test.AccountB, test.AccountC, 0) + stxnad := test.MakeAssetTransferTxn(1, 2, test.AccountA, test.AccountB, test.AccountC) var err error block.Payset[0], err = block.EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) require.NoError(t, err) @@ -337,7 +337,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { stib0, err := makeBlockFunc().EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC, 0) + stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC) stib1, err := makeBlockFunc().EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -365,7 +365,7 @@ func TestWriterTxnParticipationTable(t *testing.T) { tests = append(tests, testcase) } { - stxnad := test.MakeCreateAppTxn(test.AccountA, 0) + stxnad := test.MakeCreateAppTxn(test.AccountA) stxnad.Txn.ApplicationCallTxnFields.Accounts = []basics.Address{test.AccountB, test.AccountC} stib, err := makeBlockFunc().EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) @@ -1295,7 +1295,7 @@ func TestAddBlockInvalidInnerAsset(t *testing.T) { db, _, shutdownFunc := pgtest.SetupPostgresWithSchema(t) defer shutdownFunc() - callWithBadInner := test.MakeCreateAppTxn(test.AccountA, 0) + callWithBadInner := test.MakeCreateAppTxn(test.AccountA) callWithBadInner.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ { ApplyData: transactions.ApplyData{ @@ -1332,10 +1332,10 @@ func TestWriterAddBlockInnerTxnsAssetCreate(t *testing.T) { // App call with inner txns, should be intra 0, 1, 2, 3, 4 var appAddr basics.Address appAddr[1] = 99 - appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC, 0) + appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) // Asset create call, should have intra = 5 - assetCreate := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD, 0) + assetCreate := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall, &assetCreate) require.NoError(t, err) diff --git a/importer/helper.go b/importer/helper.go index a384daafc..0c93f552b 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -1,6 +1,8 @@ package importer import ( + "archive/tar" + "compress/bzip2" "context" "errors" "fmt" @@ -8,13 +10,21 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strconv" "strings" + "time" "github.com/algorand/go-algorand-sdk/client/v2/algod" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util" log "github.com/sirupsen/logrus" "github.com/algorand/indexer/idb" @@ -40,6 +50,43 @@ type ImportHelper struct { Log *log.Logger } +// Import is the main ImportHelper function that glues together a directory full of block files and an Importer objects. +func (h *ImportHelper) Import(db idb.IndexerDb, args []string) { + // Initial import if needed. + genesisReader := GetGenesisFile(h.GenesisJSONPath, nil, h.Log) + _, err := EnsureInitialImport(db, genesisReader, h.Log) + maybeFail(err, h.Log, "EnsureInitialImport() error") + imp := NewImporter(db) + blocks := 0 + txCount := 0 + start := time.Now() + for _, fname := range args { + matches, err := filepath.Glob(fname) + if err == nil { + pathsSorted := blockTarPaths(matches) + sort.Sort(&pathsSorted) + if h.BlockFileLimit != 0 && len(pathsSorted) > h.BlockFileLimit { + pathsSorted = pathsSorted[:h.BlockFileLimit] + } + for _, gfname := range pathsSorted { + fb, ft := importFile(gfname, imp, h.Log, h.GenesisJSONPath) + blocks += fb + txCount += ft + } + } else { + // try without passing throug glob + fb, ft := importFile(fname, imp, h.Log, h.GenesisJSONPath) + blocks += fb + txCount += ft + } + } + blockdone := time.Now() + if blocks > 0 { + dt := blockdone.Sub(start) + h.Log.Infof("%d blocks in %s, %.0f/s, %d txn, %.0f/s", blocks, dt.String(), float64(time.Second)*float64(blocks)/float64(dt), txCount, float64(time.Second)*float64(txCount)/float64(dt)) + } +} + func maybeFail(err error, l *log.Logger, errfmt string, params ...interface{}) { if err == nil { return @@ -48,6 +95,93 @@ func maybeFail(err error, l *log.Logger, errfmt string, params ...interface{}) { os.Exit(1) } +func importTar(imp Importer, tarfile io.Reader, l *log.Logger, genesisReader io.Reader) (blockCount, txCount int, err error) { + tf := tar.NewReader(tarfile) + var header *tar.Header + header, err = tf.Next() + txCount = 0 + blocks := make([]rpcs.EncodedBlockCert, 0) + for err == nil { + if header.Typeflag != tar.TypeReg { + err = fmt.Errorf("cannot deal with non-regular-file tar entry %#v", header.Name) + return + } + blockbytes := make([]byte, header.Size) + _, err = io.ReadFull(tf, blockbytes) + if err != nil { + err = fmt.Errorf("error reading tar entry %#v: %v", header.Name, err) + return + } + var blockContainer rpcs.EncodedBlockCert + err = protocol.Decode(blockbytes, &blockContainer) + if err != nil { + err = fmt.Errorf("error decoding blockbytes, %w", err) + return + } + txCount += len(blockContainer.Block.Payset) + blocks = append(blocks, blockContainer) + header, err = tf.Next() + } + if err == io.EOF { + err = nil + } + + less := func(i int, j int) bool { + return blocks[i].Block.Round() < blocks[j].Block.Round() + } + sort.Slice(blocks, less) + + genesis, err := getGenesis(genesisReader) + maybeFail(err, l, "Error getting genesis") + genesisBlock := blocks[0] + initState, err := util.CreateInitState(&genesis, &genesisBlock.Block) + maybeFail(err, l, "Error getting genesis block") + + ld, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) + maybeFail(err, l, "Cannot open ledger") + + proc, err := blockprocessor.MakeProcessor(ld, imp.ImportBlock) + maybeFail(err, l, "Error creating processor") + + for _, blockContainer := range blocks[1:] { + err = proc.Process(&blockContainer) + if err != nil { + return + } + } + + return +} + +func importFile(fname string, imp Importer, l *log.Logger, genesisPath string) (blocks, txCount int) { + blocks = 0 + txCount = 0 + l.Infof("importing %s ...", fname) + genesisReader := GetGenesisFile(genesisPath, nil, l) + if strings.HasSuffix(fname, ".tar") { + fin, err := os.Open(fname) + maybeFail(err, l, "%s: %v", fname, err) + defer fin.Close() + tblocks, btxns, err := importTar(imp, fin, l, genesisReader) + maybeFail(err, l, "%s: %v", fname, err) + blocks += tblocks + txCount += btxns + } else if strings.HasSuffix(fname, ".tar.bz2") { + fin, err := os.Open(fname) + maybeFail(err, l, "%s: %v", fname, err) + defer fin.Close() + bzin := bzip2.NewReader(fin) + tblocks, btxns, err := importTar(imp, bzin, l, genesisReader) + maybeFail(err, l, "%s: %v", fname, err) + blocks += tblocks + txCount += btxns + } else { + //assume a standalone block msgpack blob + maybeFail(errors.New("cannot import a standalone block"), l, "not supported") + } + return +} + func loadGenesis(db idb.IndexerDb, in io.Reader) (err error) { var genesis bookkeeping.Genesis gbytes, err := ioutil.ReadAll(in) @@ -168,3 +302,17 @@ func checkGenesisHash(db idb.IndexerDb, genesisReader io.Reader) error { } return nil } + +func getGenesis(in io.Reader) (bookkeeping.Genesis, error) { + var genesis bookkeeping.Genesis + gbytes, err := ioutil.ReadAll(in) + if err != nil { + return bookkeeping.Genesis{}, fmt.Errorf("error reading genesis, %v", err) + } + err = protocol.DecodeJSON(gbytes, &genesis) + if err != nil { + return bookkeeping.Genesis{}, fmt.Errorf("error decoding genesis, %v", err) + } + + return genesis, nil +} From 7a8c05d8867acc7051357445fcdd63c62db895b1 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 23 May 2022 16:28:05 -0400 Subject: [PATCH 12/37] adding ledger for eval tests --- processor/blockprocessor/block_processor.go | 42 ++- .../{ => ledger}/ledger_for_evaluator.go | 13 +- processor/ledger/ledger_for_evaluator_test.go | 316 ++++++++++++++++++ util/test/account_testutil.go | 26 +- 4 files changed, 372 insertions(+), 25 deletions(-) rename processor/{ => ledger}/ledger_for_evaluator.go (92%) create mode 100644 processor/ledger/ledger_for_evaluator_test.go diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 4a415e5a3..8480f86af 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -12,6 +12,7 @@ import ( "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" + indxledger "github.com/algorand/indexer/processor/ledger" ) type blockProcessor struct { @@ -41,6 +42,19 @@ func MakeProcessor(ledger *ledger.Ledger, handler func(block *ledgercore.Validat // Process a raw algod block func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { + apps, _ := proc.ledger.ListApplications(100, 100) + //paysets := blockCert.Block.Payset + + for _, app := range apps { + fmt.Printf("%v\n", app) + } + //for _, payset := range paysets { + // //if payset.Txn.ID().String() == "MM76PGEZHO5WRA3HJIVNKQE2Y7Y2GJS7CPIGL65MX43AUEUNYGEQ" { + // // fmt.Printf("%+v\n", payset) + // //} + // fmt.Println(payset.Txn.ID().String()) + //} + if blockCert == nil { return fmt.Errorf("Process(): cannot process a nil block") } @@ -53,27 +67,33 @@ func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { return fmt.Errorf( "Process() cannot find proto version %s", blockCert.Block.BlockHeader.CurrentProtocol) } - proto.EnableAssetCloseAmount = true + protoChanged := !proto.EnableAssetCloseAmount - ledgerForEval, err := processor.MakeLedgerForEvaluator(proc.ledger) + ledgerForEval, err := indxledger.MakeLedgerForEvaluator(proc.ledger) if err != nil { return fmt.Errorf("Process() err: %w", err) } - resources, _ := prepareEvalResources(&ledgerForEval, &blockCert.Block) + resources, err := prepareEvalResources(&ledgerForEval, &blockCert.Block) if err != nil { - panic(fmt.Errorf("Process() eval err: %w", err)) + panic(fmt.Errorf("Process() resources err: %w", err)) } - delta, _, err := + delta, modifiedTxns, err := ledger.EvalForIndexer(ledgerForEval, &blockCert.Block, proto, resources) if err != nil { return fmt.Errorf("Process() eval err: %w", err) } // validated block - vb := ledgercore.MakeValidatedBlock(blockCert.Block, delta) - if err != nil { - return fmt.Errorf("Process() block eval err: %w", err) + var vb ledgercore.ValidatedBlock + vb = ledgercore.MakeValidatedBlock(blockCert.Block, delta) + if protoChanged { + block := bookkeeping.Block{ + BlockHeader: blockCert.Block.BlockHeader, + Payset: modifiedTxns, + } + vb = ledgercore.MakeValidatedBlock(block, delta) } + // execute handler before writing to local ledger if proc.handler != nil { err = proc.handler(&vb) @@ -100,7 +120,7 @@ func (proc *blockProcessor) NextRoundToProcess() uint64 { // Preload all resources (account data, account resources, asset/app creators) for the // evaluator. -func prepareEvalResources(l *processor.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { +func prepareEvalResources(l *indxledger.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { assetCreators, appCreators, err := prepareCreators(l, block.Payset) if err != nil { return ledger.EvalForIndexerResources{}, @@ -138,7 +158,7 @@ func prepareEvalResources(l *processor.LedgerForEvaluator, block *bookkeeping.Bl } // Preload asset and app creators. -func prepareCreators(l *processor.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { +func prepareCreators(l *indxledger.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) assets, err := l.GetAssetCreator(assetsReq) @@ -154,7 +174,7 @@ func prepareCreators(l *processor.LedgerForEvaluator, payset transactions.Payset } // Preload account data and account resources. -func prepareAccountsResources(l *processor.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { +func prepareAccountsResources(l *indxledger.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { addressesReq, resourcesReq := accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) diff --git a/processor/ledger_for_evaluator.go b/processor/ledger/ledger_for_evaluator.go similarity index 92% rename from processor/ledger_for_evaluator.go rename to processor/ledger/ledger_for_evaluator.go index 859b57265..8dd0d0a03 100644 --- a/processor/ledger_for_evaluator.go +++ b/processor/ledger/ledger_for_evaluator.go @@ -1,4 +1,4 @@ -package processor +package ledger import ( "fmt" @@ -47,7 +47,12 @@ func (l LedgerForEvaluator) LookupWithoutRewards(addresses map[basics.Address]st if err != nil { return nil, err } - res[address] = &acctData + empty := ledgercore.AccountData{} + if acctData == empty { + res[address] = nil + } else { + res[address] = &acctData + } } return res, nil @@ -71,14 +76,14 @@ func (l LedgerForEvaluator) LookupResources(input map[basics.Address]map[ledger. for creatable := range creatables { switch creatable.Type { case basics.AssetCreatable: - resource, err := l.Ledger.LookupAsset(0, address, basics.AssetIndex(creatable.Index)) + resource, err := l.Ledger.LookupAsset(l.Ledger.Latest(), address, basics.AssetIndex(creatable.Index)) if err != nil { return nil, fmt.Errorf( "LookupResources() %d failed", creatable.Type) } res[address][creatable] = ledgercore.AccountResource{AssetHolding: resource.AssetHolding, AssetParams: resource.AssetParams} case basics.AppCreatable: - resource, err := l.Ledger.LookupApplication(0, address, basics.AppIndex(creatable.Index)) + resource, err := l.Ledger.LookupApplication(l.Ledger.Latest(), address, basics.AppIndex(creatable.Index)) if err != nil { return nil, fmt.Errorf( "LookupResources() %d failed", creatable.Type) diff --git a/processor/ledger/ledger_for_evaluator_test.go b/processor/ledger/ledger_for_evaluator_test.go new file mode 100644 index 000000000..788204173 --- /dev/null +++ b/processor/ledger/ledger_for_evaluator_test.go @@ -0,0 +1,316 @@ +package ledger_test + +import ( + "crypto/rand" + "log" + "testing" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/rpcs" + block_processor "github.com/algorand/indexer/processor/blockprocessor" + indxLeder "github.com/algorand/indexer/processor/ledger" + "github.com/algorand/indexer/util" + "github.com/algorand/indexer/util/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { + + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + txn := test.MakePaymentTxn(0, 100, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := ld.LatestBlockHdr() + require.NoError(t, err) + + assert.Equal(t, block.BlockHeader, ret) +} + +func TestLedgerForEvaluatorAccountDataBasic(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + block_processor.MakeProcessor(l, nil) + accountData, _, err := l.LookupWithoutRewards(0, test.AccountB) + require.NoError(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := + ld.LookupWithoutRewards(map[basics.Address]struct{}{test.AccountB: {}}) + require.NoError(t, err) + + accountDataRet := ret[test.AccountB] + require.NotNil(t, accountDataRet) + assert.Equal(t, accountData, *accountDataRet) +} + +func TestLedgerForEvaluatorAccountDataMissingAccount(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + var addr basics.Address + _, err = rand.Read(addr[:]) + ret, err := + ld.LookupWithoutRewards(map[basics.Address]struct{}{addr: {}}) + require.NoError(t, err) + + accountDataRet := ret[addr] + assert.Nil(t, accountDataRet) +} + +func TestLedgerForEvaluatorAccountAsset(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) + txn1 := test.MakeAssetConfigTxn(0, 4, 0, false, "", "", "", test.AccountA) + txn2 := test.MakeAssetConfigTxn(0, 6, 0, false, "", "", "", test.AccountA) + txn3 := test.MakeAssetConfigTxn(0, 8, 0, false, "", "", "", test.AccountB) + txn4 := test.MakeAssetDestroyTxn(1, test.AccountA) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3, &txn4) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := + ld.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AssetCreatable}: {}, + {Index: 2, Type: basics.AssetCreatable}: {}, + {Index: 3, Type: basics.AssetCreatable}: {}, + }, + test.AccountB: { + {Index: 4, Type: basics.AssetCreatable}: {}, + }, + }) + require.NoError(t, err) + + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AssetCreatable}: {}, + ledger.Creatable{Index: 2, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 4, + Frozen: false, + }, + AssetParams: &basics.AssetParams{ + Total: 4, + Decimals: 0, + DefaultFrozen: false, + UnitName: "", + AssetName: "", + URL: "", + MetadataHash: [32]byte{}, + Manager: test.AccountA, + Reserve: test.AccountA, + Freeze: test.AccountA, + Clawback: test.AccountA, + }, + }, + ledger.Creatable{Index: 3, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 6, + Frozen: false, + }, + AssetParams: &basics.AssetParams{ + Total: 6, + Decimals: 0, + DefaultFrozen: false, + UnitName: "", + AssetName: "", + URL: "", + MetadataHash: [32]byte{}, + Manager: test.AccountA, + Reserve: test.AccountA, + Freeze: test.AccountA, + Clawback: test.AccountA, + }, + }, + }, + test.AccountB: { + ledger.Creatable{Index: 4, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 8, + Frozen: false, + }, + AssetParams: &basics.AssetParams{ + Total: 8, + Decimals: 0, + DefaultFrozen: false, + UnitName: "", + AssetName: "", + URL: "", + MetadataHash: [32]byte{}, + Manager: test.AccountB, + Reserve: test.AccountB, + Freeze: test.AccountB, + Clawback: test.AccountB, + }, + }, + }, + } + assert.Equal(t, expected, ret) +} + +func TestLedgerForEvaluatorApp(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAppCallTxn(0, test.AccountA) + txn1 := test.MakeAppCallTxnWithLogs(0, test.AccountA, []string{"testing"}) + txn2 := test.MakeAppCallWithInnerTxn(test.AccountA, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) + txn3 := test.MakeAppCallWithMultiLogs(test.AccountA) + txn4 := test.MakeAppDestroyTxn(1, test.AccountA) + txn5 := test.MakeAppCallTxn(0, test.AccountB) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3, &txn4, &txn5) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := + ld.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AppCreatable}: {}, + {Index: 2, Type: basics.AppCreatable}: {}, + {Index: 3, Type: basics.AppCreatable}: {}, + {Index: 4, Type: basics.AppCreatable}: {}, + }, + test.AccountB: { + {Index: 6, Type: basics.AppCreatable}: {}, + }, + }) + require.NoError(t, err) + + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AppCreatable}: {}, + ledger.Creatable{Index: 2, Type: basics.AppCreatable}: { + AppParams: &basics.AppParams{ + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, + GlobalState: nil, + StateSchemas: basics.StateSchemas{}, + ExtraProgramPages: 0, + }, + }, + ledger.Creatable{Index: 3, Type: basics.AppCreatable}: { + AppParams: &basics.AppParams{ + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, + GlobalState: nil, + StateSchemas: basics.StateSchemas{}, + ExtraProgramPages: 0, + }, + }, + ledger.Creatable{Index: 4, Type: basics.AppCreatable}: { + AppParams: &basics.AppParams{ + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, + GlobalState: nil, + StateSchemas: basics.StateSchemas{}, + ExtraProgramPages: 0, + }, + }, + }, + test.AccountB: { + ledger.Creatable{Index: 6, Type: basics.AppCreatable}: { + AppParams: &basics.AppParams{ + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, + GlobalState: nil, + StateSchemas: basics.StateSchemas{}, + ExtraProgramPages: 0, + }, + }, + }, + } + assert.Equal(t, expected, ret) +} + +func TestLedgerForEvaluatorAsset(t *testing.T) {} + +func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { +} + +func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { + +} + +func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { + +} + +func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { + +} + +func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { + +} + +func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { + +} + +func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { + +} + +func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { + +} + +func TestLedgerForEvaluatorAccountTotals(t *testing.T) { + +} + +func makeTestLedger(t *testing.T, prefix string) *ledger.Ledger { + // initialize local ledger + genesis := test.MakeGenesis() + genesisBlock := test.MakeGenesisBlock() + initState, err := util.CreateInitState(&genesis, &genesisBlock) + if err != nil { + log.Panicf("test init err: %v", err) + } + logger := logging.NewLogger() + l, err := ledger.OpenLedger(logger, prefix, true, initState, config.GetDefaultLocal()) + if err != nil { + log.Panicf("test init err: %v", err) + } + return l +} diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index dd1ae560d..d3945095b 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -195,8 +195,8 @@ func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, }, }, Sig: Signature, @@ -215,8 +215,10 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedT GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApplicationID: basics.AppIndex(appid), - OnCompletion: transactions.DeleteApplicationOC, + ApplicationID: basics.AppIndex(appid), + OnCompletion: transactions.DeleteApplicationOC, + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, }, }, Sig: Signature, @@ -275,8 +277,10 @@ func MakeAppCallTxn(appid uint64, sender basics.Address) transactions.SignedTxnW GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApplicationID: basics.AppIndex(appid), - OnCompletion: transactions.NoOpOC, + ApplicationID: basics.AppIndex(appid), + OnCompletion: transactions.NoOpOC, + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, }, }, Sig: Signature, @@ -327,8 +331,10 @@ func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSen Sender: assetSender, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApplicationID: 789, - OnCompletion: transactions.NoOpOC, + ApplicationID: 789, + OnCompletion: transactions.NoOpOC, + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, }, }, }, @@ -490,8 +496,8 @@ func MakeAppCallWithInnerAppCall(appSender basics.Address) transactions.SignedTx GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, }, }, Sig: Signature, From 8de6a18853556e685db5b41d1ab1b989dd1fd694 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 24 May 2022 16:47:31 -0400 Subject: [PATCH 13/37] more tests --- processor/ledger/ledger_for_evaluator_test.go | 94 +++++++++++++++++-- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/processor/ledger/ledger_for_evaluator_test.go b/processor/ledger/ledger_for_evaluator_test.go index 788204173..b8043d8f1 100644 --- a/processor/ledger/ledger_for_evaluator_test.go +++ b/processor/ledger/ledger_for_evaluator_test.go @@ -79,7 +79,7 @@ func TestLedgerForEvaluatorAccountDataMissingAccount(t *testing.T) { assert.Nil(t, accountDataRet) } -func TestLedgerForEvaluatorAccountAsset(t *testing.T) { +func TestLedgerForEvaluatorAsset(t *testing.T) { l := makeTestLedger(t, "ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -262,32 +262,106 @@ func TestLedgerForEvaluatorApp(t *testing.T) { assert.Equal(t, expected, ret) } -func TestLedgerForEvaluatorAsset(t *testing.T) {} - func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAppCallTxn(0, test.AccountA) + txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := + ld.LookupResources(map[basics.Address]map[ledger.Creatable]struct{}{ + test.AccountA: { + {Index: 1, Type: basics.AppCreatable}: {}, + {Index: 2, Type: basics.AssetCreatable}: {}, + }, + }) + require.NoError(t, err) + + expected := map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource{ + test.AccountA: { + ledger.Creatable{Index: 1, Type: basics.AppCreatable}: { + AppParams: &basics.AppParams{ + ApprovalProgram: []byte{0x06, 0x81, 0x01}, + ClearStateProgram: []byte{0x06, 0x81, 0x01}, + GlobalState: nil, + StateSchemas: basics.StateSchemas{}, + ExtraProgramPages: 0, + }, + }, + ledger.Creatable{Index: 2, Type: basics.AssetCreatable}: { + AssetHolding: &basics.AssetHolding{ + Amount: 2, + Frozen: false, + }, + AssetParams: &basics.AssetParams{ + Total: 2, + Decimals: 0, + DefaultFrozen: false, + UnitName: "", + AssetName: "", + URL: "", + MetadataHash: [32]byte{}, + Manager: test.AccountA, + Reserve: test.AccountA, + Freeze: test.AccountA, + Clawback: test.AccountA, + }, + }, + }, + } + assert.Equal(t, expected, ret) } func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + block_processor.MakeProcessor(l, nil) -} + addresses := []basics.Address{ + test.AccountA, test.AccountB, test.AccountC, test.AccountD} -func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { + addressesMap := make(map[basics.Address]struct{}) + for _, address := range addresses { + addressesMap[address] = struct{}{} + } + addressesMap[test.FeeAddr] = struct{}{} + addressesMap[test.RewardAddr] = struct{}{} -} + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() -func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { + ret, err := + ld.LookupWithoutRewards(addressesMap) + require.NoError(t, err) + for _, address := range addresses { + accountData, _ := ret[address] + require.NotNil(t, accountData) + } } -func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { +func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { } -func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { +func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { } -func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { +func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { } From 088434bc70bfb1a107ef69c5c7d4e84d6379601b Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 11:05:56 -0400 Subject: [PATCH 14/37] make integration working --- processor/ledger/ledger_for_evaluator.go | 4 +- processor/ledger/ledger_for_evaluator_test.go | 226 ++++++++++++++++++ 2 files changed, 228 insertions(+), 2 deletions(-) diff --git a/processor/ledger/ledger_for_evaluator.go b/processor/ledger/ledger_for_evaluator.go index 8dd0d0a03..9c00352d9 100644 --- a/processor/ledger/ledger_for_evaluator.go +++ b/processor/ledger/ledger_for_evaluator.go @@ -109,7 +109,7 @@ func (l LedgerForEvaluator) GetAssetCreator(indices map[basics.AssetIndex]struct res := make(map[basics.AssetIndex]ledger.FoundAddress, len(indices)) for _, index := range indicesArr { cidx := basics.CreatableIndex(index) - address, exists, err := l.Ledger.GetCreatorForRound(0, cidx, basics.AssetCreatable) + address, exists, err := l.Ledger.GetCreatorForRound(l.Ledger.Latest(), cidx, basics.AssetCreatable) if err != nil { return nil, fmt.Errorf("GetAssetCreator() err: %w", err) } @@ -129,7 +129,7 @@ func (l LedgerForEvaluator) GetAppCreator(indices map[basics.AppIndex]struct{}) res := make(map[basics.AppIndex]ledger.FoundAddress, len(indices)) for _, index := range indicesArr { cidx := basics.CreatableIndex(index) - address, exists, err := l.Ledger.GetCreatorForRound(0, cidx, basics.AssetCreatable) + address, exists, err := l.Ledger.GetCreatorForRound(l.Ledger.Latest(), cidx, basics.AppCreatable) if err != nil { return nil, fmt.Errorf("GetAppCreator() err: %w", err) } diff --git a/processor/ledger/ledger_for_evaluator_test.go b/processor/ledger/ledger_for_evaluator_test.go index b8043d8f1..9f5f087ad 100644 --- a/processor/ledger/ledger_for_evaluator_test.go +++ b/processor/ledger/ledger_for_evaluator_test.go @@ -354,22 +354,248 @@ func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { } func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + ret, err := ld.GetAssetCreator( + map[basics.AssetIndex]struct{}{basics.AssetIndex(1): {}}) + require.NoError(t, err) + + foundAddress, ok := ret[basics.AssetIndex(1)] + require.True(t, ok) + + expected := ledger.FoundAddress{ + Address: test.AccountA, + Exists: true, + } + assert.Equal(t, expected, foundAddress) +} + +func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) + txn1 := test.MakeAssetDestroyTxn(1, test.AccountA) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := ld.GetAssetCreator( + map[basics.AssetIndex]struct{}{basics.AssetIndex(1): {}}) + require.NoError(t, err) + + foundAddress, ok := ret[basics.AssetIndex(1)] + require.True(t, ok) + + assert.False(t, foundAddress.Exists) } func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) + txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountB) + txn2 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountC) + txn3 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountD) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + indices := map[basics.AssetIndex]struct{}{ + 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}, 7: {}, 8: {}} + ret, err := ld.GetAssetCreator(indices) + require.NoError(t, err) + + creatorsMap := map[basics.AssetIndex]basics.Address{ + 1: test.AccountA, + 2: test.AccountB, + 3: test.AccountC, + 4: test.AccountD, + } + + for i := 1; i <= 4; i++ { + index := basics.AssetIndex(i) + + foundAddress, ok := ret[index] + require.True(t, ok) + + expected := ledger.FoundAddress{ + Address: creatorsMap[index], + Exists: true, + } + assert.Equal(t, expected, foundAddress) + } + for i := 5; i <= 8; i++ { + index := basics.AssetIndex(i) + + foundAddress, ok := ret[index] + require.True(t, ok) + + assert.False(t, foundAddress.Exists) + } } func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAppCallTxn(0, test.AccountA) + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := ld.GetAppCreator( + map[basics.AppIndex]struct{}{basics.AppIndex(1): {}}) + require.NoError(t, err) + + foundAddress, ok := ret[basics.AppIndex(1)] + require.True(t, ok) + + expected := ledger.FoundAddress{ + Address: test.AccountA, + Exists: true, + } + assert.Equal(t, expected, foundAddress) + +} + +func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAppCallTxn(0, test.AccountA) + txn1 := test.MakeAppDestroyTxn(1, test.AccountA) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + ret, err := ld.GetAppCreator( + map[basics.AppIndex]struct{}{basics.AppIndex(1): {}}) + require.NoError(t, err) + + foundAddress, ok := ret[basics.AppIndex(1)] + require.True(t, ok) + + assert.False(t, foundAddress.Exists) } func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + pr, _ := block_processor.MakeProcessor(l, nil) + + txn0 := test.MakeAppCallTxn(0, test.AccountA) + txn1 := test.MakeAppCallTxn(0, test.AccountB) + txn2 := test.MakeAppCallTxn(0, test.AccountC) + txn3 := test.MakeAppCallTxn(0, test.AccountD) + + block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3) + assert.Nil(t, err) + rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} + err = pr.Process(&rawBlock) + assert.Nil(t, err) + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + creatorsMap := map[basics.AppIndex]basics.Address{ + 1: test.AccountA, + 2: test.AccountB, + 3: test.AccountC, + 4: test.AccountD, + } + + indices := map[basics.AppIndex]struct{}{ + 1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}, 7: {}, 8: {}} + ret, err := ld.GetAppCreator(indices) + require.NoError(t, err) + + assert.Equal(t, len(indices), len(ret)) + for i := 1; i <= 4; i++ { + index := basics.AppIndex(i) + + foundAddress, ok := ret[index] + require.True(t, ok) + + expected := ledger.FoundAddress{ + Address: creatorsMap[index], + Exists: true, + } + assert.Equal(t, expected, foundAddress) + } + for i := 5; i <= 8; i++ { + index := basics.AppIndex(i) + + foundAddress, ok := ret[index] + require.True(t, ok) + + assert.False(t, foundAddress.Exists) + } } func TestLedgerForEvaluatorAccountTotals(t *testing.T) { + l := makeTestLedger(t, "ledger") + defer l.Close() + + ld, err := indxLeder.MakeLedgerForEvaluator(l) + require.NoError(t, err) + defer ld.Close() + + accountTotalsRead, err := ld.LatestTotals() + require.NoError(t, err) + + _, total, _ := l.LatestTotals() + assert.Equal(t, total, accountTotalsRead) } From dfd2e8395b5f9734fabd215ad06a1baee4b869f6 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 11:20:54 -0400 Subject: [PATCH 15/37] fix linting errors --- processor/blockprocessor/block_processor.go | 2 +- processor/{ledger => eval}/ledger_for_evaluator.go | 2 +- processor/{ledger => eval}/ledger_for_evaluator_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename processor/{ledger => eval}/ledger_for_evaluator.go (99%) rename processor/{ledger => eval}/ledger_for_evaluator_test.go (99%) diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 8480f86af..b36d09e74 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -12,7 +12,7 @@ import ( "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" - indxledger "github.com/algorand/indexer/processor/ledger" + indxledger "github.com/algorand/indexer/processor/eval" ) type blockProcessor struct { diff --git a/processor/ledger/ledger_for_evaluator.go b/processor/eval/ledger_for_evaluator.go similarity index 99% rename from processor/ledger/ledger_for_evaluator.go rename to processor/eval/ledger_for_evaluator.go index 9c00352d9..c9b2c3707 100644 --- a/processor/ledger/ledger_for_evaluator.go +++ b/processor/eval/ledger_for_evaluator.go @@ -1,4 +1,4 @@ -package ledger +package eval import ( "fmt" diff --git a/processor/ledger/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go similarity index 99% rename from processor/ledger/ledger_for_evaluator_test.go rename to processor/eval/ledger_for_evaluator_test.go index 9f5f087ad..22fbb02f8 100644 --- a/processor/ledger/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -1,4 +1,4 @@ -package ledger_test +package eval_test import ( "crypto/rand" @@ -13,7 +13,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" block_processor "github.com/algorand/indexer/processor/blockprocessor" - indxLeder "github.com/algorand/indexer/processor/ledger" + indxLeder "github.com/algorand/indexer/processor/eval" "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/test" "github.com/stretchr/testify/assert" From a40789f285f4f2649c780dc936ccd9ffd945f09e Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 12:51:59 -0400 Subject: [PATCH 16/37] remove debugging lines --- processor/blockprocessor/block_processor.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index b36d09e74..f293f4102 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -42,18 +42,6 @@ func MakeProcessor(ledger *ledger.Ledger, handler func(block *ledgercore.Validat // Process a raw algod block func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { - apps, _ := proc.ledger.ListApplications(100, 100) - //paysets := blockCert.Block.Payset - - for _, app := range apps { - fmt.Printf("%v\n", app) - } - //for _, payset := range paysets { - // //if payset.Txn.ID().String() == "MM76PGEZHO5WRA3HJIVNKQE2Y7Y2GJS7CPIGL65MX43AUEUNYGEQ" { - // // fmt.Printf("%+v\n", payset) - // //} - // fmt.Println(payset.Txn.ID().String()) - //} if blockCert == nil { return fmt.Errorf("Process(): cannot process a nil block") From 1abe9f904e9e90cc25f41abaea6a40b3295ee99a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 13:14:25 -0400 Subject: [PATCH 17/37] error fix --- processor/blockprocessor/block_processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index f293f4102..6fc555935 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -56,6 +56,7 @@ func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { "Process() cannot find proto version %s", blockCert.Block.BlockHeader.CurrentProtocol) } protoChanged := !proto.EnableAssetCloseAmount + proto.EnableAssetCloseAmount = true ledgerForEval, err := indxledger.MakeLedgerForEvaluator(proc.ledger) if err != nil { From d0fe6a43f5b294a39860c172f6df141013147563 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 13:24:54 -0400 Subject: [PATCH 18/37] fix test failure --- cmd/algorand-indexer/daemon_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index f39efa85c..6372d6262 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -24,7 +24,7 @@ import ( type mockImporter struct { } -var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 processor.nextRoundToProcess: 1") +var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 proc.nextRoundToProcess: 1") func (imp *mockImporter) ImportBlock(blockContainer *rpcs.EncodedBlockCert) error { return nil From 39dda1bdbcd933d91ef605ec5059289b90445461 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 14:55:53 -0400 Subject: [PATCH 19/37] refactoring --- README.md | 2 +- api/handlers_e2e_test.go | 40 ++---- cmd/algorand-indexer/daemon.go | 37 ++--- cmd/algorand-indexer/daemon_test.go | 24 +--- cmd/import-validator/core/service.go | 15 +- idb/postgres/internal/writer/writer_test.go | 21 ++- idb/postgres/postgres.go | 143 ++++++++++---------- util/test/testutil.go | 18 +++ 8 files changed, 146 insertions(+), 154 deletions(-) diff --git a/README.md b/README.md index bbd237b32..c7c23fe8a 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Settings can be provided from the command line, a configuration file, or an envi | default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | | max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | | default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | -| data-dir | i | data-dir | INDEXER_DATA | +| data-dir | i | data-dir | INDEXER_DATA_DIR | ## Command line The command line arguments always take priority over the config file and environment variables. diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index d0b0c2a3c..049cfd32d 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -12,16 +12,12 @@ import ( "testing" "time" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" "github.com/labstack/echo/v4" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -79,20 +75,6 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin return db, newShutdownFunc } -func makeTestLedger(t *testing.T) *ledger.Ledger { - genesis := test.MakeGenesis() - genesisBlock := test.MakeGenesisBlock() - initState, err := util.CreateInitState(&genesis, &genesisBlock) - if err != nil { - logrus.Panicf("test init err: %v", err) - } - l, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) - if err != nil { - logrus.Panicf("test init err: %v", err) - } - return l -} - func TestApplicationHandlers(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() @@ -110,7 +92,6 @@ func TestApplicationHandlers(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, - LastValid: 10, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -130,7 +111,7 @@ func TestApplicationHandlers(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -261,7 +242,7 @@ func TestAccountExcludeParameters(t *testing.T) { &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -471,7 +452,7 @@ func TestAccountMaxResultsLimit(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -878,7 +859,7 @@ func TestInnerTxn(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -935,7 +916,7 @@ func TestPagingRootTxnDeduplication(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1042,7 +1023,7 @@ func TestPagingRootTxnDeduplication(t *testing.T) { require.NotNil(t, response.Transactions) require.Len(t, *response.Transactions, 1) require.NotNil(t, (*response.Transactions)[0]) - //require.Len(t, *(*response.Transactions)[0].InnerTxns, 2) + require.Len(t, *(*response.Transactions)[0].InnerTxns, 2) }) } @@ -1069,7 +1050,6 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, - LastValid: 10, }, KeyregTxnFields: transactions.KeyregTxnFields{ VotePK: votePK, @@ -1088,7 +1068,7 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1196,7 +1176,7 @@ func TestAccountClearsNonUTF8(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1322,7 +1302,7 @@ func TestLookupInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1425,7 +1405,7 @@ func TestLookupMultiInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 5be946c5f..f783ddf16 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "io" + "io/ioutil" "os" "os/signal" "path" @@ -138,12 +140,7 @@ var daemonCmd = &cobra.Command{ os.Exit(1) } wg.Add(1) - genesis, err := getGenesis(bot.Algod()) - maybeFail(err, "Error getting genesis") - genesisBlock, err := getGenesisBlock(bot.Algod()) - maybeFail(err, "Error getting genesis block") - initState, err := util.CreateInitState(&genesis, &genesisBlock) - maybeFail(err, "Error getting genesis block") + go func() { defer wg.Done() @@ -157,15 +154,23 @@ var daemonCmd = &cobra.Command{ logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) + + genesis, err := getGenesis(genesisReader) + maybeFail(err, "Error reading genesis") + genesisBlock, err := getGenesisBlock(bot.Algod()) + maybeFail(err, "Error getting genesis block") + initState, err := util.CreateInitState(&genesis, &genesisBlock) + maybeFail(err, "Error creating init state") + logger.Info("Initializing local ledger.") localLedger, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(indexerDataDir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) - maybeFail(err, "%v", err) + maybeFail(err, "failed to open ledger %v", err) defer localLedger.Close() bot.SetNextRound(uint64(localLedger.Latest()) + 1) proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportBlock) if err != nil { - maybeFail(err, err.Error()) + maybeFail(err, "blockprocessor.MakeProcessor() err %v", err) } handler := blockHandler(proc, 1*time.Second) bot.SetBlockHandler(handler) @@ -340,19 +345,17 @@ func handleBlock(block *rpcs.EncodedBlockCert, proc processor.Processor) error { return nil } -func getGenesis(client *algod.Client) (bookkeeping.Genesis, error) { - data, err := client.GetGenesis().Do(context.Background()) +func getGenesis(reader io.Reader) (bookkeeping.Genesis, error) { + var genesis bookkeeping.Genesis + gbytes, err := ioutil.ReadAll(reader) if err != nil { - return bookkeeping.Genesis{}, fmt.Errorf("getGenesis() client err: %w", err) + maybeFail(err, "error reading genesis, %v", err) } - - var res bookkeeping.Genesis - err = protocol.DecodeJSON([]byte(data), &res) + err = protocol.DecodeJSON(gbytes, &genesis) if err != nil { - return bookkeeping.Genesis{}, fmt.Errorf("getGenesis() decode err: %w", err) + maybeFail(err, "error reading genesis, %v", err) } - - return res, nil + return genesis, nil } func getGenesisBlock(client *algod.Client) (bookkeeping.Block, error) { diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index 6372d6262..d75e4e947 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -3,20 +3,15 @@ package main import ( "context" "errors" - "log" "sync" "testing" "time" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" - test_util "github.com/algorand/indexer/util/test" + itest "github.com/algorand/indexer/util/test" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" ) @@ -45,7 +40,7 @@ func TestImportRetryAndCancel(t *testing.T) { // create handler with mock importer and start, it should generate errors until cancelled. imp := &mockImporter{} - l := makeTestLedger(t, "local_ledger") + l := itest.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, nil) assert.Nil(t, err) @@ -79,18 +74,3 @@ func TestImportRetryAndCancel(t *testing.T) { cancel() wg.Wait() } - -func makeTestLedger(t *testing.T, prefix string) *ledger.Ledger { - // initialize local ledger - genesis := test_util.MakeGenesis() - genesisBlock := test_util.MakeGenesisBlock() - initState, err := util.CreateInitState(&genesis, &genesisBlock) - if err != nil { - log.Panicf("test init err: %v", err) - } - l, err := ledger.OpenLedger(logging.NewLogger(), prefix, true, initState, config.GetDefaultLocal()) - if err != nil { - log.Panicf("test init err: %v", err) - } - return l -} diff --git a/cmd/import-validator/core/service.go b/cmd/import-validator/core/service.go index fe341f20e..628699da5 100644 --- a/cmd/import-validator/core/service.go +++ b/cmd/import-validator/core/service.go @@ -350,18 +350,21 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg }() if nextRoundLedger >= nextRoundIndexer { + wg.Add(1) prc, err := blockprocessor.MakeProcessor(l, db.AddBlock) if err != nil { return fmt.Errorf("catchup() err: %w", err) } blockCert := rpcs.EncodedBlockCert{Block: block.Block, Certificate: block.Certificate} - start := time.Now() - prc.Process(&blockCert) - fmt.Printf( - "%d transactions imported in %v\n", - len(block.Block.Payset), time.Since(start)) + go func() { + start := time.Now() + err1 = prc.Process(&blockCert) + fmt.Printf( + "%d transactions imported in %v\n", + len(block.Block.Payset), time.Since(start)) + wg.Done() + }() } - wg.Wait() if err0 != nil { return fmt.Errorf("catchup() err0: %w", err0) diff --git a/idb/postgres/internal/writer/writer_test.go b/idb/postgres/internal/writer/writer_test.go index 5b8940ea7..3b0902c0e 100644 --- a/idb/postgres/internal/writer/writer_test.go +++ b/idb/postgres/internal/writer/writer_test.go @@ -189,13 +189,16 @@ func TestWriterTxnTableBasic(t *testing.T) { Payset: make([]transactions.SignedTxnInBlock, 2), } - stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) + stxnad0 := test.MakePaymentTxn( + 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, + basics.Address{}) var err error block.Payset[0], err = block.BlockHeader.EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA) + stxnad1 := test.MakeAssetConfigTxn( + 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountA) block.Payset[1], err = block.BlockHeader.EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -333,11 +336,14 @@ func TestWriterTxnParticipationTable(t *testing.T) { var tests []testtype { - stxnad0 := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) + stxnad0 := test.MakePaymentTxn( + 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, + basics.Address{}) stib0, err := makeBlockFunc().EncodeSignedTxn(stxnad0.SignedTxn, stxnad0.ApplyData) require.NoError(t, err) - stxnad1 := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC) + stxnad1 := test.MakeAssetConfigTxn( + 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountC) stib1, err := makeBlockFunc().EncodeSignedTxn(stxnad1.SignedTxn, stxnad1.ApplyData) require.NoError(t, err) @@ -624,7 +630,9 @@ func TestWriterDeleteAccountDoesNotDeleteKeytype(t *testing.T) { Payset: make(transactions.Payset, 1), } - stxnad := test.MakePaymentTxn(1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) + stxnad := test.MakePaymentTxn( + 1000, 1, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, + basics.Address{}) stxnad.Sig[0] = 5 // set signature so that keytype for account is updated var err error block.Payset[0], err = block.EncodeSignedTxn(stxnad.SignedTxn, stxnad.ApplyData) @@ -1335,7 +1343,8 @@ func TestWriterAddBlockInnerTxnsAssetCreate(t *testing.T) { appCall := test.MakeAppCallWithInnerTxn(test.AccountA, appAddr, test.AccountB, appAddr, test.AccountC) // Asset create call, should have intra = 5 - assetCreate := test.MakeAssetConfigTxn(0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD) + assetCreate := test.MakeAssetConfigTxn( + 0, 100, 1, false, "ma", "myasset", "myasset.com", test.AccountD) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall, &assetCreate) require.NoError(t, err) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 4434dfbff..f139d19b6 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -263,96 +263,95 @@ func (db *IndexerDb) AddBlock(vb *ledgercore.ValidatedBlock) error { if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - if block.Round() > basics.Round(importstate.NextRoundToAccount) { + if block.Round() != basics.Round(importstate.NextRoundToAccount) { return fmt.Errorf( "AddBlock() adding block round %d but next round to account is %d", block.Round(), importstate.NextRoundToAccount) - } else if block.Round() == basics.Round(importstate.NextRoundToAccount) { - importstate.NextRoundToAccount++ - err = db.setImportState(tx, &importstate) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } + } + importstate.NextRoundToAccount++ + err = db.setImportState(tx, &importstate) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) + } - w, err := writer.MakeWriter(tx) + w, err := writer.MakeWriter(tx) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) + } + defer w.Close() + + if block.Round() == basics.Round(0) { + // Block 0 is special, we cannot run the evaluator on it. + err := w.AddBlock0(&block) if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - defer w.Close() - - if block.Round() == basics.Round(0) { - // Block 0 is special, we cannot run the evaluator on it. - err := w.AddBlock0(&block) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - } else { - proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] - if !ok { - return fmt.Errorf( - "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + } else { + proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] + if !ok { + return fmt.Errorf( + "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + } + protoChanged := !proto.EnableAssetCloseAmount + proto.EnableAssetCloseAmount = true + + var wg sync.WaitGroup + defer wg.Wait() + + // Write transaction participation and possibly transactions in a parallel db + // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start + // writing transactions contained in the block early. + var err0 error + wg.Add(1) + go func() { + defer wg.Done() + + f := func(tx pgx.Tx) error { + if !protoChanged { + err := writer.AddTransactions(&block, block.Payset, tx) + if err != nil { + return err + } + } + return writer.AddTransactionParticipation(&block, tx) } - protoChanged := !proto.EnableAssetCloseAmount - proto.EnableAssetCloseAmount = true - - var wg sync.WaitGroup - defer wg.Wait() + err0 = db.txWithRetry(serializable, f) + }() - // Write transaction participation and possibly transactions in a parallel db - // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start - // writing transactions contained in the block early. - var err0 error + var err1 error + // Skip if transaction writing has already started. + if protoChanged { + // Write transactions in a parallel db transaction. wg.Add(1) go func() { defer wg.Done() f := func(tx pgx.Tx) error { - if !protoChanged { - err := writer.AddTransactions(&block, block.Payset, tx) - if err != nil { - return err - } - } - return writer.AddTransactionParticipation(&block, tx) + return writer.AddTransactions(&block, block.Payset, tx) } - err0 = db.txWithRetry(serializable, f) + err1 = db.txWithRetry(serializable, f) }() + } - var err1 error - // Skip if transaction writing has already started. - if protoChanged { - // Write transactions in a parallel db transaction. - wg.Add(1) - go func() { - defer wg.Done() - - f := func(tx pgx.Tx) error { - return writer.AddTransactions(&block, block.Payset, tx) - } - err1 = db.txWithRetry(serializable, f) - }() - } - - err = w.AddBlock(&block, block.Payset, vb.Delta()) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } + err = w.AddBlock(&block, block.Payset, vb.Delta()) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) + } - // Wait for goroutines to finish and check for errors. If there is an error, we - // return our own error so that the main transaction does not commit. Hence, - // `txn` and `txn_participation` tables can only be ahead but not behind - // the other state. - wg.Wait() - isUniqueViolationFunc := func(err error) bool { - var pgerr *pgconn.PgError - return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) - } - if (err0 != nil) && !isUniqueViolationFunc(err0) { - return fmt.Errorf("AddBlock() err0: %w", err0) - } - if (err1 != nil) && !isUniqueViolationFunc(err1) { - return fmt.Errorf("AddBlock() err1: %w", err1) - } + // Wait for goroutines to finish and check for errors. If there is an error, we + // return our own error so that the main transaction does not commit. Hence, + // `txn` and `txn_participation` tables can only be ahead but not behind + // the other state. + wg.Wait() + isUniqueViolationFunc := func(err error) bool { + var pgerr *pgconn.PgError + return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) + } + if (err0 != nil) && !isUniqueViolationFunc(err0) { + return fmt.Errorf("AddBlock() err0: %w", err0) + } + if (err1 != nil) && !isUniqueViolationFunc(err1) { + return fmt.Errorf("AddBlock() err1: %w", err1) } } return nil diff --git a/util/test/testutil.go b/util/test/testutil.go index d97f731ed..e73c9b59c 100644 --- a/util/test/testutil.go +++ b/util/test/testutil.go @@ -7,7 +7,11 @@ import ( "os" "runtime" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/logging" + "github.com/sirupsen/logrus" "github.com/algorand/indexer/idb" "github.com/algorand/indexer/util" @@ -114,3 +118,17 @@ func PrintTxnQuery(db idb.IndexerDb, q idb.TransactionFilter) { exitValue = 1 } } + +func MakeTestLedger(prefix string) *ledger.Ledger { + genesis := MakeGenesis() + genesisBlock := MakeGenesisBlock() + initState, err := util.CreateInitState(&genesis, &genesisBlock) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + l, err := ledger.OpenLedger(logging.NewLogger(), prefix, true, initState, config.GetDefaultLocal()) + if err != nil { + logrus.Panicf("test init err: %v", err) + } + return l +} From 1e7c0a6f16a5b81ed4c75bd520287cfdc1f22e8a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 15:20:47 -0400 Subject: [PATCH 20/37] more refactoring --- cmd/algorand-indexer/daemon.go | 6 +- .../postgres_integration_common_test.go | 9 ++- idb/postgres/postgres_integration_test.go | 3 - importer/helper.go | 25 +++---- processor/eval/ledger_for_evaluator_test.go | 68 +++++++------------ util/test/account_testutil.go | 20 +++--- util/test/testutil.go | 1 + 7 files changed, 50 insertions(+), 82 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index f783ddf16..0a876855f 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -140,7 +140,6 @@ var daemonCmd = &cobra.Command{ os.Exit(1) } wg.Add(1) - go func() { defer wg.Done() @@ -151,11 +150,10 @@ var daemonCmd = &cobra.Command{ genesisReader := importer.GetGenesisFile(genesisJSONPath, bot.Algod(), logger) _, err := importer.EnsureInitialImport(db, genesisReader, logger) maybeFail(err, "importer.EnsureInitialImport() error") - logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) - genesis, err := getGenesis(genesisReader) + genesis, err := readGenesis(genesisReader) maybeFail(err, "Error reading genesis") genesisBlock, err := getGenesisBlock(bot.Algod()) maybeFail(err, "Error getting genesis block") @@ -345,7 +343,7 @@ func handleBlock(block *rpcs.EncodedBlockCert, proc processor.Processor) error { return nil } -func getGenesis(reader io.Reader) (bookkeeping.Genesis, error) { +func readGenesis(reader io.Reader) (bookkeeping.Genesis, error) { var genesis bookkeeping.Genesis gbytes, err := ioutil.ReadAll(reader) if err != nil { diff --git a/idb/postgres/postgres_integration_common_test.go b/idb/postgres/postgres_integration_common_test.go index 9546b6466..e0bd4aa92 100644 --- a/idb/postgres/postgres_integration_common_test.go +++ b/idb/postgres/postgres_integration_common_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/require" @@ -19,10 +18,10 @@ func setupIdbWithConnectionString(t *testing.T, connStr string, genesis bookkeep err = idb.LoadGenesis(genesis) require.NoError(t, err) - - vb := ledgercore.MakeValidatedBlock(genesisBlock, ledgercore.StateDelta{}) - err = idb.AddBlock(&vb) - require.NoError(t, err) + // + //vb := ledgercore.MakeValidatedBlock(genesisBlock, ledgercore.StateDelta{}) + //err = idb.AddBlock(&vb) + //require.NoError(t, err) return idb } diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index e38399c35..8b9e8e4f7 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -746,7 +746,6 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, - LastValid: 10, }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ AssetParams: basics.AssetParams{ @@ -907,7 +906,6 @@ func TestAppExtraPages(t *testing.T) { Header: transactions.Header{ Sender: test.AccountA, GenesisHash: test.GenesisHash, - LastValid: 10, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, @@ -1286,7 +1284,6 @@ func TestReconfigAsset(t *testing.T) { Sender: test.AccountA, Fee: basics.MicroAlgos{Raw: 1000}, GenesisHash: test.GenesisHash, - LastValid: 10, }, AssetConfigTxnFields: transactions.AssetConfigTxnFields{ ConfigAsset: basics.AssetIndex(assetID), diff --git a/importer/helper.go b/importer/helper.go index 0c93f552b..f7197ff78 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -131,8 +131,15 @@ func importTar(imp Importer, tarfile io.Reader, l *log.Logger, genesisReader io. } sort.Slice(blocks, less) - genesis, err := getGenesis(genesisReader) - maybeFail(err, l, "Error getting genesis") + var genesis bookkeeping.Genesis + gbytes, err := ioutil.ReadAll(genesisReader) + if err != nil { + maybeFail(err, l, "error reading genesis, %v", err) + } + err = protocol.DecodeJSON(gbytes, &genesis) + if err != nil { + maybeFail(err, l, "error decoding genesis, %v", err) + } genesisBlock := blocks[0] initState, err := util.CreateInitState(&genesis, &genesisBlock.Block) maybeFail(err, l, "Error getting genesis block") @@ -302,17 +309,3 @@ func checkGenesisHash(db idb.IndexerDb, genesisReader io.Reader) error { } return nil } - -func getGenesis(in io.Reader) (bookkeeping.Genesis, error) { - var genesis bookkeeping.Genesis - gbytes, err := ioutil.ReadAll(in) - if err != nil { - return bookkeeping.Genesis{}, fmt.Errorf("error reading genesis, %v", err) - } - err = protocol.DecodeJSON(gbytes, &genesis) - if err != nil { - return bookkeeping.Genesis{}, fmt.Errorf("error decoding genesis, %v", err) - } - - return genesis, nil -} diff --git a/processor/eval/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go index 22fbb02f8..c90748fc3 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -2,19 +2,15 @@ package eval_test import ( "crypto/rand" - "log" "testing" "github.com/algorand/go-algorand/agreement" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" block_processor "github.com/algorand/indexer/processor/blockprocessor" indxLeder "github.com/algorand/indexer/processor/eval" - "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,7 +18,7 @@ import ( func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) txn := test.MakePaymentTxn(0, 100, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) @@ -43,7 +39,7 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { } func TestLedgerForEvaluatorAccountDataBasic(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() block_processor.MakeProcessor(l, nil) accountData, _, err := l.LookupWithoutRewards(0, test.AccountB) @@ -63,7 +59,7 @@ func TestLedgerForEvaluatorAccountDataBasic(t *testing.T) { } func TestLedgerForEvaluatorAccountDataMissingAccount(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() ld, err := indxLeder.MakeLedgerForEvaluator(l) require.NoError(t, err) @@ -80,7 +76,7 @@ func TestLedgerForEvaluatorAccountDataMissingAccount(t *testing.T) { } func TestLedgerForEvaluatorAsset(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -181,7 +177,7 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { } func TestLedgerForEvaluatorApp(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -221,8 +217,8 @@ func TestLedgerForEvaluatorApp(t *testing.T) { ledger.Creatable{Index: 1, Type: basics.AppCreatable}: {}, ledger.Creatable{Index: 2, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, GlobalState: nil, StateSchemas: basics.StateSchemas{}, ExtraProgramPages: 0, @@ -230,8 +226,8 @@ func TestLedgerForEvaluatorApp(t *testing.T) { }, ledger.Creatable{Index: 3, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, GlobalState: nil, StateSchemas: basics.StateSchemas{}, ExtraProgramPages: 0, @@ -239,8 +235,8 @@ func TestLedgerForEvaluatorApp(t *testing.T) { }, ledger.Creatable{Index: 4, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, GlobalState: nil, StateSchemas: basics.StateSchemas{}, ExtraProgramPages: 0, @@ -250,8 +246,8 @@ func TestLedgerForEvaluatorApp(t *testing.T) { test.AccountB: { ledger.Creatable{Index: 6, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, GlobalState: nil, StateSchemas: basics.StateSchemas{}, ExtraProgramPages: 0, @@ -263,7 +259,7 @@ func TestLedgerForEvaluatorApp(t *testing.T) { } func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -293,8 +289,8 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { test.AccountA: { ledger.Creatable{Index: 1, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, GlobalState: nil, StateSchemas: basics.StateSchemas{}, ExtraProgramPages: 0, @@ -325,7 +321,7 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { } func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() block_processor.MakeProcessor(l, nil) @@ -354,7 +350,7 @@ func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { } func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -385,7 +381,7 @@ func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { } func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -414,7 +410,7 @@ func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -468,7 +464,7 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { } func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -500,7 +496,7 @@ func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { } func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -529,7 +525,7 @@ func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) @@ -584,7 +580,7 @@ func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { } func TestLedgerForEvaluatorAccountTotals(t *testing.T) { - l := makeTestLedger(t, "ledger") + l := test.MakeTestLedger("ledger") defer l.Close() ld, err := indxLeder.MakeLedgerForEvaluator(l) @@ -598,19 +594,3 @@ func TestLedgerForEvaluatorAccountTotals(t *testing.T) { assert.Equal(t, total, accountTotalsRead) } - -func makeTestLedger(t *testing.T, prefix string) *ledger.Ledger { - // initialize local ledger - genesis := test.MakeGenesis() - genesisBlock := test.MakeGenesisBlock() - initState, err := util.CreateInitState(&genesis, &genesisBlock) - if err != nil { - log.Panicf("test init err: %v", err) - } - logger := logging.NewLogger() - l, err := ledger.OpenLedger(logger, prefix, true, initState, config.GetDefaultLocal()) - if err != nil { - log.Panicf("test init err: %v", err) - } - return l -} diff --git a/util/test/account_testutil.go b/util/test/account_testutil.go index d3945095b..17409a8a6 100644 --- a/util/test/account_testutil.go +++ b/util/test/account_testutil.go @@ -195,8 +195,8 @@ func MakeCreateAppTxn(sender basics.Address) transactions.SignedTxnWithAD { GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, }, }, Sig: Signature, @@ -217,8 +217,8 @@ func MakeAppDestroyTxn(appid uint64, sender basics.Address) transactions.SignedT ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), OnCompletion: transactions.DeleteApplicationOC, - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, }, }, Sig: Signature, @@ -279,8 +279,8 @@ func MakeAppCallTxn(appid uint64, sender basics.Address) transactions.SignedTxnW ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: basics.AppIndex(appid), OnCompletion: transactions.NoOpOC, - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, }, }, Sig: Signature, @@ -333,8 +333,8 @@ func MakeAppCallWithInnerTxn(appSender, paymentSender, paymentReceiver, assetSen ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ ApplicationID: 789, OnCompletion: transactions.NoOpOC, - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, }, }, }, @@ -496,8 +496,8 @@ func MakeAppCallWithInnerAppCall(appSender basics.Address) transactions.SignedTx GenesisHash: GenesisHash, }, ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{ - ApprovalProgram: []byte{0x06, 0x81, 0x01}, - ClearStateProgram: []byte{0x06, 0x81, 0x01}, + ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, }, }, Sig: Signature, diff --git a/util/test/testutil.go b/util/test/testutil.go index e73c9b59c..e8c671e43 100644 --- a/util/test/testutil.go +++ b/util/test/testutil.go @@ -119,6 +119,7 @@ func PrintTxnQuery(db idb.IndexerDb, q idb.TransactionFilter) { } } +// MakeTestLedger creates an in-memory local ledger func MakeTestLedger(prefix string) *ledger.Ledger { genesis := MakeGenesis() genesisBlock := MakeGenesisBlock() From 2ca22f36adeefb224a10598a237e6dc124d02e84 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 15:38:49 -0400 Subject: [PATCH 21/37] fix failed test --- idb/postgres/postgres_integration_common_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/idb/postgres/postgres_integration_common_test.go b/idb/postgres/postgres_integration_common_test.go index e0bd4aa92..9546b6466 100644 --- a/idb/postgres/postgres_integration_common_test.go +++ b/idb/postgres/postgres_integration_common_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/require" @@ -18,10 +19,10 @@ func setupIdbWithConnectionString(t *testing.T, connStr string, genesis bookkeep err = idb.LoadGenesis(genesis) require.NoError(t, err) - // - //vb := ledgercore.MakeValidatedBlock(genesisBlock, ledgercore.StateDelta{}) - //err = idb.AddBlock(&vb) - //require.NoError(t, err) + + vb := ledgercore.MakeValidatedBlock(genesisBlock, ledgercore.StateDelta{}) + err = idb.AddBlock(&vb) + require.NoError(t, err) return idb } From c4ab18166c230915ca86b48f666badc4f19e3222 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 16:00:39 -0400 Subject: [PATCH 22/37] fix --- idb/postgres/postgres.go | 143 ++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index f139d19b6..4434dfbff 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -263,95 +263,96 @@ func (db *IndexerDb) AddBlock(vb *ledgercore.ValidatedBlock) error { if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - if block.Round() != basics.Round(importstate.NextRoundToAccount) { + if block.Round() > basics.Round(importstate.NextRoundToAccount) { return fmt.Errorf( "AddBlock() adding block round %d but next round to account is %d", block.Round(), importstate.NextRoundToAccount) - } - importstate.NextRoundToAccount++ - err = db.setImportState(tx, &importstate) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - - w, err := writer.MakeWriter(tx) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } - defer w.Close() - - if block.Round() == basics.Round(0) { - // Block 0 is special, we cannot run the evaluator on it. - err := w.AddBlock0(&block) + } else if block.Round() == basics.Round(importstate.NextRoundToAccount) { + importstate.NextRoundToAccount++ + err = db.setImportState(tx, &importstate) if err != nil { return fmt.Errorf("AddBlock() err: %w", err) } - } else { - proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] - if !ok { - return fmt.Errorf( - "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + + w, err := writer.MakeWriter(tx) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) } - protoChanged := !proto.EnableAssetCloseAmount - proto.EnableAssetCloseAmount = true - - var wg sync.WaitGroup - defer wg.Wait() - - // Write transaction participation and possibly transactions in a parallel db - // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start - // writing transactions contained in the block early. - var err0 error - wg.Add(1) - go func() { - defer wg.Done() - - f := func(tx pgx.Tx) error { - if !protoChanged { - err := writer.AddTransactions(&block, block.Payset, tx) - if err != nil { - return err - } - } - return writer.AddTransactionParticipation(&block, tx) + defer w.Close() + + if block.Round() == basics.Round(0) { + // Block 0 is special, we cannot run the evaluator on it. + err := w.AddBlock0(&block) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) } - err0 = db.txWithRetry(serializable, f) - }() + } else { + proto, ok := config.Consensus[block.BlockHeader.CurrentProtocol] + if !ok { + return fmt.Errorf( + "AddBlock() cannot find proto version %s", block.BlockHeader.CurrentProtocol) + } + protoChanged := !proto.EnableAssetCloseAmount + proto.EnableAssetCloseAmount = true + + var wg sync.WaitGroup + defer wg.Wait() - var err1 error - // Skip if transaction writing has already started. - if protoChanged { - // Write transactions in a parallel db transaction. + // Write transaction participation and possibly transactions in a parallel db + // transaction. If `proto.EnableAssetCloseAmount` is already true, we can start + // writing transactions contained in the block early. + var err0 error wg.Add(1) go func() { defer wg.Done() f := func(tx pgx.Tx) error { - return writer.AddTransactions(&block, block.Payset, tx) + if !protoChanged { + err := writer.AddTransactions(&block, block.Payset, tx) + if err != nil { + return err + } + } + return writer.AddTransactionParticipation(&block, tx) } - err1 = db.txWithRetry(serializable, f) + err0 = db.txWithRetry(serializable, f) }() - } - err = w.AddBlock(&block, block.Payset, vb.Delta()) - if err != nil { - return fmt.Errorf("AddBlock() err: %w", err) - } + var err1 error + // Skip if transaction writing has already started. + if protoChanged { + // Write transactions in a parallel db transaction. + wg.Add(1) + go func() { + defer wg.Done() - // Wait for goroutines to finish and check for errors. If there is an error, we - // return our own error so that the main transaction does not commit. Hence, - // `txn` and `txn_participation` tables can only be ahead but not behind - // the other state. - wg.Wait() - isUniqueViolationFunc := func(err error) bool { - var pgerr *pgconn.PgError - return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) - } - if (err0 != nil) && !isUniqueViolationFunc(err0) { - return fmt.Errorf("AddBlock() err0: %w", err0) - } - if (err1 != nil) && !isUniqueViolationFunc(err1) { - return fmt.Errorf("AddBlock() err1: %w", err1) + f := func(tx pgx.Tx) error { + return writer.AddTransactions(&block, block.Payset, tx) + } + err1 = db.txWithRetry(serializable, f) + }() + } + + err = w.AddBlock(&block, block.Payset, vb.Delta()) + if err != nil { + return fmt.Errorf("AddBlock() err: %w", err) + } + + // Wait for goroutines to finish and check for errors. If there is an error, we + // return our own error so that the main transaction does not commit. Hence, + // `txn` and `txn_participation` tables can only be ahead but not behind + // the other state. + wg.Wait() + isUniqueViolationFunc := func(err error) bool { + var pgerr *pgconn.PgError + return errors.As(err, &pgerr) && (pgerr.Code == pgerrcode.UniqueViolation) + } + if (err0 != nil) && !isUniqueViolationFunc(err0) { + return fmt.Errorf("AddBlock() err0: %w", err0) + } + if (err1 != nil) && !isUniqueViolationFunc(err1) { + return fmt.Errorf("AddBlock() err1: %w", err1) + } } } return nil From 4d38249322690876f4bf607e34a0e7b1e609a908 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Wed, 25 May 2022 16:21:39 -0400 Subject: [PATCH 23/37] fix failing test --- cmd/algorand-indexer/daemon.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 0a876855f..2c0cfdb77 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -153,8 +153,9 @@ var daemonCmd = &cobra.Command{ logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) + genesisReader = importer.GetGenesisFile(genesisJSONPath, bot.Algod(), logger) genesis, err := readGenesis(genesisReader) - maybeFail(err, "Error reading genesis") + maybeFail(err, "Error reading genesis file") genesisBlock, err := getGenesisBlock(bot.Algod()) maybeFail(err, "Error getting genesis block") initState, err := util.CreateInitState(&genesis, &genesisBlock) @@ -347,11 +348,11 @@ func readGenesis(reader io.Reader) (bookkeeping.Genesis, error) { var genesis bookkeeping.Genesis gbytes, err := ioutil.ReadAll(reader) if err != nil { - maybeFail(err, "error reading genesis, %v", err) + return bookkeeping.Genesis{}, fmt.Errorf("readGenesis() err: %w", err) } err = protocol.DecodeJSON(gbytes, &genesis) if err != nil { - maybeFail(err, "error reading genesis, %v", err) + return bookkeeping.Genesis{}, fmt.Errorf("readGenesis() err: %w", err) } return genesis, nil } From 6ea76debea68f7f385c75475b4b985040dd163cc Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Thu, 26 May 2022 13:01:42 -0400 Subject: [PATCH 24/37] ready for review --- cmd/algorand-indexer/daemon.go | 3 + cmd/algorand-indexer/daemon_test.go | 40 +++- cmd/import-validator/core/service.go | 1 + idb/postgres/postgres.go | 1 + idb/postgres/postgres_integration_test.go | 246 +++++++++++--------- processor/blockprocessor/block_processor.go | 10 +- processor/eval/ledger_for_evaluator_test.go | 3 +- 7 files changed, 184 insertions(+), 120 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 2c0cfdb77..b25b6e6ed 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -346,6 +346,9 @@ func handleBlock(block *rpcs.EncodedBlockCert, proc processor.Processor) error { func readGenesis(reader io.Reader) (bookkeeping.Genesis, error) { var genesis bookkeeping.Genesis + if reader == nil { + return bookkeeping.Genesis{}, fmt.Errorf("readGenesis() err: reader is nil") + } gbytes, err := ioutil.ReadAll(reader) if err != nil { return bookkeeping.Genesis{}, fmt.Errorf("readGenesis() err: %w", err) diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index d75e4e947..ba553aab3 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -3,10 +3,13 @@ package main import ( "context" "errors" + "io" + "strings" "sync" "testing" "time" + "github.com/algorand/go-algorand-sdk/encoding/json" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" @@ -21,14 +24,10 @@ type mockImporter struct { var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 proc.nextRoundToProcess: 1") -func (imp *mockImporter) ImportBlock(blockContainer *rpcs.EncodedBlockCert) error { +func (imp *mockImporter) ImportBlock(vb *ledgercore.ValidatedBlock) error { return nil } -func (imp *mockImporter) ImportValidatedBlock(vb *ledgercore.ValidatedBlock) error { - return errMockImportBlock -} - func TestImportRetryAndCancel(t *testing.T) { // connect debug logger nullLogger, hook := test.NewNullLogger() @@ -44,7 +43,7 @@ func TestImportRetryAndCancel(t *testing.T) { defer l.Close() proc, err := blockprocessor.MakeProcessor(l, nil) assert.Nil(t, err) - proc.SetHandler(imp.ImportValidatedBlock) + proc.SetHandler(imp.ImportBlock) handler := blockHandler(proc, 50*time.Millisecond) var wg sync.WaitGroup wg.Add(1) @@ -74,3 +73,32 @@ func TestImportRetryAndCancel(t *testing.T) { cancel() wg.Wait() } + +func TestReadGenesis(t *testing.T) { + var reader io.Reader + // nil reader + _, err := readGenesis(reader) + assert.Contains(t, err.Error(), "readGenesis() err: reader is nil") + // no match struct field + genesisStr := "{\"version\": 2}" + reader = strings.NewReader(genesisStr) + _, err = readGenesis(reader) + assert.Contains(t, err.Error(), "json decode error") + + genesis := bookkeeping.Genesis{ + SchemaID: "1", + Network: "test", + Proto: "test", + RewardsPool: "AAAA", + FeeSink: "AAAA", + } + + // read and decode genesis + reader = strings.NewReader(string(json.Encode(genesis))) + _, err = readGenesis(reader) + assert.Nil(t, err) + // read from empty reader + _, err = readGenesis(reader) + assert.Contains(t, err.Error(), "readGenesis() err: EOF") + +} diff --git a/cmd/import-validator/core/service.go b/cmd/import-validator/core/service.go index 628699da5..af538105e 100644 --- a/cmd/import-validator/core/service.go +++ b/cmd/import-validator/core/service.go @@ -365,6 +365,7 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg wg.Done() }() } + wg.Wait() if err0 != nil { return fmt.Errorf("catchup() err0: %w", err0) diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index 4434dfbff..0fca17ce6 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -355,6 +355,7 @@ func (db *IndexerDb) AddBlock(vb *ledgercore.ValidatedBlock) error { } } } + return nil } err := db.txWithRetry(serializable, f) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index 8b9e8e4f7..bf46ffe19 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -10,12 +10,9 @@ import ( "testing" "time" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" - "github.com/algorand/go-algorand/ledger" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/go-codec/codec" @@ -25,7 +22,6 @@ import ( pgutil "github.com/algorand/indexer/idb/postgres/internal/util" "github.com/algorand/indexer/importer" "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" "github.com/sirupsen/logrus" @@ -37,20 +33,6 @@ import ( "github.com/algorand/indexer/util/test" ) -func makeTestLedger(t *testing.T) *ledger.Ledger { - genesis := test.MakeGenesis() - genesisBlock := test.MakeGenesisBlock() - initState, err := util.CreateInitState(&genesis, &genesisBlock) - if err != nil { - logrus.Panicf("test init err: %v", err) - } - l, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) - if err != nil { - logrus.Panicf("test init err: %v", err) - } - return l -} - // TestMaxRoundOnUninitializedDB makes sure we return 0 when getting the max round on a new DB. func TestMaxRoundOnUninitializedDB(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) @@ -183,18 +165,17 @@ func TestAssetCloseReopenTransfer(t *testing.T) { optInB := test.MakeAssetOptInTxn(assetid, test.AccountB) optInC := test.MakeAssetOptInTxn(assetid, test.AccountC) closeA := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC) - optInDA := test.MakeAssetOptInTxn(assetid, test.AccountA) payMain := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createAsset, &optInA, &fundA, &optInB, - &optInC, &closeA, &optInDA, &payMain) + &optInC, &closeA, &optInA, &payMain) require.NoError(t, err) ////////// // When // We commit the block to the database ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -215,7 +196,7 @@ func TestAssetCloseReopenTransfer(t *testing.T) { assertAccountAsset(t, db.db, test.AccountD, assetid, false, total-2*amt) } -//TestReCreateAssetHolding checks the optin value of a defunct +// TestReCreateAssetHolding checks the optin value of a defunct func TestReCreateAssetHolding(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() @@ -230,22 +211,25 @@ func TestReCreateAssetHolding(t *testing.T) { // A new asset with default-frozen, AccountB opts-in and has its frozen state // toggled. /////////// Then AccountB opts-out then opts-in again. - createAssetFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), frozen, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) + createAssetFrozen := test.MakeAssetConfigTxn( + 0, total, uint64(6), frozen, "icicles", "frozen coin", + "http://antarctica.com", test.AccountA) optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) - unfreezeB := test.MakeAssetFreezeTxn(assetid, !frozen, test.AccountA, test.AccountB) - optoutB := test.MakeAssetTransferTxn(assetid, 0, test.AccountB, test.AccountC, test.AccountD) - optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB) + unfreezeB := test.MakeAssetFreezeTxn( + assetid, !frozen, test.AccountA, test.AccountB) + optoutB := test.MakeAssetTransferTxn( + assetid, 0, test.AccountB, test.AccountC, test.AccountD) var err error block, err = test.MakeBlockForTxns( block.BlockHeader, &createAssetFrozen, &optinB, &unfreezeB, - &optoutB, &optinB2) + &optoutB, &optinB) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -272,19 +256,20 @@ func TestNoopOptins(t *testing.T) { /////////// assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountD) + createAsset := test.MakeAssetConfigTxn( + 0, uint64(1000000), uint64(6), true, "icicles", "frozen coin", + "http://antarctica.com", test.AccountD) optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) unfreezeB := test.MakeAssetFreezeTxn(assetid, false, test.AccountD, test.AccountB) - optinB2 := test.MakeAssetOptInTxn(assetid, test.AccountB) block, err := test.MakeBlockForTxns( - test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &unfreezeB, &optinB2) + test.MakeGenesisBlock().BlockHeader, &createAsset, &optinB, &unfreezeB, &optinB) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -303,16 +288,18 @@ func TestMultipleWriters(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - amt := uint64(100000) //min balance + amt := uint64(10000) /////////// // Given // Send amt to AccountE /////////// - payAccountE := test.MakePaymentTxn(1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, basics.Address{}) + payAccountE := test.MakePaymentTxn( + 1000, amt, 0, 0, 0, 0, test.AccountD, test.AccountE, basics.Address{}, + basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &payAccountE) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -370,20 +357,26 @@ func TestBlockWithTransactions(t *testing.T) { /////////// // Given // A block at round `round` with 5 transactions. /////////// - txn1 := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountD) + txn1 := test.MakeAssetConfigTxn( + 0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", + test.AccountD) txn2 := test.MakeAssetOptInTxn(assetid, test.AccountA) - txn3 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + txn3 := test.MakeAssetTransferTxn( + assetid, amt, test.AccountD, test.AccountA, basics.Address{}) txn4 := test.MakeAssetOptInTxn(assetid, test.AccountB) txn5 := test.MakeAssetOptInTxn(assetid, test.AccountC) - txn6 := test.MakeAssetTransferTxn(assetid, 1000, test.AccountA, test.AccountB, test.AccountC) - txn7 := test.MakeAssetTransferTxn(assetid, 0, test.AccountA, test.AccountA, basics.Address{}) - txn8 := test.MakeAssetTransferTxn(assetid, amt, test.AccountD, test.AccountA, basics.Address{}) + txn6 := test.MakeAssetTransferTxn( + assetid, 1000, test.AccountA, test.AccountB, test.AccountC) + txn7 := test.MakeAssetTransferTxn( + assetid, 0, test.AccountA, test.AccountA, basics.Address{}) + txn8 := test.MakeAssetTransferTxn( + assetid, amt, test.AccountD, test.AccountA, basics.Address{}) txns := []*transactions.SignedTxnWithAD{ &txn1, &txn2, &txn3, &txn4, &txn5, &txn6, &txn7, &txn8} block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, txns...) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -423,11 +416,12 @@ func TestRekeyBasic(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) + txn := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -460,7 +454,7 @@ func TestRekeyToItself(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -488,7 +482,6 @@ func TestRekeyToItself(t *testing.T) { ad, err := encoding.DecodeTrimmedLcAccountData(accountDataStr) require.NoError(t, err, "failed to parse account data json") - fmt.Println("addr:", ad.AuthAddr.String()) assert.Equal(t, basics.Address{}, ad.AuthAddr) } @@ -499,15 +492,20 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn0 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) - txn1 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountA) - txn2 := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC) + txn0 := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, + test.AccountB) + txn1 := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, + basics.Address{}) + txn2 := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountC) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -535,21 +533,22 @@ func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { /////////// // Given // Send rekey transaction /////////// - txn := test.MakePaymentTxn(1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 1000, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, + basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) ////////// // Then // No error when committing to the DB. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") blockCert := rpcs.EncodedBlockCert{Block: block} err = proc.Process(&blockCert) require.NoError(t, err) - } // TestIgnoreDefaultFrozenConfigUpdate the creator asset holding should ignore default-frozen = true. @@ -563,8 +562,12 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { /////////// // Given // A new asset with default-frozen = true, and AccountB opting into it. /////////// - createAssetNotFrozen := test.MakeAssetConfigTxn(0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) - modifyAssetToFrozen := test.MakeAssetConfigTxn(assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", test.AccountA) + createAssetNotFrozen := test.MakeAssetConfigTxn( + 0, total, uint64(6), false, "icicles", "frozen coin", "http://antarctica.com", + test.AccountA) + modifyAssetToFrozen := test.MakeAssetConfigTxn( + assetid, total, uint64(6), true, "icicles", "frozen coin", "http://antarctica.com", + test.AccountA) optin := test.MakeAssetOptInTxn(assetid, test.AccountB) block, err := test.MakeBlockForTxns( @@ -575,7 +578,7 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -602,14 +605,16 @@ func TestZeroTotalAssetCreate(t *testing.T) { /////////// // Given // A new asset with total = 0. /////////// - createAsset := test.MakeAssetConfigTxn(0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) + createAsset := test.MakeAssetConfigTxn( + 0, total, uint64(6), false, "mcn", "my coin", "http://antarctica.com", + test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) ////////// // When // We commit the round accounting to the database. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -668,7 +673,7 @@ func TestDestroyAssetBasic(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -710,7 +715,7 @@ func TestDestroyAssetZeroSupply(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -769,13 +774,14 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") blockCert := rpcs.EncodedBlockCert{Block: block} err = proc.Process(&blockCert) require.NoError(t, err) + // Check that the creator's asset holding is deleted. assertAssetHoldingDates(t, db.db, test.AccountA, assetID, sql.NullBool{Valid: true, Bool: true}, @@ -806,7 +812,9 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { // Create a block with freeze txn assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) + createAsset := test.MakeAssetConfigTxn( + 0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", + test.AccountA) optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) freeze := test.MakeAssetFreezeTxn(assetid, true, test.AccountA, test.AccountB) @@ -817,7 +825,7 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -864,7 +872,7 @@ func TestInnerTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -920,7 +928,7 @@ func TestAppExtraPages(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -994,11 +1002,12 @@ func TestKeytypeBasic(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Sig - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1010,7 +1019,8 @@ func TestKeytypeBasic(t *testing.T) { assertKeytype(t, db, test.AccountA, &keytype) // Msig - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) @@ -1021,7 +1031,7 @@ func TestKeytypeBasic(t *testing.T) { require.NoError(t, err) keytype = "msig" - assertKeytype(t, db, test.AccountB, &keytype) + assertKeytype(t, db, test.AccountA, &keytype) } // Test that asset amount >= 2^63 is handled correctly. Due to the specifics of @@ -1031,11 +1041,12 @@ func TestLargeAssetAmount(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA) + txn := test.MakeAssetConfigTxn( + 0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1175,17 +1186,19 @@ func TestNonDisplayableUTF8(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) + txn := test.MakeAssetConfigTxn( + 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) // Try to add cheeky inner txns lazily by adding an AD to the acfg txn txn.ApplyData.EvalDelta.InnerTxns = []transactions.SignedTxnWithAD{ - test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA), + test.MakeAssetConfigTxn( + 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA), } txn.ApplyData.EvalDelta.InnerTxns[0].ConfigAsset = basics.AssetIndex(innerAssetID) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) // Test 1: import/accounting should work. - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1265,10 +1278,11 @@ func TestReconfigAsset(t *testing.T) { url := "https://algorand.com" assetID := uint64(1) - txn := test.MakeAssetConfigTxn(0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) + txn := test.MakeAssetConfigTxn( + 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1327,11 +1341,12 @@ func TestKeytypeResetsOnRekey(t *testing.T) { defer shutdownFunc() // Sig - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1343,7 +1358,8 @@ func TestKeytypeResetsOnRekey(t *testing.T) { assertKeytype(t, db, test.AccountA, &keytype) // Rekey. - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) + txn = test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, test.AccountB) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) @@ -1354,7 +1370,8 @@ func TestKeytypeResetsOnRekey(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Msig - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountA, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) txn.Sig = crypto.Signature{} txn.Msig.Subsigs = append(txn.Msig.Subsigs, crypto.MultisigSubsig{}) txn.AuthAddr = test.AccountB @@ -1366,7 +1383,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { require.NoError(t, err) keytype = "msig" - assertKeytype(t, db, test.AccountB, &keytype) + assertKeytype(t, db, test.AccountA, &keytype) } // Test that after closing the account, keytype will be correctly set. @@ -1377,11 +1394,12 @@ func TestKeytypeDeletedAccount(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) - closeTxn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}) + closeTxn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, test.AccountB, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &closeTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1425,19 +1443,23 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { assetid := uint64(1) - createAsset := test.MakeAssetConfigTxn(0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", test.AccountA) + createAsset := test.MakeAssetConfigTxn( + 0, uint64(1000000), uint64(6), false, "mcn", "my coin", "http://antarctica.com", + test.AccountA) optinB := test.MakeAssetOptInTxn(assetid, test.AccountB) - transferAB := test.MakeAssetTransferTxn(assetid, 100, test.AccountA, test.AccountB, basics.Address{}) + transferAB := test.MakeAssetTransferTxn( + assetid, 100, test.AccountA, test.AccountB, basics.Address{}) optinC := test.MakeAssetOptInTxn(assetid, test.AccountC) // Close B to C. - closeB := test.MakeAssetTransferTxn(assetid, 30, test.AccountB, test.AccountA, test.AccountC) + closeB := test.MakeAssetTransferTxn( + assetid, 30, test.AccountB, test.AccountA, test.AccountC) block, err := test.MakeBlockForTxns( block.BlockHeader, &createAsset, &optinB, &transferAB, &optinC, &closeB) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1478,7 +1500,7 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { require.NoError(t, err) assert.Equal(t, uint64(0), round) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1514,13 +1536,15 @@ func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() - createTxn := test.MakePaymentTxn(0, 100000, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) - deleteTxn := test.MakePaymentTxn(0, 100000, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) + createTxn := test.MakePaymentTxn( + 0, 5, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) + deleteTxn := test.MakePaymentTxn( + 0, 2, 3, 0, 0, 0, test.AccountE, test.AccountB, test.AccountC, basics.Address{}) block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1558,7 +1582,7 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1618,7 +1642,7 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1657,7 +1681,7 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1784,7 +1808,7 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { rootTxid := appCall.Txn.ID() err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1895,7 +1919,7 @@ func TestNonUTF8Logs(t *testing.T) { require.NoError(t, err) // Test 1: import/accounting should work. - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1946,8 +1970,10 @@ func TestTxnAssetID(t *testing.T) { defer shutdownFunc() assetid := uint64(1) - createAssetTxn := test.MakeAssetConfigTxn(0, 0, 0, false, "myasset", "ma", "", test.AccountA) - configAssetTxn := test.MakeAssetConfigTxn(assetid, 0, 0, false, "myasset", "ma", "", test.AccountA) + createAssetTxn := test.MakeAssetConfigTxn( + 0, 0, 0, false, "myasset", "ma", "", test.AccountA) + configAssetTxn := test.MakeAssetConfigTxn( + assetid, 0, 0, false, "myasset", "ma", "", test.AccountA) appid := uint64(3) createAppTxn := test.MakeCreateAppTxn(test.AccountA) destroyAppTxn := test.MakeAppDestroyTxn(appid, test.AccountA) @@ -1957,7 +1983,7 @@ func TestTxnAssetID(t *testing.T) { &createAppTxn, &destroyAppTxn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -1987,7 +2013,7 @@ func TestBadTxnJsonEncoding(t *testing.T) { // Need to import a block header because the transactions query joins on it. block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -2000,7 +2026,7 @@ func TestBadTxnJsonEncoding(t *testing.T) { badJSON := `{"aaaaaaaa": 0}` query := `INSERT INTO txn (round, intra, typeenum, asset, txid, txn, extra) - VALUES (1, $1, 0, 0, $2, $3, $4)` + VALUES (1, $1, 0, 0, $2, $3, $4)` _, err = db.db.Exec( context.Background(), query, rootIntra, rootTxid, badJSON, @@ -2055,10 +2081,11 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { assertKeytype(t, db, test.AccountA, nil) // Sigtype of account B becomes "sig". - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -2067,7 +2094,8 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { require.NoError(t, err) // Sigtype of account A becomes "sig" and B remains the same. - txn = test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) + txn = test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) blockCert = rpcs.EncodedBlockCert{Block: block} @@ -2099,10 +2127,11 @@ func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { require.NoError(t, err) } - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -2117,10 +2146,11 @@ func TestAddBlockTxnParticipationAdded(t *testing.T) { db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) defer shutdownFunc() - txn := test.MakePaymentTxn(0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn( + 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") @@ -2166,7 +2196,7 @@ func TestTransactionsTxnAhead(t *testing.T) { { block, err := test.MakeBlockForTxns(block.BlockHeader) require.NoError(t, err) - l := makeTestLedger(t) + l := test.MakeTestLedger("ledger") defer l.Close() proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 6fc555935..3c9277b9d 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -12,7 +12,7 @@ import ( "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" - indxledger "github.com/algorand/indexer/processor/eval" + iledger "github.com/algorand/indexer/processor/eval" ) type blockProcessor struct { @@ -58,7 +58,7 @@ func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { protoChanged := !proto.EnableAssetCloseAmount proto.EnableAssetCloseAmount = true - ledgerForEval, err := indxledger.MakeLedgerForEvaluator(proc.ledger) + ledgerForEval, err := iledger.MakeLedgerForEvaluator(proc.ledger) if err != nil { return fmt.Errorf("Process() err: %w", err) } @@ -109,7 +109,7 @@ func (proc *blockProcessor) NextRoundToProcess() uint64 { // Preload all resources (account data, account resources, asset/app creators) for the // evaluator. -func prepareEvalResources(l *indxledger.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { +func prepareEvalResources(l *iledger.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { assetCreators, appCreators, err := prepareCreators(l, block.Payset) if err != nil { return ledger.EvalForIndexerResources{}, @@ -147,7 +147,7 @@ func prepareEvalResources(l *indxledger.LedgerForEvaluator, block *bookkeeping.B } // Preload asset and app creators. -func prepareCreators(l *indxledger.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { +func prepareCreators(l *iledger.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) assets, err := l.GetAssetCreator(assetsReq) @@ -163,7 +163,7 @@ func prepareCreators(l *indxledger.LedgerForEvaluator, payset transactions.Payse } // Preload account data and account resources. -func prepareAccountsResources(l *indxledger.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { +func prepareAccountsResources(l *iledger.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { addressesReq, resourcesReq := accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) diff --git a/processor/eval/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go index c90748fc3..3478a4624 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -21,7 +21,8 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() pr, _ := block_processor.MakeProcessor(l, nil) - txn := test.MakePaymentTxn(0, 100, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) + txn := test.MakePaymentTxn(0, 100, 0, 1, 1, + 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} From baaeeb279cfedc1c60ea292f75effd0b0d3addef Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 3 Jun 2022 19:10:50 -0400 Subject: [PATCH 25/37] test changes --- api/handlers_e2e_test.go | 123 ++---- cmd/algorand-indexer/daemon.go | 22 +- cmd/algorand-indexer/daemon_test.go | 4 +- cmd/import-validator/core/service.go | 8 +- .../postgres_integration_common_test.go | 12 +- idb/postgres/postgres_integration_test.go | 411 ++++++------------ importer/helper.go | 2 +- processor/blockprocessor/block_processor.go | 46 +- .../blockprocessor/block_processor_test.go | 37 +- processor/eval/ledger_for_evaluator_test.go | 24 +- util/test/testutil.go | 3 +- 11 files changed, 268 insertions(+), 424 deletions(-) diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index 049cfd32d..6cabbc454 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -16,7 +16,9 @@ import ( "github.com/algorand/go-algorand/crypto/merklesignature" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/processor" "github.com/algorand/indexer/processor/blockprocessor" "github.com/labstack/echo/v4" "github.com/sirupsen/logrus" @@ -58,7 +60,7 @@ func testServerImplementation(db idb.IndexerDb) *ServerImplementation { return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts} } -func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*postgres.IndexerDb /*db*/, func() /*shutdownFunc*/) { +func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), processor.Processor, *ledger.Ledger) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) db, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) @@ -72,12 +74,16 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin err = db.LoadGenesis(genesis) require.NoError(t, err) - return db, newShutdownFunc + l := test.MakeTestLedger("ledger") + proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + return db, newShutdownFunc, proc, l } func TestApplicationHandlers(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // A block containing an app call txn with ExtraProgramPages, that the creator and another account have opted into @@ -111,13 +117,8 @@ func TestApplicationHandlers(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) ////////// // When // We query the app @@ -222,8 +223,9 @@ func TestApplicationHandlers(t *testing.T) { } func TestAccountExcludeParameters(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // A block containing a creator of an app, an asset, who also holds and has opted-into those apps. @@ -242,13 +244,8 @@ func TestAccountExcludeParameters(t *testing.T) { &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) ////////// // When // We look up the address using various exclude parameters. @@ -398,8 +395,9 @@ type accountsErrorResponse struct { } func TestAccountMaxResultsLimit(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // A block containing an address that has created 10 apps, deleted 5 apps, and created 10 assets, @@ -452,13 +450,8 @@ func TestAccountMaxResultsLimit(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) ////////// // When // We look up the address using a ServerImplementation with a maxAccountsAPIResults limit set, @@ -782,8 +775,9 @@ func TestAccountMaxResultsLimit(t *testing.T) { } func TestBlockNotFound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // An empty database. @@ -847,8 +841,9 @@ func TestInnerTxn(t *testing.T) { }, } - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // a DB with some inner txns in it. @@ -859,13 +854,8 @@ func TestInnerTxn(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -900,8 +890,9 @@ func TestInnerTxn(t *testing.T) { // transaction group does not allow the root transaction to be returned on both // pages. func TestPagingRootTxnDeduplication(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // a DB with some inner txns in it. @@ -916,13 +907,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) testcases := []struct { name string @@ -1028,8 +1014,9 @@ func TestPagingRootTxnDeduplication(t *testing.T) { } func TestKeyregTransactionWithStateProofKeys(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // A block containing a key reg txn with state proof key @@ -1068,13 +1055,8 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) e := echo.New() { @@ -1127,8 +1109,9 @@ func TestVersion(t *testing.T) { /////////// // Given // An API and context /////////// - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() api := testServerImplementation(db) e := echo.New() @@ -1156,8 +1139,9 @@ func TestVersion(t *testing.T) { } func TestAccountClearsNonUTF8(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // a DB with some inner txns in it. @@ -1176,13 +1160,8 @@ func TestAccountClearsNonUTF8(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) verify := func(params generated.AssetParams) { compareB64 := func(expected string, actual *[]byte) { @@ -1291,8 +1270,9 @@ func TestLookupInnerLogs(t *testing.T) { }, } - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // a DB with some inner txns in it. @@ -1302,13 +1282,8 @@ func TestLookupInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { @@ -1394,8 +1369,9 @@ func TestLookupMultiInnerLogs(t *testing.T) { }, } - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) defer shutdownFunc() + defer l.Close() /////////// // Given // a DB with some inner txns in it. @@ -1405,13 +1381,8 @@ func TestLookupMultiInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) - require.NoError(t, err, "failed to commit") + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index b25b6e6ed..8f5ce7187 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -7,18 +7,13 @@ import ( "io/ioutil" "os" "os/signal" - "path" - "path/filepath" "strings" "sync" "syscall" "time" "github.com/algorand/go-algorand-sdk/client/v2/algod" - algodConfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/api" @@ -88,10 +83,6 @@ var daemonCmd = &cobra.Command{ algodDataDir = os.Getenv("ALGORAND_DATA") } - if indexerDataDir == "" { - indexerDataDir = os.Getenv("INDEXER_DATA") - } - if indexerDataDir != "" { if _, err := os.Stat(indexerDataDir); os.IsNotExist(err) { err := os.Mkdir(indexerDataDir, 0755) @@ -153,6 +144,7 @@ var daemonCmd = &cobra.Command{ logger.Info("Initializing block import handler.") imp := importer.NewImporter(db) + logger.Info("Initializing local ledger.") genesisReader = importer.GetGenesisFile(genesisJSONPath, bot.Algod(), logger) genesis, err := readGenesis(genesisReader) maybeFail(err, "Error reading genesis file") @@ -161,16 +153,12 @@ var daemonCmd = &cobra.Command{ initState, err := util.CreateInitState(&genesis, &genesisBlock) maybeFail(err, "Error creating init state") - logger.Info("Initializing local ledger.") - localLedger, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(indexerDataDir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) - maybeFail(err, "failed to open ledger %v", err) - defer localLedger.Close() - bot.SetNextRound(uint64(localLedger.Latest()) + 1) - - proc, err := blockprocessor.MakeProcessor(localLedger, imp.ImportBlock) + proc, err := blockprocessor.MakeProcessor(initState, indexerDataDir, imp.ImportBlock) if err != nil { maybeFail(err, "blockprocessor.MakeProcessor() err %v", err) } + + bot.SetNextRound(proc.NextRoundToProcess()) handler := blockHandler(proc, 1*time.Second) bot.SetBlockHandler(handler) @@ -233,6 +221,8 @@ func init() { viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") viper.RegisterAlias("token", "api-token") + viper.RegisterAlias("data-dir", "data_dir") + } // makeOptions converts CLI options to server options diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index ba553aab3..4669e04d4 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -22,7 +22,7 @@ import ( type mockImporter struct { } -var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 proc.nextRoundToProcess: 1") +var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 nextRoundToProcess: 1") func (imp *mockImporter) ImportBlock(vb *ledgercore.ValidatedBlock) error { return nil @@ -41,7 +41,7 @@ func TestImportRetryAndCancel(t *testing.T) { imp := &mockImporter{} l := itest.MakeTestLedger("ledger") defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, nil) + proc, err := blockprocessor.MakeProcessorWithLedger(l, nil) assert.Nil(t, err) proc.SetHandler(imp.ImportBlock) handler := blockHandler(proc, 50*time.Millisecond) diff --git a/cmd/import-validator/core/service.go b/cmd/import-validator/core/service.go index af538105e..e37e1d82f 100644 --- a/cmd/import-validator/core/service.go +++ b/cmd/import-validator/core/service.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/algorand/indexer/processor/blockprocessor" "github.com/sirupsen/logrus" "github.com/algorand/go-algorand-sdk/client/v2/algod" @@ -26,6 +25,7 @@ import ( "github.com/algorand/indexer/fetcher" "github.com/algorand/indexer/idb" "github.com/algorand/indexer/idb/postgres" + "github.com/algorand/indexer/processor/blockprocessor" "github.com/algorand/indexer/util" ) @@ -121,13 +121,13 @@ func openLedger(ledgerPath string, genesis *bookkeeping.Genesis, genesisBlock *b GenesisHash: genesisBlock.GenesisHash(), } - ledger, err := ledger.OpenLedger( + l, err := ledger.OpenLedger( logger, path.Join(ledgerPath, "ledger"), false, initState, config.GetDefaultLocal()) if err != nil { return nil, fmt.Errorf("openLedger() open err: %w", err) } - return ledger, nil + return l, nil } func getModifiedState(l *ledger.Ledger, block *bookkeeping.Block) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}, error) { @@ -351,7 +351,7 @@ func catchup(db *postgres.IndexerDb, l *ledger.Ledger, bot fetcher.Fetcher, logg if nextRoundLedger >= nextRoundIndexer { wg.Add(1) - prc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + prc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock) if err != nil { return fmt.Errorf("catchup() err: %w", err) } diff --git a/idb/postgres/postgres_integration_common_test.go b/idb/postgres/postgres_integration_common_test.go index 9546b6466..f8740eba4 100644 --- a/idb/postgres/postgres_integration_common_test.go +++ b/idb/postgres/postgres_integration_common_test.go @@ -5,7 +5,11 @@ import ( "testing" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/indexer/processor" + "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/util/test" "github.com/jackc/pgx/v4/pgxpool" "github.com/stretchr/testify/require" @@ -27,7 +31,7 @@ func setupIdbWithConnectionString(t *testing.T, connStr string, genesis bookkeep return idb } -func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*IndexerDb /*db*/, func() /*shutdownFunc*/) { +func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*IndexerDb /*db*/, func() /*shutdownFunc*/, processor.Processor, *ledger.Ledger) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) db := setupIdbWithConnectionString(t, connStr, genesis, genesisBlock) @@ -36,7 +40,11 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin shutdownFunc() } - return db, newShutdownFunc + l := test.MakeTestLedger("ledger") + proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock) + require.NoError(t, err, "failed to open ledger") + + return db, newShutdownFunc, proc, l } // Helper to execute a query returning an integer, for example COUNT(*). Returns -1 on an error. diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index bf46ffe19..d4013d79b 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -149,8 +149,9 @@ func assertAccountAsset(t *testing.T, db *pgxpool.Pool, addr basics.Address, ass // TestAssetCloseReopenTransfer tests a scenario that requires asset subround accounting func TestAssetCloseReopenTransfer(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() assetid := uint64(1) amt := uint64(10000) @@ -175,12 +176,7 @@ func TestAssetCloseReopenTransfer(t *testing.T) { ////////// // When // We commit the block to the database ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -198,8 +194,9 @@ func TestAssetCloseReopenTransfer(t *testing.T) { // TestReCreateAssetHolding checks the optin value of a defunct func TestReCreateAssetHolding(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() total := uint64(1000000) @@ -229,12 +226,7 @@ func TestReCreateAssetHolding(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -246,8 +238,9 @@ func TestReCreateAssetHolding(t *testing.T) { // TestMultipleAssetOptins make sure no-op transactions don't reset the default frozen value. func TestNoopOptins(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() /////////// // Given // @@ -269,12 +262,7 @@ func TestNoopOptins(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -285,8 +273,9 @@ func TestNoopOptins(t *testing.T) { // TestMultipleWriters tests that accounting cannot be double committed. func TestMultipleWriters(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() amt := uint64(10000) @@ -299,11 +288,6 @@ func TestMultipleWriters(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &payAccountE) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} ////////// // When // We attempt commit the round accounting multiple times. @@ -317,7 +301,7 @@ func TestMultipleWriters(t *testing.T) { go func() { defer wg.Done() <-start - errors <- proc.Process(&blockCert) + errors <- proc.Process(&rpcs.EncodedBlockCert{Block: block}) }() } close(start) @@ -346,8 +330,9 @@ func TestMultipleWriters(t *testing.T) { // TestBlockWithTransactions tests that the block with transactions endpoint works. func TestBlockWithTransactions(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() round := uint64(1) assetid := uint64(1) @@ -376,12 +361,8 @@ func TestBlockWithTransactions(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, txns...) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -410,8 +391,9 @@ func TestBlockWithTransactions(t *testing.T) { } func TestRekeyBasic(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() /////////// // Given // Send rekey transaction @@ -421,12 +403,7 @@ func TestRekeyBasic(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -443,8 +420,9 @@ func TestRekeyBasic(t *testing.T) { } func TestRekeyToItself(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() /////////// // Given // Send rekey transactions @@ -454,12 +432,7 @@ func TestRekeyToItself(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txn = test.MakePaymentTxn( @@ -468,8 +441,7 @@ func TestRekeyToItself(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -486,8 +458,9 @@ func TestRekeyToItself(t *testing.T) { } func TestRekeyThreeTimesInSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() /////////// // Given // Send rekey transaction @@ -505,12 +478,7 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -527,8 +495,9 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { } func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + _, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() /////////// // Given // Send rekey transaction @@ -542,19 +511,15 @@ func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { ////////// // Then // No error when committing to the DB. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } // TestIgnoreDefaultFrozenConfigUpdate the creator asset holding should ignore default-frozen = true. func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() assetid := uint64(1) total := uint64(1000000) @@ -578,12 +543,7 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -596,8 +556,9 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { // TestZeroTotalAssetCreate tests that the asset holding with total of 0 is created. func TestZeroTotalAssetCreate(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() assetid := uint64(1) total := uint64(0) @@ -614,12 +575,7 @@ func TestZeroTotalAssetCreate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -663,8 +619,9 @@ func assertAssetHoldingDates(t *testing.T, db *pgxpool.Pool, address basics.Addr } func TestDestroyAssetBasic(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() assetID := uint64(1) @@ -673,12 +630,7 @@ func TestDestroyAssetBasic(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Destroy an asset. @@ -686,8 +638,7 @@ func TestDestroyAssetBasic(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the asset is deleted. @@ -704,8 +655,9 @@ func TestDestroyAssetBasic(t *testing.T) { } func TestDestroyAssetZeroSupply(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() assetID := uint64(1) @@ -715,12 +667,7 @@ func TestDestroyAssetZeroSupply(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the asset is deleted. @@ -737,9 +684,11 @@ func TestDestroyAssetZeroSupply(t *testing.T) { } func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assetID := uint64(1) // Create an asset. Create a transaction where all special addresses are different @@ -774,12 +723,7 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the creator's asset holding is deleted. @@ -802,9 +746,11 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { // Test that block import adds the freeze/sender accounts to txn_participation. func TestAssetFreezeTxnParticipation(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + /////////// // Given // A block containing an asset freeze txn /////////// @@ -825,12 +771,7 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -850,9 +791,11 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { // Test that block import adds accounts from inner txns to txn_participation. func TestInnerTxnParticipation(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + /////////// // Given // A block containing an app call txn with inners /////////// @@ -872,12 +815,7 @@ func TestInnerTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -900,9 +838,11 @@ func TestInnerTxnParticipation(t *testing.T) { } func TestAppExtraPages(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + // Create an app. // Create a transaction with ExtraProgramPages field set to 1 @@ -928,12 +868,7 @@ func TestAppExtraPages(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) row := db.db.QueryRow(context.Background(), "SELECT index, params FROM app WHERE creator = $1", test.AccountA[:]) @@ -996,9 +931,11 @@ func assertKeytype(t *testing.T, db *IndexerDb, address basics.Address, keytype func TestKeytypeBasic(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assertKeytype(t, db, test.AccountA, nil) // Sig @@ -1007,12 +944,7 @@ func TestKeytypeBasic(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := "sig" @@ -1026,8 +958,7 @@ func TestKeytypeBasic(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype = "msig" @@ -1037,21 +968,18 @@ func TestKeytypeBasic(t *testing.T) { // Test that asset amount >= 2^63 is handled correctly. Due to the specifics of // postgres it might be a problem. func TestLargeAssetAmount(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assetid := uint64(1) txn := test.MakeAssetConfigTxn( 0, math.MaxUint64, 0, false, "mc", "mycoin", "", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) { @@ -1183,9 +1111,11 @@ func TestNonDisplayableUTF8(t *testing.T) { url := testcase.AssetURL t.Run(testcase.Name, func(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + txn := test.MakeAssetConfigTxn( 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) // Try to add cheeky inner txns lazily by adding an AD to the acfg txn @@ -1198,12 +1128,7 @@ func TestNonDisplayableUTF8(t *testing.T) { require.NoError(t, err) // Test 1: import/accounting should work. - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1270,9 +1195,11 @@ func TestNonDisplayableUTF8(t *testing.T) { // TestReconfigAsset make sure we properly handle asset param merges. func TestReconfigAsset(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + unit := "co\000in" name := "algo" url := "https://algorand.com" @@ -1282,12 +1209,7 @@ func TestReconfigAsset(t *testing.T) { 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txn = transactions.SignedTxnWithAD{ @@ -1312,8 +1234,7 @@ func TestReconfigAsset(t *testing.T) { } block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1337,21 +1258,18 @@ func TestReconfigAsset(t *testing.T) { func TestKeytypeResetsOnRekey(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + // Sig txn := test.MakePaymentTxn( 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := "sig" @@ -1363,8 +1281,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) assertKeytype(t, db, test.AccountA, nil) @@ -1378,8 +1295,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype = "msig" @@ -1389,9 +1305,11 @@ func TestKeytypeResetsOnRekey(t *testing.T) { // Test that after closing the account, keytype will be correctly set. func TestKeytypeDeletedAccount(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assertKeytype(t, db, test.AccountA, nil) closeTxn := test.MakePaymentTxn( @@ -1399,12 +1317,7 @@ func TestKeytypeDeletedAccount(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &closeTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := "sig" @@ -1413,8 +1326,9 @@ func TestKeytypeDeletedAccount(t *testing.T) { // TestAddBlockGenesis tests that adding block 0 is successful. func TestAddBlockGenesis(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() opts := idb.GetBlockOptions{ Transactions: true, @@ -1438,8 +1352,9 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { block := test.MakeGenesisBlock() block.UpgradeState.CurrentProtocol = protocol.ConsensusV24 - db, shutdownFunc := setupIdb(t, genesis, block) + db, shutdownFunc, proc, l := setupIdb(t, genesis, block) defer shutdownFunc() + defer l.Close() assetid := uint64(1) @@ -1459,12 +1374,7 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { &optinC, &closeB) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check asset close amount in the `closeB` transaction. @@ -1489,6 +1399,7 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) defer shutdownFunc() + db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) require.NoError(t, err) defer db.Close() @@ -1502,7 +1413,7 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") round, err = db.GetNextRoundToAccount() @@ -1522,7 +1433,7 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader) require.NoError(t, err) blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) round, err = db.GetNextRoundToAccount() @@ -1533,9 +1444,11 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { // Test that AddBlock makes a record of an account that gets created and deleted in // the same round. func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + createTxn := test.MakePaymentTxn( 0, 5, 0, 0, 0, 0, test.AccountA, test.AccountE, basics.Address{}, basics.Address{}) deleteTxn := test.MakePaymentTxn( @@ -1544,12 +1457,7 @@ func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.AccountQueryOptions{ @@ -1572,9 +1480,11 @@ func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { // Test that AddBlock makes a record of an asset that is created and deleted in // the same round. func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assetid := uint64(1) createTxn := test.MakeAssetConfigTxn(0, 3, 0, false, "", "", "", test.AccountA) deleteTxn := test.MakeAssetDestroyTxn(assetid, test.AccountA) @@ -1582,12 +1492,7 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Asset global state. @@ -1632,9 +1537,11 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { // Test that AddBlock makes a record of an app that is created and deleted in // the same round. func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + appid := uint64(1) createTxn := test.MakeCreateAppTxn(test.AccountA) deleteTxn := test.MakeAppDestroyTxn(appid, test.AccountA) @@ -1642,12 +1549,7 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.ApplicationQuery{ @@ -1670,9 +1572,11 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { // Test that AddBlock makes a record of an app that is created and deleted in // the same round. func TestAddBlockAppOptInOutSameRound(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + appid := uint64(1) createTxn := test.MakeCreateAppTxn(test.AccountA) optInTxn := test.MakeAppOptInTxn(appid, test.AccountB) @@ -1681,12 +1585,7 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.AccountQueryOptions{ @@ -1797,6 +1696,7 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { // Given: A DB with one transaction containing inner transactions [app -> pay -> xfer] pdb, connStr, shutdownFunc := pgtest.SetupPostgres(t) defer shutdownFunc() + db := setupIdbWithConnectionString( t, connStr, test.MakeGenesis(), test.MakeGenesisBlock()) defer db.Close() @@ -1810,7 +1710,7 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { err = pgutil.TxWithRetry(pdb, serializable, func(tx pgx.Tx) error { l := test.MakeTestLedger("ledger") defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) + proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock) require.NoError(t, err, "failed to open ledger") blockCert := rpcs.EncodedBlockCert{Block: block} return proc.Process(&blockCert) @@ -1885,9 +1785,11 @@ func TestNonUTF8Logs(t *testing.T) { testcase := testcase t.Run(testcase.Name, func(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + createAppTxn := test.MakeCreateAppTxn(test.AccountA) createAppTxn.ApplyData.EvalDelta = transactions.EvalDelta{ Logs: testcase.Logs, @@ -1919,12 +1821,7 @@ func TestNonUTF8Logs(t *testing.T) { require.NoError(t, err) // Test 1: import/accounting should work. - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: transaction results properly serialized @@ -1948,6 +1845,7 @@ func TestNonUTF8Logs(t *testing.T) { func TestLoadGenesisAccountTotals(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) defer shutdownFunc() + db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) require.NoError(t, err) defer db.Close() @@ -1966,9 +1864,11 @@ func TestLoadGenesisAccountTotals(t *testing.T) { } func TestTxnAssetID(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assetid := uint64(1) createAssetTxn := test.MakeAssetConfigTxn( 0, 0, 0, false, "myasset", "ma", "", test.AccountA) @@ -1983,12 +1883,7 @@ func TestTxnAssetID(t *testing.T) { &createAppTxn, &destroyAppTxn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) @@ -2007,18 +1902,15 @@ func TestTxnAssetID(t *testing.T) { } func TestBadTxnJsonEncoding(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + // Need to import a block header because the transactions query joins on it. block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) rootTxid := "abc" @@ -2075,9 +1967,11 @@ func TestBadTxnJsonEncoding(t *testing.T) { func TestKeytypeDoNotResetReceiver(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + assertKeytype(t, db, test.AccountA, nil) // Sigtype of account B becomes "sig". @@ -2085,12 +1979,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Sigtype of account A becomes "sig" and B remains the same. @@ -2098,8 +1987,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := "sig" @@ -2111,9 +1999,11 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { // the current round, AddBlock() still runs successfully. func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + { query := `INSERT INTO txn (round, intra, typeenum, asset, txn, extra) VALUES (1, 0, 0, 0, 'null'::jsonb, 'null'::jsonb)` @@ -2131,31 +2021,23 @@ func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } // Test that AddBlock() writes to `txn_participation` table. func TestAddBlockTxnParticipationAdded(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() + txn := test.MakePaymentTxn( 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) tf := idb.TransactionFilter{ @@ -2174,8 +2056,9 @@ func TestAddBlockTxnParticipationAdded(t *testing.T) { // Transactions() doesn't return the rows ahead of the state. func TestTransactionsTxnAhead(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), block) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() // Insert a transaction row at round 1 and check that Transactions() does not return // it. @@ -2194,14 +2077,7 @@ func TestTransactionsTxnAhead(t *testing.T) { // Now add an empty round 1 block, and verify that Transactions() returns the // fake transaction. { - block, err := test.MakeBlockForTxns(block.BlockHeader) - require.NoError(t, err) - l := test.MakeTestLedger("ledger") - defer l.Close() - proc, err := blockprocessor.MakeProcessor(l, db.AddBlock) - require.NoError(t, err, "failed to open ledger") - blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err := proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } { @@ -2217,6 +2093,7 @@ func TestTransactionsTxnAhead(t *testing.T) { func TestGenesisHashCheckAtDBSetup(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) defer shutdownFunc() + genesis := test.MakeGenesis() db := setupIdbWithConnectionString( t, connStr, genesis, test.MakeGenesisBlock()) @@ -2246,6 +2123,7 @@ type ImportState struct { func TestGenesisHashCheckAtInitialImport(t *testing.T) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) defer shutdownFunc() + genesis := test.MakeGenesis() db, _, err := OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) require.NoError(t, err) @@ -2296,8 +2174,9 @@ func getResults(ctx context.Context, rows <-chan idb.AccountRow) (result []idb.A } func TestIndexerDb_GetAccounts(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer l.Close() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/importer/helper.go b/importer/helper.go index f7197ff78..f67339b75 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -147,7 +147,7 @@ func importTar(imp Importer, tarfile io.Reader, l *log.Logger, genesisReader io. ld, err := ledger.OpenLedger(logging.NewLogger(), "ledger", true, initState, config.GetDefaultLocal()) maybeFail(err, l, "Cannot open ledger") - proc, err := blockprocessor.MakeProcessor(ld, imp.ImportBlock) + proc, err := blockprocessor.MakeProcessorWithLedger(ld, imp.ImportBlock) maybeFail(err, l, "Error creating processor") for _, blockContainer := range blocks[1:] { diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 3c9277b9d..36e2bd205 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -2,13 +2,17 @@ package blockprocessor import ( "fmt" + "path" + "path/filepath" "github.com/algorand/go-algorand/config" + algodConfig "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" @@ -16,28 +20,41 @@ import ( ) type blockProcessor struct { - handler func(block *ledgercore.ValidatedBlock) error - nextRoundToProcess uint64 - ledger *ledger.Ledger + handler func(block *ledgercore.ValidatedBlock) error + ledger *ledger.Ledger +} + +// MakeProcessorWithLedger creates a block processor with a given ledger +func MakeProcessorWithLedger(l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { + if l == nil { + return nil, fmt.Errorf("MakeProcessorWithLedger(): local ledger not initialized") + } + addGenesisBlock(l, handler) + return &blockProcessor{ledger: l, handler: handler}, nil } // MakeProcessor creates a block processor -func MakeProcessor(ledger *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { - if ledger == nil { - return nil, fmt.Errorf("MakeProcessor(): local ledger not initialized") +func MakeProcessor(initState ledgercore.InitState, datadir string, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { + l, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(datadir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) + if err != nil { + return nil, fmt.Errorf("MakeProcessor() err: %w", err) } - if handler != nil && ledger.Latest() == 0 { - blk, err := ledger.Block(0) + return MakeProcessorWithLedger(l, handler) +} + +func addGenesisBlock(l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) error { + if handler != nil && uint64(l.Latest()) == 0 { + blk, err := l.Block(0) if err != nil { - return nil, fmt.Errorf("MakeProcessor() err: %w", err) + return fmt.Errorf("addGenesisBlock() err: %w", err) } vb := ledgercore.MakeValidatedBlock(blk, ledgercore.StateDelta{}) err = handler(&vb) if err != nil { - return nil, fmt.Errorf("MakeProcessor() handler err: %w", err) + return fmt.Errorf("addGenesisBlock() handler err: %w", err) } } - return &blockProcessor{ledger: ledger, nextRoundToProcess: uint64(ledger.Latest() + 1), handler: handler}, nil + return nil } // Process a raw algod block @@ -46,8 +63,8 @@ func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { if blockCert == nil { return fmt.Errorf("Process(): cannot process a nil block") } - if uint64(blockCert.Block.Round()) != proc.nextRoundToProcess { - return fmt.Errorf("Process() invalid round blockCert.Block.Round(): %d proc.nextRoundToProcess: %d", blockCert.Block.Round(), proc.nextRoundToProcess) + if uint64(blockCert.Block.Round()) != uint64(proc.ledger.Latest())+1 { + return fmt.Errorf("Process() invalid round blockCert.Block.Round(): %d nextRoundToProcess: %d", blockCert.Block.Round(), uint64(proc.ledger.Latest())+1) } proto, ok := config.Consensus[blockCert.Block.BlockHeader.CurrentProtocol] @@ -95,7 +112,6 @@ func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { if err != nil { return fmt.Errorf("Process() add validated block err: %w", err) } - proc.nextRoundToProcess = uint64(proc.ledger.Latest()) + 1 return nil } @@ -104,7 +120,7 @@ func (proc *blockProcessor) SetHandler(handler func(block *ledgercore.ValidatedB } func (proc *blockProcessor) NextRoundToProcess() uint64 { - return proc.nextRoundToProcess + return uint64(proc.ledger.Latest()) + 1 } // Preload all resources (account data, account resources, asset/app creators) for the diff --git a/processor/blockprocessor/block_processor_test.go b/processor/blockprocessor/block_processor_test.go index 848a3b084..6272ec145 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/processor/blockprocessor/block_processor_test.go @@ -2,31 +2,27 @@ package blockprocessor_test import ( "fmt" - "log" "testing" "github.com/algorand/go-algorand/agreement" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" - "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/rpcs" block_processor "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/test" "github.com/stretchr/testify/assert" ) func TestProcess(t *testing.T) { - l := makeTestLedger(t, "local_ledger") + l := test.MakeTestLedger("local_ledger") + defer l.Close() genesisBlock, err := l.Block(basics.Round(0)) assert.Nil(t, err) // create processor handler := func(vb *ledgercore.ValidatedBlock) error { return nil } - pr, _ := block_processor.MakeProcessor(l, handler) + pr, _ := block_processor.MakeProcessorWithLedger(l, handler) prevHeader := genesisBlock.BlockHeader assert.Equal(t, basics.Round(0), l.Latest()) // create a few rounds @@ -50,11 +46,12 @@ func TestProcess(t *testing.T) { } func TestFailedProcess(t *testing.T) { - l := makeTestLedger(t, "local_ledger2") + l := test.MakeTestLedger("local_ledger") + defer l.Close() // invalid processor - pr, err := block_processor.MakeProcessor(nil, nil) + pr, err := block_processor.MakeProcessorWithLedger(nil, nil) assert.Contains(t, err.Error(), "MakeProcessor(): local ledger not initialized") - pr, err = block_processor.MakeProcessor(l, nil) + pr, err = block_processor.MakeProcessorWithLedger(l, nil) assert.Nil(t, err) err = pr.Process(nil) assert.Contains(t, err.Error(), "Process(): cannot process a nil block") @@ -100,9 +97,9 @@ func TestFailedProcess(t *testing.T) { handler := func(vb *ledgercore.ValidatedBlock) error { return fmt.Errorf("handler error") } - _, err = block_processor.MakeProcessor(l, handler) + _, err = block_processor.MakeProcessorWithLedger(l, handler) assert.Contains(t, err.Error(), "MakeProcessor() handler err") - pr, _ = block_processor.MakeProcessor(l, nil) + pr, _ = block_processor.MakeProcessorWithLedger(l, nil) txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) @@ -111,19 +108,3 @@ func TestFailedProcess(t *testing.T) { err = pr.Process(&rawBlock) assert.Contains(t, err.Error(), "Process() handler err") } - -func makeTestLedger(t *testing.T, prefix string) *ledger.Ledger { - // initialize local ledger - genesis := test.MakeGenesis() - genesisBlock := test.MakeGenesisBlock() - initState, err := util.CreateInitState(&genesis, &genesisBlock) - if err != nil { - log.Panicf("test init err: %v", err) - } - logger := logging.NewLogger() - l, err := ledger.OpenLedger(logger, prefix, true, initState, config.GetDefaultLocal()) - if err != nil { - log.Panicf("test init err: %v", err) - } - return l -} diff --git a/processor/eval/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go index 3478a4624..517bda3dc 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -20,7 +20,7 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn := test.MakePaymentTxn(0, 100, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) @@ -42,7 +42,7 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { func TestLedgerForEvaluatorAccountDataBasic(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - block_processor.MakeProcessor(l, nil) + block_processor.MakeProcessorWithLedger(l, nil) accountData, _, err := l.LookupWithoutRewards(0, test.AccountB) require.NoError(t, err) @@ -79,7 +79,7 @@ func TestLedgerForEvaluatorAccountDataMissingAccount(t *testing.T) { func TestLedgerForEvaluatorAsset(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 4, 0, false, "", "", "", test.AccountA) @@ -180,7 +180,7 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { func TestLedgerForEvaluatorApp(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAppCallTxn(0, test.AccountA) txn1 := test.MakeAppCallTxnWithLogs(0, test.AccountA, []string{"testing"}) @@ -262,7 +262,7 @@ func TestLedgerForEvaluatorApp(t *testing.T) { func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAppCallTxn(0, test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) @@ -324,7 +324,7 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - block_processor.MakeProcessor(l, nil) + block_processor.MakeProcessorWithLedger(l, nil) addresses := []basics.Address{ test.AccountA, test.AccountB, test.AccountC, test.AccountD} @@ -353,7 +353,7 @@ func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) @@ -384,7 +384,7 @@ func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetDestroyTxn(1, test.AccountA) @@ -413,7 +413,7 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountB) @@ -467,7 +467,7 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAppCallTxn(0, test.AccountA) @@ -499,7 +499,7 @@ func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAppCallTxn(0, test.AccountA) txn1 := test.MakeAppDestroyTxn(1, test.AccountA) @@ -528,7 +528,7 @@ func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { l := test.MakeTestLedger("ledger") defer l.Close() - pr, _ := block_processor.MakeProcessor(l, nil) + pr, _ := block_processor.MakeProcessorWithLedger(l, nil) txn0 := test.MakeAppCallTxn(0, test.AccountA) txn1 := test.MakeAppCallTxn(0, test.AccountB) diff --git a/util/test/testutil.go b/util/test/testutil.go index e8c671e43..085e21f9b 100644 --- a/util/test/testutil.go +++ b/util/test/testutil.go @@ -11,10 +11,9 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" - "github.com/sirupsen/logrus" - "github.com/algorand/indexer/idb" "github.com/algorand/indexer/util" + "github.com/sirupsen/logrus" ) var quiet = false From ce326071a9bb4d607976c4422b2bfe6a126ae248 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 3 Jun 2022 20:49:51 -0400 Subject: [PATCH 26/37] fix vet issues --- idb/postgres/postgres_rand_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/idb/postgres/postgres_rand_test.go b/idb/postgres/postgres_rand_test.go index e4fd44be8..826bc37c2 100644 --- a/idb/postgres/postgres_rand_test.go +++ b/idb/postgres/postgres_rand_test.go @@ -49,8 +49,9 @@ func generateAccountData() ledgercore.AccountData { // and that there are no problems around passing account address pointers to the postgres // driver which could be the same pointer if we are not careful. func TestWriteReadAccountData(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, ld := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer ld.Close() addresses := make(map[basics.Address]struct{}) var delta ledgercore.StateDelta @@ -213,8 +214,9 @@ func generateAppLocalStateDelta(t *testing.T) ledgercore.AppLocalStateDelta { // and that there are no problems around passing account address pointers to the postgres // driver which could be the same pointer if we are not careful. func TestWriteReadResources(t *testing.T) { - db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, _, ld := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) defer shutdownFunc() + defer ld.Close() resources := make(map[basics.Address]map[ledger.Creatable]struct{}) var delta ledgercore.StateDelta From c6411ce91a9f9ed4f5e3e81b95c594d58dc445e3 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 3 Jun 2022 21:09:15 -0400 Subject: [PATCH 27/37] fix failing test --- processor/blockprocessor/block_processor.go | 7 +++++-- processor/blockprocessor/block_processor_test.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 36e2bd205..72afecbf9 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -27,9 +27,12 @@ type blockProcessor struct { // MakeProcessorWithLedger creates a block processor with a given ledger func MakeProcessorWithLedger(l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { if l == nil { - return nil, fmt.Errorf("MakeProcessorWithLedger(): local ledger not initialized") + return nil, fmt.Errorf("MakeProcessorWithLedger() err: local ledger not initialized") + } + err := addGenesisBlock(l, handler) + if err != nil { + return nil, fmt.Errorf("MakeProcessorWithLedger() err: %w", err) } - addGenesisBlock(l, handler) return &blockProcessor{ledger: l, handler: handler}, nil } diff --git a/processor/blockprocessor/block_processor_test.go b/processor/blockprocessor/block_processor_test.go index 6272ec145..2ff7f9471 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/processor/blockprocessor/block_processor_test.go @@ -50,7 +50,7 @@ func TestFailedProcess(t *testing.T) { defer l.Close() // invalid processor pr, err := block_processor.MakeProcessorWithLedger(nil, nil) - assert.Contains(t, err.Error(), "MakeProcessor(): local ledger not initialized") + assert.Contains(t, err.Error(), "MakeProcessorWithLedger() err: local ledger not initialized") pr, err = block_processor.MakeProcessorWithLedger(l, nil) assert.Nil(t, err) err = pr.Process(nil) @@ -98,7 +98,7 @@ func TestFailedProcess(t *testing.T) { return fmt.Errorf("handler error") } _, err = block_processor.MakeProcessorWithLedger(l, handler) - assert.Contains(t, err.Error(), "MakeProcessor() handler err") + assert.Contains(t, err.Error(), "handler error") pr, _ = block_processor.MakeProcessorWithLedger(l, nil) txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) From 27b0a07a5725cdc9d19c52092b371e22300287fe Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 3 Jun 2022 21:36:22 -0400 Subject: [PATCH 28/37] fix failing test --- idb/postgres/postgres_integration_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index d4013d79b..a77d218f8 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -2056,7 +2056,7 @@ func TestAddBlockTxnParticipationAdded(t *testing.T) { // Transactions() doesn't return the rows ahead of the state. func TestTransactionsTxnAhead(t *testing.T) { block := test.MakeGenesisBlock() - db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock()) + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis(), block) defer shutdownFunc() defer l.Close() @@ -2077,7 +2077,9 @@ func TestTransactionsTxnAhead(t *testing.T) { // Now add an empty round 1 block, and verify that Transactions() returns the // fake transaction. { - err := proc.Process(&rpcs.EncodedBlockCert{Block: block}) + block, err := test.MakeBlockForTxns(block.BlockHeader) + require.NoError(t, err) + err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } { From d06852d2da7711a467835a24b19a0087a22f172f Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Fri, 3 Jun 2022 22:59:55 -0400 Subject: [PATCH 29/37] update e2e test --- misc/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/Dockerfile b/misc/Dockerfile index 6c5e5c5f4..35d9f7c4d 100644 --- a/misc/Dockerfile +++ b/misc/Dockerfile @@ -25,6 +25,6 @@ RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer RUN make RUN pip3 install -r misc/requirements.txt -ENV INDEXER_DATA="${HOME}/indexer/" +ENV INDEXER_DATA_DIR="${HOME}/indexer/" # Run test script ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && python3 misc/e2elive.py --connection-string \"$CONNECTION_STRING\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] From f36732e5d7fb1d24fb1fa83c486e017533df7b4a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 6 Jun 2022 09:58:38 -0400 Subject: [PATCH 30/37] fix env var --- cmd/algorand-indexer/daemon.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 8f5ce7187..aba6280ea 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -221,7 +221,6 @@ func init() { viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") viper.RegisterAlias("token", "api-token") - viper.RegisterAlias("data-dir", "data_dir") } From 531588f3a6455fffaa9afd1f6590b3238982446a Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 6 Jun 2022 10:01:56 -0400 Subject: [PATCH 31/37] add an alias --- cmd/algorand-indexer/daemon.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index aba6280ea..24374ae29 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -221,6 +221,7 @@ func init() { viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") viper.RegisterAlias("token", "api-token") + viper.RegisterAlias("data-dir", "data") } From 93b39884683a32a9a6d65630a25f221cf4db8670 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 6 Jun 2022 10:26:37 -0400 Subject: [PATCH 32/37] use dynamic values in test --- processor/eval/ledger_for_evaluator.go | 2 +- processor/eval/ledger_for_evaluator_test.go | 130 ++++++++++---------- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/processor/eval/ledger_for_evaluator.go b/processor/eval/ledger_for_evaluator.go index c9b2c3707..72517c601 100644 --- a/processor/eval/ledger_for_evaluator.go +++ b/processor/eval/ledger_for_evaluator.go @@ -10,7 +10,7 @@ import ( ) // LedgerForEvaluator implements the indexerLedgerForEval interface from -// go-algorand ledger/eval.go and is used for accounting. +// go-algorand ledger/evalindexer.go and is used for accounting. type LedgerForEvaluator struct { Ledger *ledger.Ledger } diff --git a/processor/eval/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go index 517bda3dc..45c918700 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -115,61 +115,61 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { ledger.Creatable{Index: 1, Type: basics.AssetCreatable}: {}, ledger.Creatable{Index: 2, Type: basics.AssetCreatable}: { AssetHolding: &basics.AssetHolding{ - Amount: 4, - Frozen: false, + Amount: txn1.Txn.AssetParams.Total, + Frozen: txn1.Txn.AssetFrozen, }, AssetParams: &basics.AssetParams{ - Total: 4, - Decimals: 0, - DefaultFrozen: false, - UnitName: "", - AssetName: "", - URL: "", - MetadataHash: [32]byte{}, - Manager: test.AccountA, - Reserve: test.AccountA, - Freeze: test.AccountA, - Clawback: test.AccountA, + Total: txn1.Txn.AssetParams.Total, + Decimals: txn1.Txn.AssetParams.Decimals, + DefaultFrozen: txn1.Txn.AssetParams.DefaultFrozen, + UnitName: txn1.Txn.AssetParams.UnitName, + AssetName: txn1.Txn.AssetParams.AssetName, + URL: txn1.Txn.AssetParams.URL, + MetadataHash: txn1.Txn.AssetParams.MetadataHash, + Manager: txn1.Txn.AssetParams.Manager, + Reserve: txn1.Txn.AssetParams.Reserve, + Freeze: txn1.Txn.AssetParams.Freeze, + Clawback: txn1.Txn.AssetParams.Clawback, }, }, ledger.Creatable{Index: 3, Type: basics.AssetCreatable}: { AssetHolding: &basics.AssetHolding{ - Amount: 6, - Frozen: false, + Amount: txn2.Txn.AssetParams.Total, + Frozen: txn2.Txn.AssetFrozen, }, AssetParams: &basics.AssetParams{ - Total: 6, - Decimals: 0, - DefaultFrozen: false, - UnitName: "", - AssetName: "", - URL: "", - MetadataHash: [32]byte{}, - Manager: test.AccountA, - Reserve: test.AccountA, - Freeze: test.AccountA, - Clawback: test.AccountA, + Total: txn2.Txn.AssetParams.Total, + Decimals: txn2.Txn.AssetParams.Decimals, + DefaultFrozen: txn2.Txn.AssetParams.DefaultFrozen, + UnitName: txn2.Txn.AssetParams.UnitName, + AssetName: txn2.Txn.AssetParams.AssetName, + URL: txn2.Txn.AssetParams.URL, + MetadataHash: txn2.Txn.AssetParams.MetadataHash, + Manager: txn2.Txn.AssetParams.Manager, + Reserve: txn2.Txn.AssetParams.Reserve, + Freeze: txn2.Txn.AssetParams.Freeze, + Clawback: txn2.Txn.AssetParams.Clawback, }, }, }, test.AccountB: { ledger.Creatable{Index: 4, Type: basics.AssetCreatable}: { AssetHolding: &basics.AssetHolding{ - Amount: 8, - Frozen: false, + Amount: txn3.Txn.AssetParams.Total, + Frozen: txn3.Txn.AssetFrozen, }, AssetParams: &basics.AssetParams{ - Total: 8, - Decimals: 0, - DefaultFrozen: false, - UnitName: "", - AssetName: "", - URL: "", - MetadataHash: [32]byte{}, - Manager: test.AccountB, - Reserve: test.AccountB, - Freeze: test.AccountB, - Clawback: test.AccountB, + Total: txn3.Txn.AssetParams.Total, + Decimals: txn3.Txn.AssetParams.Decimals, + DefaultFrozen: txn3.Txn.AssetParams.DefaultFrozen, + UnitName: txn3.Txn.AssetParams.UnitName, + AssetName: txn3.Txn.AssetParams.AssetName, + URL: txn3.Txn.AssetParams.URL, + MetadataHash: txn3.Txn.AssetParams.MetadataHash, + Manager: txn3.Txn.AssetParams.Manager, + Reserve: txn3.Txn.AssetParams.Reserve, + Freeze: txn3.Txn.AssetParams.Freeze, + Clawback: txn3.Txn.AssetParams.Clawback, }, }, }, @@ -218,40 +218,40 @@ func TestLedgerForEvaluatorApp(t *testing.T) { ledger.Creatable{Index: 1, Type: basics.AppCreatable}: {}, ledger.Creatable{Index: 2, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: txn1.Txn.ApprovalProgram, + ClearStateProgram: txn1.Txn.ClearStateProgram, GlobalState: nil, StateSchemas: basics.StateSchemas{}, - ExtraProgramPages: 0, + ExtraProgramPages: txn1.Txn.ExtraProgramPages, }, }, ledger.Creatable{Index: 3, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: txn2.Txn.ApprovalProgram, + ClearStateProgram: txn2.Txn.ClearStateProgram, GlobalState: nil, StateSchemas: basics.StateSchemas{}, - ExtraProgramPages: 0, + ExtraProgramPages: txn2.Txn.ExtraProgramPages, }, }, ledger.Creatable{Index: 4, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: txn3.Txn.ApprovalProgram, + ClearStateProgram: txn3.Txn.ClearStateProgram, GlobalState: nil, StateSchemas: basics.StateSchemas{}, - ExtraProgramPages: 0, + ExtraProgramPages: txn3.Txn.ExtraProgramPages, }, }, }, test.AccountB: { ledger.Creatable{Index: 6, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: txn5.Txn.ApprovalProgram, + ClearStateProgram: txn5.Txn.ClearStateProgram, GlobalState: nil, StateSchemas: basics.StateSchemas{}, - ExtraProgramPages: 0, + ExtraProgramPages: txn5.Txn.ExtraProgramPages, }, }, }, @@ -290,11 +290,11 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { test.AccountA: { ledger.Creatable{Index: 1, Type: basics.AppCreatable}: { AppParams: &basics.AppParams{ - ApprovalProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, - ClearStateProgram: []byte{0x02, 0x20, 0x01, 0x01, 0x22}, + ApprovalProgram: txn0.Txn.ApprovalProgram, + ClearStateProgram: txn0.Txn.ClearStateProgram, GlobalState: nil, StateSchemas: basics.StateSchemas{}, - ExtraProgramPages: 0, + ExtraProgramPages: txn0.Txn.ExtraProgramPages, }, }, ledger.Creatable{Index: 2, Type: basics.AssetCreatable}: { @@ -303,17 +303,17 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { Frozen: false, }, AssetParams: &basics.AssetParams{ - Total: 2, - Decimals: 0, - DefaultFrozen: false, - UnitName: "", - AssetName: "", - URL: "", - MetadataHash: [32]byte{}, - Manager: test.AccountA, - Reserve: test.AccountA, - Freeze: test.AccountA, - Clawback: test.AccountA, + Total: txn1.Txn.AssetParams.Total, + Decimals: txn1.Txn.AssetParams.Decimals, + DefaultFrozen: txn1.Txn.AssetParams.DefaultFrozen, + UnitName: txn1.Txn.AssetParams.UnitName, + AssetName: txn1.Txn.AssetParams.AssetName, + URL: txn1.Txn.AssetParams.URL, + MetadataHash: txn1.Txn.AssetParams.MetadataHash, + Manager: txn1.Txn.AssetParams.Manager, + Reserve: txn1.Txn.AssetParams.Reserve, + Freeze: txn1.Txn.AssetParams.Freeze, + Clawback: txn1.Txn.AssetParams.Clawback, }, }, }, From 2a5cb8bc33b57ff2c00853843c50d1e0a9263dc5 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Mon, 6 Jun 2022 10:29:56 -0400 Subject: [PATCH 33/37] remove env var alias --- cmd/algorand-indexer/daemon.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 24374ae29..aba6280ea 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -221,7 +221,6 @@ func init() { viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") viper.RegisterAlias("token", "api-token") - viper.RegisterAlias("data-dir", "data") } From 66c3c7041fcb38e4f258a301a700e0f0e46b02a1 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 7 Jun 2022 15:05:43 -0400 Subject: [PATCH 34/37] pr reviews --- README.md | 54 ++++++++++---------- cmd/algorand-indexer/daemon.go | 1 + misc/Dockerfile | 2 +- processor/eval/ledger_for_evaluator_test.go | 56 ++------------------- 4 files changed, 33 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index c7c23fe8a..78d471130 100644 --- a/README.md +++ b/README.md @@ -168,34 +168,34 @@ If the maximum number of connections/active queries is reached, subsequent conne Settings can be provided from the command line, a configuration file, or an environment variable -| Command Line Flag (long) | (short) | Config File | Environment Variable | -|-------------------------------|---------|-------------------------------|---------------------------------------| -| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | -| pidfile | | pidfile | INDEXER_PIDFILE | -| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | -| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | -| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | -| genesis | g | genesis | INDEXER_GENESIS | -| server | S | server-address | INDEXER_SERVER_ADDRESS | -| no-algod | | no-algod | INDEXER_NO_ALGOD | -| token | t | api-token | INDEXER_API_TOKEN | -| dev-mode | | dev-mode | INDEXER_DEV_MODE | -| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | -| max-conn | | max-conn | INDEXER_MAX_CONN | -| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT | -| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT | +| Command Line Flag (long) | (short) | Config File | Environment Variable | +|-------------------------------|---------|---------------------------|-----------------------------------| +| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | +| pidfile | | pidfile | INDEXER_PIDFILE | +| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | +| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | +| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | +| genesis | g | genesis | INDEXER_GENESIS | +| server | S | server-address | INDEXER_SERVER_ADDRESS | +| no-algod | | no-algod | INDEXER_NO_ALGOD | +| token | t | api-token | INDEXER_API_TOKEN | +| dev-mode | | dev-mode | INDEXER_DEV_MODE | +| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | +| max-conn | | max-conn | INDEXER_MAX_CONN | +| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT | +| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT | | max-api-resources-per-account | | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT | -| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT | -| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT | -| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT | -| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT | -| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT | -| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT | -| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT | -| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | -| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | -| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | -| data-dir | i | data-dir | INDEXER_DATA_DIR | +| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT | +| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT | +| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT | +| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT | +| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT | +| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT | +| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT | +| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | +| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | +| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | +| data-dir | i | data | INDEXER_DATA | ## Command line The command line arguments always take priority over the config file and environment variables. diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index aba6280ea..24374ae29 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -221,6 +221,7 @@ func init() { viper.RegisterAlias("algod-net", "algod-address") viper.RegisterAlias("server", "server-address") viper.RegisterAlias("token", "api-token") + viper.RegisterAlias("data-dir", "data") } diff --git a/misc/Dockerfile b/misc/Dockerfile index 35d9f7c4d..6c5e5c5f4 100644 --- a/misc/Dockerfile +++ b/misc/Dockerfile @@ -25,6 +25,6 @@ RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer RUN make RUN pip3 install -r misc/requirements.txt -ENV INDEXER_DATA_DIR="${HOME}/indexer/" +ENV INDEXER_DATA="${HOME}/indexer/" # Run test script ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && python3 misc/e2elive.py --connection-string \"$CONNECTION_STRING\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] diff --git a/processor/eval/ledger_for_evaluator_test.go b/processor/eval/ledger_for_evaluator_test.go index 45c918700..73b51b249 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/processor/eval/ledger_for_evaluator_test.go @@ -118,38 +118,14 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { Amount: txn1.Txn.AssetParams.Total, Frozen: txn1.Txn.AssetFrozen, }, - AssetParams: &basics.AssetParams{ - Total: txn1.Txn.AssetParams.Total, - Decimals: txn1.Txn.AssetParams.Decimals, - DefaultFrozen: txn1.Txn.AssetParams.DefaultFrozen, - UnitName: txn1.Txn.AssetParams.UnitName, - AssetName: txn1.Txn.AssetParams.AssetName, - URL: txn1.Txn.AssetParams.URL, - MetadataHash: txn1.Txn.AssetParams.MetadataHash, - Manager: txn1.Txn.AssetParams.Manager, - Reserve: txn1.Txn.AssetParams.Reserve, - Freeze: txn1.Txn.AssetParams.Freeze, - Clawback: txn1.Txn.AssetParams.Clawback, - }, + AssetParams: &txn1.Txn.AssetParams, }, ledger.Creatable{Index: 3, Type: basics.AssetCreatable}: { AssetHolding: &basics.AssetHolding{ Amount: txn2.Txn.AssetParams.Total, Frozen: txn2.Txn.AssetFrozen, }, - AssetParams: &basics.AssetParams{ - Total: txn2.Txn.AssetParams.Total, - Decimals: txn2.Txn.AssetParams.Decimals, - DefaultFrozen: txn2.Txn.AssetParams.DefaultFrozen, - UnitName: txn2.Txn.AssetParams.UnitName, - AssetName: txn2.Txn.AssetParams.AssetName, - URL: txn2.Txn.AssetParams.URL, - MetadataHash: txn2.Txn.AssetParams.MetadataHash, - Manager: txn2.Txn.AssetParams.Manager, - Reserve: txn2.Txn.AssetParams.Reserve, - Freeze: txn2.Txn.AssetParams.Freeze, - Clawback: txn2.Txn.AssetParams.Clawback, - }, + AssetParams: &txn2.Txn.AssetParams, }, }, test.AccountB: { @@ -158,19 +134,7 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { Amount: txn3.Txn.AssetParams.Total, Frozen: txn3.Txn.AssetFrozen, }, - AssetParams: &basics.AssetParams{ - Total: txn3.Txn.AssetParams.Total, - Decimals: txn3.Txn.AssetParams.Decimals, - DefaultFrozen: txn3.Txn.AssetParams.DefaultFrozen, - UnitName: txn3.Txn.AssetParams.UnitName, - AssetName: txn3.Txn.AssetParams.AssetName, - URL: txn3.Txn.AssetParams.URL, - MetadataHash: txn3.Txn.AssetParams.MetadataHash, - Manager: txn3.Txn.AssetParams.Manager, - Reserve: txn3.Txn.AssetParams.Reserve, - Freeze: txn3.Txn.AssetParams.Freeze, - Clawback: txn3.Txn.AssetParams.Clawback, - }, + AssetParams: &txn3.Txn.AssetParams, }, }, } @@ -302,19 +266,7 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { Amount: 2, Frozen: false, }, - AssetParams: &basics.AssetParams{ - Total: txn1.Txn.AssetParams.Total, - Decimals: txn1.Txn.AssetParams.Decimals, - DefaultFrozen: txn1.Txn.AssetParams.DefaultFrozen, - UnitName: txn1.Txn.AssetParams.UnitName, - AssetName: txn1.Txn.AssetParams.AssetName, - URL: txn1.Txn.AssetParams.URL, - MetadataHash: txn1.Txn.AssetParams.MetadataHash, - Manager: txn1.Txn.AssetParams.Manager, - Reserve: txn1.Txn.AssetParams.Reserve, - Freeze: txn1.Txn.AssetParams.Freeze, - Clawback: txn1.Txn.AssetParams.Clawback, - }, + AssetParams: &txn1.Txn.AssetParams, }, }, } From aef1b3b15147fefe980f4bc3a2b122f0830d28fb Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 7 Jun 2022 17:31:20 -0400 Subject: [PATCH 35/37] update ledger init --- cmd/algorand-indexer/daemon.go | 5 +---- processor/blockprocessor/block_processor.go | 7 ++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 24374ae29..8913e8325 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -24,7 +24,6 @@ import ( "github.com/algorand/indexer/importer" "github.com/algorand/indexer/processor" "github.com/algorand/indexer/processor/blockprocessor" - "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/metrics" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -150,10 +149,8 @@ var daemonCmd = &cobra.Command{ maybeFail(err, "Error reading genesis file") genesisBlock, err := getGenesisBlock(bot.Algod()) maybeFail(err, "Error getting genesis block") - initState, err := util.CreateInitState(&genesis, &genesisBlock) - maybeFail(err, "Error creating init state") - proc, err := blockprocessor.MakeProcessor(initState, indexerDataDir, imp.ImportBlock) + proc, err := blockprocessor.MakeProcessor(&genesis, &genesisBlock, indexerDataDir, imp.ImportBlock) if err != nil { maybeFail(err, "blockprocessor.MakeProcessor() err %v", err) } diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go index 72afecbf9..cea421566 100644 --- a/processor/blockprocessor/block_processor.go +++ b/processor/blockprocessor/block_processor.go @@ -17,6 +17,7 @@ import ( "github.com/algorand/indexer/accounting" "github.com/algorand/indexer/processor" iledger "github.com/algorand/indexer/processor/eval" + "github.com/algorand/indexer/util" ) type blockProcessor struct { @@ -37,7 +38,11 @@ func MakeProcessorWithLedger(l *ledger.Ledger, handler func(block *ledgercore.Va } // MakeProcessor creates a block processor -func MakeProcessor(initState ledgercore.InitState, datadir string, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { +func MakeProcessor(genesis *bookkeeping.Genesis, genesisBlock *bookkeeping.Block, datadir string, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { + initState, err := util.CreateInitState(genesis, genesisBlock) + if err != nil { + return nil, fmt.Errorf("MakeProcessor() err: %w", err) + } l, err := ledger.OpenLedger(logging.NewLogger(), filepath.Join(path.Dir(datadir), "ledger"), false, initState, algodConfig.GetDefaultLocal()) if err != nil { return nil, fmt.Errorf("MakeProcessor() err: %w", err) From b1bc03db19c911e3cb24ab6e19ec62bc57b8b5d2 Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 7 Jun 2022 18:40:56 -0400 Subject: [PATCH 36/37] format readme --- README.md | 65 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 78d471130..3040096b8 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ Indexer is part of the [sandbox](https://github.com/algorand/sandbox) private ne - Search and filter accounts, transactions, assets, and asset balances with many different parameters. - Pagination of results. - Enriched transaction and account data: - - Confirmation round (block containing the transaction) - - Confirmation time - - Signature type - - Close amounts - - Create/delete rounds. + - Confirmation round (block containing the transaction) + - Confirmation time + - Signature type + - Close amounts + - Create/delete rounds. - Human readable field names instead of the space optimized protocol level names. # Contributing @@ -168,34 +168,35 @@ If the maximum number of connections/active queries is reached, subsequent conne Settings can be provided from the command line, a configuration file, or an environment variable -| Command Line Flag (long) | (short) | Config File | Environment Variable | -|-------------------------------|---------|---------------------------|-----------------------------------| -| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | -| pidfile | | pidfile | INDEXER_PIDFILE | -| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | -| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | -| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | -| genesis | g | genesis | INDEXER_GENESIS | -| server | S | server-address | INDEXER_SERVER_ADDRESS | -| no-algod | | no-algod | INDEXER_NO_ALGOD | -| token | t | api-token | INDEXER_API_TOKEN | -| dev-mode | | dev-mode | INDEXER_DEV_MODE | -| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | -| max-conn | | max-conn | INDEXER_MAX_CONN | -| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT | -| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT | +| Command Line Flag (long) | (short) | Config File | Environment Variable | +|-------------------------------|---------|-------------------------------|---------------------------------------| +| postgres | P | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING | +| pidfile | | pidfile | INDEXER_PIDFILE | +| algod | d | algod-data-dir | INDEXER_ALGOD_DATA_DIR | +| algod-net | | algod-address | INDEXER_ALGOD_ADDRESS | +| algod-token | | algod-token | INDEXER_ALGOD_TOKEN | +| genesis | g | genesis | INDEXER_GENESIS | +| server | S | server-address | INDEXER_SERVER_ADDRESS | +| no-algod | | no-algod | INDEXER_NO_ALGOD | +| token | t | api-token | INDEXER_API_TOKEN | +| dev-mode | | dev-mode | INDEXER_DEV_MODE | +| metrics-mode | | metrics-mode | INDEXER_METRICS_MODE | +| max-conn | | max-conn | INDEXER_MAX_CONN | +| write-timeout | | write-timeout | INDEXER_WRITE_TIMEOUT | +| read-timeout | | read-timeout | INDEXER_READ_TIMEOUT | | max-api-resources-per-account | | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT | -| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT | -| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT | -| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT | -| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT | -| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT | -| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT | -| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT | -| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | -| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | -| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | -| data-dir | i | data | INDEXER_DATA | +| max-transactions-limit | | max-transactions-limit | INDEXER_MAX_TRANSACTIONS_LIMIT | +| default-transactions-limit | | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT | +| max-accounts-limit | | max-accounts-limit | INDEXER_MAX_ACCOUNTS_LIMIT | +| default-accounts-limit | | default-accounts-limit | INDEXER_DEFAULT_ACCOUNTS_LIMIT | +| max-assets-limit | | max-assets-limit | INDEXER_MAX_ASSETS_LIMIT | +| default-assets-limit | | default-assets-limit | INDEXER_DEFAULT_ASSETS_LIMIT | +| max-balances-limit | | max-balances-limit | INDEXER_MAX_BALANCES_LIMIT | +| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT | +| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT | +| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT | +| data-dir | i | data | INDEXER_DATA | + ## Command line The command line arguments always take priority over the config file and environment variables. From 664bdd0142654e72249c2515f50dee0f432b128b Mon Sep 17 00:00:00 2001 From: "shiqi.zheng@algorand.com" Date: Tue, 7 Jun 2022 18:42:50 -0400 Subject: [PATCH 37/37] format readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3040096b8..99e84347a 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ Indexer is part of the [sandbox](https://github.com/algorand/sandbox) private ne - Search and filter accounts, transactions, assets, and asset balances with many different parameters. - Pagination of results. - Enriched transaction and account data: - - Confirmation round (block containing the transaction) - - Confirmation time - - Signature type - - Close amounts - - Create/delete rounds. + - Confirmation round (block containing the transaction) + - Confirmation time + - Signature type + - Close amounts + - Create/delete rounds. - Human readable field names instead of the space optimized protocol level names. # Contributing