From e228d447101ab727b4fd0b021308987493421620 Mon Sep 17 00:00:00 2001 From: shiqizng <80276844+shiqizng@users.noreply.github.com> Date: Mon, 3 Oct 2022 12:08:22 -0400 Subject: [PATCH 1/6] fix: use CI_E2E_FILENAME in s3 prefix (#1255) * fix e2e --- misc/e2elive.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/misc/e2elive.py b/misc/e2elive.py index 0d3714d34..b03742533 100644 --- a/misc/e2elive.py +++ b/misc/e2elive.py @@ -83,12 +83,17 @@ def main(): from botocore.config import Config from botocore import UNSIGNED s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED)) + prefix = "indexer/e2e4" if "/" in tarname: - tarname = tarname.split("/")[1] + # parse tarname like fafa8862/rel-nightly + cmhash_tarnme = tarname.split("/") + cmhash = cmhash_tarnme[0] + tarname =cmhash_tarnme[1] + prefix+="/"+cmhash tarpath = os.path.join(tempdir, tarname) else: tarpath = os.path.join(tempdir, tarname) - prefix = "indexer/e2e4" + success = firstFromS3Prefix(s3, bucket, prefix, tarname, outpath=tarpath) if not success: raise Exception( @@ -203,7 +208,7 @@ def countblocks(path): return row[0] -def tryhealthurl(healthurl, verbose=False, waitforround=100): +def tryhealthurl(healthurl, verbose=False, waitforround=200): try: response = urllib.request.urlopen(healthurl) if response.code != 200: From 78fac368144dcc84c233332313b96c2909244649 Mon Sep 17 00:00:00 2001 From: Zeph Grunschlag Date: Wed, 5 Oct 2022 09:52:20 -0500 Subject: [PATCH 2/6] Bug-Fix: Update go-algorand Submodule and Wrap Ledger in MakeNewCatchpointCatchupAccessor (#1257) * MakeCatchpointCatchupAccessor * update test job's CI_E2E_FILENAME to latest rel-nightly s3 key --- .circleci/config.yml | 2 +- processor/blockprocessor/internal/catchupservice.go | 3 ++- third_party/go-algorand | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 298f1a4c9..7fa6c2ea7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,7 @@ jobs: go_version: type: string environment: - CI_E2E_FILENAME: "fafa8862/rel-nightly" + CI_E2E_FILENAME: "fad05790/rel-nightly" steps: - go/install: version: << parameters.go_version >> diff --git a/processor/blockprocessor/internal/catchupservice.go b/processor/blockprocessor/internal/catchupservice.go index 73bf1c4c6..0ef4411dd 100644 --- a/processor/blockprocessor/internal/catchupservice.go +++ b/processor/blockprocessor/internal/catchupservice.go @@ -10,6 +10,7 @@ import ( "github.com/algorand/go-algorand/catchup" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" @@ -97,7 +98,7 @@ func CatchupServiceCatchup(ctx context.Context, logger *log.Logger, catchpoint, node, wrappedLogger, net, - l, + ledger.MakeCatchpointCatchupAccessor(l, wrappedLogger), cfg, ) if err != nil { diff --git a/third_party/go-algorand b/third_party/go-algorand index b01652680..cbdc68ad3 160000 --- a/third_party/go-algorand +++ b/third_party/go-algorand @@ -1 +1 @@ -Subproject commit b01652680098c6a64bf6c6bb8246adc407162164 +Subproject commit cbdc68ad329d46441bc4947f27d3f6622ffa6248 From 1b5efe081dec99842267052f1ca8248ab3b09b81 Mon Sep 17 00:00:00 2001 From: AlgoStephenAkiki <85183435+AlgoStephenAkiki@users.noreply.github.com> Date: Sat, 8 Oct 2022 08:29:05 -0400 Subject: [PATCH 3/6] Fix configuration of logger (#1259) (#1269) Resolves #1246 Fixes order of configuration so that logger can be set properly --- cmd/algorand-indexer/daemon.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index 3ef7254eb..83f36ee85 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -206,7 +206,6 @@ func loadIndexerConfig(indexerDataDir string, configFile string) error { logger.WithError(err).Errorf("invalid config file (%s): %v", viper.ConfigFileUsed(), err) return err } - logger.Infof("Using configuration file: %s\n", resolvedConfigPath) return err } @@ -266,11 +265,6 @@ func createIndexerPidFile(pidFilePath string) error { func runDaemon(daemonConfig *daemonConfig) error { var err error config.BindFlagSet(daemonConfig.flags) - err = configureLogger() - if err != nil { - fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err) - return err - } // Create the data directory if necessary/possible if err = configureIndexerDataDir(daemonConfig.indexerDataDir); err != nil { @@ -289,6 +283,15 @@ func runDaemon(daemonConfig *daemonConfig) error { return err } + // Configure the logger after we load all indexer configs + err = configureLogger() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to configure logger: %v", err) + return err + } + + logger.Infof("Using configuration file: %s", viper.ConfigFileUsed()) + if daemonConfig.pidFilePath != "" { err = createIndexerPidFile(daemonConfig.pidFilePath) if err != nil { From b729f51f2004151994b098c5e336f537adb7280e Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 20 Oct 2022 08:50:23 -0400 Subject: [PATCH 4/6] testing: Validator improvements. (#1282) Validator improvements. Report the canonical address name even if the b64 address is used. Avoid reporting skipped accounts unless using new '--print-skipped' flag. Fix REST API error encoding. Detect algod result limit reached errors. --- cmd/validator/core/command.go | 13 ++- cmd/validator/core/validator.go | 19 ++-- cmd/validator/core/validator_test.go | 146 +++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 cmd/validator/core/validator_test.go diff --git a/cmd/validator/core/command.go b/cmd/validator/core/command.go index cf97c7d1a..39249fc03 100644 --- a/cmd/validator/core/command.go +++ b/cmd/validator/core/command.go @@ -22,6 +22,7 @@ func init() { processorNum int printCurl bool errorLogFile string + printSkipped bool ) ValidatorCmd = &cobra.Command{ @@ -29,7 +30,7 @@ func init() { Short: "validator", Long: "Compare algod and indexer to each other and report any discrepencies.", Run: func(cmd *cobra.Command, _ []string) { - run(config, errorLogFile, addr, threads, processorNum, printCurl) + run(config, errorLogFile, addr, threads, processorNum, printCurl, printSkipped) }, } @@ -46,9 +47,10 @@ func init() { ValidatorCmd.Flags().IntVar(&processorNum, "processor", 0, "Choose compare algorithm [0 = Struct, 1 = Reflection]") ValidatorCmd.Flags().BoolVar(&printCurl, "print-commands", false, "Print curl commands, including tokens, to query algod and indexer.") ValidatorCmd.Flags().StringVarP(&errorLogFile, "error-log-file", "e", "", "When specified, error messages are written to this file instead of to stderr.") + ValidatorCmd.Flags().BoolVar(&printSkipped, "print-skipped", false, "Include accounts which were skipped in the error log.") } -func run(config Params, errorLogFile, addr string, threads int, processorNum int, printCurl bool) { +func run(config Params, errorLogFile, addr string, threads int, processorNum int, printCurl, printSkipped bool) { if len(config.AlgodURL) == 0 { ErrorLog.Fatalf("algod-url parameter is required.") } @@ -91,7 +93,7 @@ func run(config Params, errorLogFile, addr string, threads int, processorNum int }() // This will keep going until the results channel is closed. - numErrors := resultsPrinter(config, printCurl, results) + numErrors := resultsPrinter(config, printCurl, printSkipped, results) if numErrors > 0 { os.Exit(1) } @@ -134,7 +136,7 @@ func resultChar(success bool, retries int, skipReason Skip) string { } // resultsPrinter reads the results channel and prints it to the error log. Returns the number of errors. -func resultsPrinter(config Params, printCurl bool, results <-chan Result) int { +func resultsPrinter(config Params, printCurl, printSkipped bool, results <-chan Result) int { numResults := 0 numErrors := 0 skipCounts := make(map[Skip]uint64) @@ -176,6 +178,9 @@ func resultsPrinter(config Params, printCurl bool, results <-chan Result) int { if r.Error != nil || !r.Equal { if r.SkipReason != NotSkipped { skipCounts[r.SkipReason]++ + if !printSkipped { + continue + } } else { numErrors++ } diff --git a/cmd/validator/core/validator.go b/cmd/validator/core/validator.go index e4d5a496d..e54bd33f0 100644 --- a/cmd/validator/core/validator.go +++ b/cmd/validator/core/validator.go @@ -122,7 +122,7 @@ func Start(work <-chan string, processorID ProcessorID, threads int, config Para func CallProcessor(processor Processor, addrInput string, config Params, results chan<- Result) { addr, err := normalizeAddress(addrInput) if err != nil { - results <- resultError(err, addrInput) + results <- resultError(err, addr) return } @@ -133,8 +133,13 @@ func CallProcessor(processor Processor, addrInput string, config Params, results // catches up with the first algod account query. algodData, err := getData(algodDataURL, config.AlgodToken) if err != nil { - err = fmt.Errorf("error getting algod data (%d): %w", algodData, err) - results <- resultError(err, addrInput) + err = fmt.Errorf("error getting algod data (%s): %w", algodData, err) + switch { + case strings.Contains(string(algodData), api.ErrResultLimitReached): + results <- resultSkip(err, addr, SkipLimitReached) + default: + results <- resultError(err, addr) + } return } @@ -142,14 +147,14 @@ func CallProcessor(processor Processor, addrInput string, config Params, results for i := 0; true; i++ { indexerData, err := getData(indexerDataURL, config.IndexerToken) if err != nil { - err = fmt.Errorf("error getting indexer data (%d): %w", indexerData, err) + err = fmt.Errorf("error getting indexer data (%s): %w", indexerData, err) switch { case strings.Contains(string(indexerData), api.ErrResultLimitReached): - results <- resultSkip(err, addrInput, SkipLimitReached) + results <- resultSkip(err, addr, SkipLimitReached) case strings.Contains(string(indexerData), api.ErrNoAccountsFound): - results <- resultSkip(err, addrInput, SkipAccountNotFound) + results <- resultSkip(err, addr, SkipAccountNotFound) default: - results <- resultError(err, addrInput) + results <- resultError(err, addr) } return } diff --git a/cmd/validator/core/validator_test.go b/cmd/validator/core/validator_test.go new file mode 100644 index 000000000..652ba7d92 --- /dev/null +++ b/cmd/validator/core/validator_test.go @@ -0,0 +1,146 @@ +package core + +import ( + "fmt" + "testing" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand-sdk/types" + + "github.com/algorand/indexer/api" +) + +type mockProcessor struct { + result Result + err error +} + +func (m *mockProcessor) ProcessAddress(_ []byte, _ []byte) (Result, error) { + return m.result, m.err +} + +func TestCallProcessor(t *testing.T) { + mockParams := Params{ + AlgodURL: "http://localhost/algod", + IndexerURL: "http://localhost/indexer", + } + var testAddr types.Address + testAddr[0] = 1 + + type args struct { + processor Processor + addrInput string + config Params + algodData string + algodCode int + indexerData string + indexerCode int + errorStr string + } + tests := []struct { + name string + args args + }{ + { + name: "Bad address", + args: args{addrInput: "399 Boylston Street", errorStr: "unable to decode address"}, + }, { + name: "algod: acct limit", + args: args{ + algodCode: 400, + algodData: api.ErrResultLimitReached, + errorStr: fmt.Sprintf("error getting algod data (%s)", api.ErrResultLimitReached), + }, + }, { + name: "algod: unknown error", + args: args{ + algodCode: 500, + algodData: "server sent GOAWAY and closed the connection", + errorStr: "error getting algod data (server sent GOAWAY and closed the connection): bad status: 500", + }, + }, { + name: "indexer: acct limit", + args: args{ + indexerCode: 400, + indexerData: api.ErrResultLimitReached, + errorStr: fmt.Sprintf("error getting indexer data (%s)", api.ErrResultLimitReached), + }, + }, { + name: "indexer: acct not found", + args: args{ + indexerCode: 404, + indexerData: api.ErrNoAccountsFound, + errorStr: fmt.Sprintf("error getting indexer data (%s)", api.ErrNoAccountsFound), + }, + }, { + name: "indexer: unknown", + args: args{ + indexerCode: 500, + indexerData: "server sent GOAWAY and closed the connection", + errorStr: "error getting indexer data (server sent GOAWAY and closed the connection): bad status: 500", + }, + }, { + name: "processor: error", + args: args{ + processor: &mockProcessor{ + err: fmt.Errorf("ask again tomorrow"), + }, + addrInput: testAddr.String(), + errorStr: fmt.Sprintf("error processing account %s: ask again tomorrow", testAddr.String()), + }, + }, + } + setDefaults := func(a args) args { + if a.config == (Params{}) { + a.config = mockParams + } + if a.algodCode == 0 { + a.algodCode = 200 + if a.algodData == "" { + a.algodData = "{}" + } + } + if a.indexerCode == 0 { + a.indexerCode = 200 + if a.indexerData == "" { + a.indexerData = "{}" + } + } + return a + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.args = setDefaults(tt.args) + // Given: mock http responses + if tt.args.algodData != "" || tt.args.indexerData != "" { + httpmock.Activate() + defer httpmock.DeactivateAndReset() + } + if tt.args.algodData != "" { + httpmock.RegisterResponder("GET", "=~"+mockParams.AlgodURL+".*", + httpmock.NewStringResponder(tt.args.algodCode, tt.args.algodData)) + } + if tt.args.indexerData != "" { + httpmock.RegisterResponder("GET", "=~"+mockParams.IndexerURL+".*", + httpmock.NewStringResponder(tt.args.indexerCode, tt.args.indexerData)) + } + + // When: call CallProcessor + // We only use CallProcessor once, so a buffered channel of 1 should be sufficient. + results := make(chan Result, 1) + CallProcessor(tt.args.processor, tt.args.addrInput, tt.args.config, results) + close(results) + + // Then: we get the expected result. + for result := range results { + if tt.args.errorStr != "" { + require.ErrorContains(t, result.Error, tt.args.errorStr) + } else { + require.NoError(t, result.Error) + } + } + }) + } +} From d9007e832ad37d3ca1f6f9fc6a2d9d2f87040802 Mon Sep 17 00:00:00 2001 From: John Lee Date: Thu, 20 Oct 2022 18:18:15 -0400 Subject: [PATCH 5/6] DevOps: Update go-algorand submodule to v3.11.2-beta (#1286) --- go.mod | 3 ++- go.sum | 2 ++ third_party/go-algorand | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 8ea1c33b4..3ab1e7e4c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/algorand/go-algorand v0.0.0-20220211161928-53b157beb10f github.com/algorand/go-algorand-sdk v1.9.1 github.com/algorand/go-codec/codec v1.1.8 + github.com/algorand/go-deadlock v0.2.2 github.com/algorand/oapi-codegen v1.3.7 github.com/davecgh/go-spew v1.1.1 github.com/getkin/kin-openapi v0.22.0 @@ -29,10 +30,10 @@ require ( ) require ( + github.com/DataDog/zstd v1.5.2 // indirect github.com/Microsoft/go-winio v0.4.14 // indirect github.com/algorand/avm-abi v0.1.0 // indirect github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 // indirect - github.com/algorand/go-deadlock v0.2.2 // indirect github.com/algorand/go-sumhash v0.1.0 // indirect github.com/algorand/msgp v1.1.52 // indirect github.com/algorand/websocket v1.4.5 // indirect diff --git a/go.sum b/go.sum index 56c8d41a3..479af940b 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= diff --git a/third_party/go-algorand b/third_party/go-algorand index cbdc68ad3..afd8b05b9 160000 --- a/third_party/go-algorand +++ b/third_party/go-algorand @@ -1 +1 @@ -Subproject commit cbdc68ad329d46441bc4947f27d3f6622ffa6248 +Subproject commit afd8b05b9a8fa9b49a37159d975fc2367da72fcd From 39159bd7d19ae7ed408299117b929e23dd8d0466 Mon Sep 17 00:00:00 2001 From: DevOps Service Date: Fri, 21 Oct 2022 16:59:40 +0000 Subject: [PATCH 6/6] Bump version to 2.14.2 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index b70ae75a8..7243b12cf 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.14.1 +2.14.2