From 6d442673b7279eef4b7aa88d0af0ceb72bba7fe9 Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Tue, 24 Nov 2020 09:47:29 -0500 Subject: [PATCH] adding a function to safely open boltdb files and recover from any panics thrown by boltdb when opening a file. (#2983) --- pkg/storage/stores/shipper/compactor/table.go | 5 ++--- pkg/storage/stores/shipper/downloads/table.go | 7 +++---- pkg/storage/stores/shipper/uploads/table.go | 6 +++--- pkg/storage/stores/shipper/util/util.go | 13 +++++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pkg/storage/stores/shipper/compactor/table.go b/pkg/storage/stores/shipper/compactor/table.go index 5d58789c3c96..33f2758fb7a0 100644 --- a/pkg/storage/stores/shipper/compactor/table.go +++ b/pkg/storage/stores/shipper/compactor/table.go @@ -9,7 +9,6 @@ import ( "time" "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/local" chunk_util "github.com/cortexproject/cortex/pkg/chunk/util" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" @@ -82,7 +81,7 @@ func (t *table) compact() error { } }() - t.compactedDB, err = local.OpenBoltdbFile(filepath.Join(t.workingDirectory, fmt.Sprint(time.Now().Unix()))) + t.compactedDB, err = shipper_util.SafeOpenBoltdbFile(filepath.Join(t.workingDirectory, fmt.Sprint(time.Now().Unix()))) if err != nil { return err } @@ -221,7 +220,7 @@ func (t *table) writeBatch(batch []indexEntry) error { func (t *table) readFile(path string) error { level.Debug(util.Logger).Log("msg", "reading file for compaction", "path", path) - db, err := local.OpenBoltdbFile(path) + db, err := shipper_util.SafeOpenBoltdbFile(path) if err != nil { return err } diff --git a/pkg/storage/stores/shipper/downloads/table.go b/pkg/storage/stores/shipper/downloads/table.go index 7c955ea7395b..2dc42178a62e 100644 --- a/pkg/storage/stores/shipper/downloads/table.go +++ b/pkg/storage/stores/shipper/downloads/table.go @@ -13,7 +13,6 @@ import ( "time" "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/local" chunk_util "github.com/cortexproject/cortex/pkg/chunk/util" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/spanlogger" @@ -142,7 +141,7 @@ func LoadTable(ctx context.Context, name, cacheLocation string, storageClient St } // if we fail to open a boltdb file, lets skip it and let sync operation re-download the file from storage. - boltdb, err := local.OpenBoltdbFile(filepath.Join(folderPath, fileInfo.Name())) + boltdb, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(folderPath, fileInfo.Name())) if err != nil { level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to open existing boltdb file %s, continuing without it to let the sync operation catch up", filepath.Join(folderPath, fileInfo.Name())), "err", err) continue @@ -218,7 +217,7 @@ func (t *Table) init(ctx context.Context, spanLogger log.Logger) (err error) { } filePath := path.Join(folderPath, dbName) - boltdb, err := local.OpenBoltdbFile(filePath) + boltdb, err := shipper_util.SafeOpenBoltdbFile(filePath) if err != nil { return err } @@ -452,7 +451,7 @@ func (t *Table) downloadFile(ctx context.Context, storageObject chunk.StorageObj t.dbsMtx.Lock() defer t.dbsMtx.Unlock() - boltdb, err := local.OpenBoltdbFile(filePath) + boltdb, err := shipper_util.SafeOpenBoltdbFile(filePath) if err != nil { return err } diff --git a/pkg/storage/stores/shipper/uploads/table.go b/pkg/storage/stores/shipper/uploads/table.go index 390eac05f096..8520517e8f9f 100644 --- a/pkg/storage/stores/shipper/uploads/table.go +++ b/pkg/storage/stores/shipper/uploads/table.go @@ -168,7 +168,7 @@ func (lt *Table) Snapshot() error { return err } - snapshot.boltdb, err = local.OpenBoltdbFile(filePath) + snapshot.boltdb, err = shipper_util.SafeOpenBoltdbFile(filePath) if err != nil { return err } @@ -228,7 +228,7 @@ func (lt *Table) getOrAddDB(name string) (*bbolt.DB, error) { db, ok = lt.dbs[name] if !ok { - db, err = local.OpenBoltdbFile(filepath.Join(lt.path, name)) + db, err = shipper_util.SafeOpenBoltdbFile(filepath.Join(lt.path, name)) if err != nil { return nil, err } @@ -490,7 +490,7 @@ func loadBoltDBsFromDir(dir string) (map[string]*bbolt.DB, error) { continue } - db, err := local.OpenBoltdbFile(filepath.Join(dir, fileInfo.Name())) + db, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(dir, fileInfo.Name())) if err != nil { return nil, err } diff --git a/pkg/storage/stores/shipper/util/util.go b/pkg/storage/stores/shipper/util/util.go index 5012654d9a94..4b34ebe208c5 100644 --- a/pkg/storage/stores/shipper/util/util.go +++ b/pkg/storage/stores/shipper/util/util.go @@ -7,6 +7,9 @@ import ( "os" "strings" + "github.com/cortexproject/cortex/pkg/chunk/local" + "go.etcd.io/bbolt" + "github.com/grafana/loki/pkg/chunkenc" "github.com/cortexproject/cortex/pkg/util" @@ -116,3 +119,13 @@ func CompressFile(src, dest string) error { return compressedFile.Sync() } + +// SafeOpenBoltdbFile will recover from a panic opening a DB file, and return the panic message in the err return object. +func SafeOpenBoltdbFile(path string) (boltdb *bbolt.DB, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("recovered from panic opening boltdb file: %v", r) + } + }() + return local.OpenBoltdbFile(path) +}