adding a function to safely open boltdb files and recover from any pa…

…nics thrown by boltdb when opening a file. (#2983)

pkg/storage/stores/shipper/compactor/table.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
-	"github.com/cortexproject/cortex/pkg/chunk/local"
 	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/go-kit/kit/log/level"
@@ -82,7 +81,7 @@ func (t *table) compact() error {
 		}
 	}()
 
-	t.compactedDB, err = local.OpenBoltdbFile(filepath.Join(t.workingDirectory, fmt.Sprint(time.Now().Unix())))
+	t.compactedDB, err = shipper_util.SafeOpenBoltdbFile(filepath.Join(t.workingDirectory, fmt.Sprint(time.Now().Unix())))
 	if err != nil {
 		return err
 	}
@@ -221,7 +220,7 @@ func (t *table) writeBatch(batch []indexEntry) error {
 func (t *table) readFile(path string) error {
 	level.Debug(util.Logger).Log("msg", "reading file for compaction", "path", path)
 
-	db, err := local.OpenBoltdbFile(path)
+	db, err := shipper_util.SafeOpenBoltdbFile(path)
 	if err != nil {
 		return err
 	}

pkg/storage/stores/shipper/downloads/table.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/cortexproject/cortex/pkg/chunk"
-	"github.com/cortexproject/cortex/pkg/chunk/local"
 	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/spanlogger"
@@ -142,7 +141,7 @@ func LoadTable(ctx context.Context, name, cacheLocation string, storageClient St
 		}
 
 		// if we fail to open a boltdb file, lets skip it and let sync operation re-download the file from storage.
-		boltdb, err := local.OpenBoltdbFile(filepath.Join(folderPath, fileInfo.Name()))
+		boltdb, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(folderPath, fileInfo.Name()))
 		if err != nil {
 			level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to open existing boltdb file %s, continuing without it to let the sync operation catch up", filepath.Join(folderPath, fileInfo.Name())), "err", err)
 			continue
@@ -218,7 +217,7 @@ func (t *Table) init(ctx context.Context, spanLogger log.Logger) (err error) {
 		}
 
 		filePath := path.Join(folderPath, dbName)
-		boltdb, err := local.OpenBoltdbFile(filePath)
+		boltdb, err := shipper_util.SafeOpenBoltdbFile(filePath)
 		if err != nil {
 			return err
 		}
@@ -452,7 +451,7 @@ func (t *Table) downloadFile(ctx context.Context, storageObject chunk.StorageObj
 	t.dbsMtx.Lock()
 	defer t.dbsMtx.Unlock()
 
-	boltdb, err := local.OpenBoltdbFile(filePath)
+	boltdb, err := shipper_util.SafeOpenBoltdbFile(filePath)
 	if err != nil {
 		return err
 	}

pkg/storage/stores/shipper/uploads/table.go

@@ -168,7 +168,7 @@ func (lt *Table) Snapshot() error {
 			return err
 		}
 
-		snapshot.boltdb, err = local.OpenBoltdbFile(filePath)
+		snapshot.boltdb, err = shipper_util.SafeOpenBoltdbFile(filePath)
 		if err != nil {
 			return err
 		}
@@ -228,7 +228,7 @@ func (lt *Table) getOrAddDB(name string) (*bbolt.DB, error) {
 
 	db, ok = lt.dbs[name]
 	if !ok {
-		db, err = local.OpenBoltdbFile(filepath.Join(lt.path, name))
+		db, err = shipper_util.SafeOpenBoltdbFile(filepath.Join(lt.path, name))
 		if err != nil {
 			return nil, err
 		}
@@ -490,7 +490,7 @@ func loadBoltDBsFromDir(dir string) (map[string]*bbolt.DB, error) {
 			continue
 		}
 
-		db, err := local.OpenBoltdbFile(filepath.Join(dir, fileInfo.Name()))
+		db, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(dir, fileInfo.Name()))
 		if err != nil {
 			return nil, err
 		}

pkg/storage/stores/shipper/util/util.go

@@ -7,6 +7,9 @@ import (
 	"os"
 	"strings"
 
+	"github.com/cortexproject/cortex/pkg/chunk/local"
+	"go.etcd.io/bbolt"
+
 	"github.com/grafana/loki/pkg/chunkenc"
 
 	"github.com/cortexproject/cortex/pkg/util"
@@ -116,3 +119,13 @@ func CompressFile(src, dest string) error {
 
 	return compressedFile.Sync()
 }
+
+// SafeOpenBoltdbFile will recover from a panic opening a DB file, and return the panic message in the err return object.
+func SafeOpenBoltdbFile(path string) (boltdb *bbolt.DB, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("recovered from panic opening boltdb file: %v", r)
+		}
+	}()
+	return local.OpenBoltdbFile(path)
+}