Commit 9988c587 authored by Bilal Akhtar's avatar Bilal Akhtar

cli: Resolve past engine type correctly with encryption-at-rest

The default engine type resolves to the last used engine type,
which is determined by reading the OPTIONS file in the store directory.
However if encrypted at rest is enabled, we need to set up the encrypted
vfs first before that file can be read. This change makes that happen.

Fixes #48148.

Release note (cli change): Make --storage-engine sticky (i.e. resolve
to the last used engine type when unspecified) even when specified
stores are encrypted at rest.
parent 53607a3a
......@@ -139,7 +139,7 @@ func OpenEngine(dir string, stopper *stop.Stopper, opts OpenEngineOptions) (stor
}
var db storage.Engine
storageEngine := resolveStorageEngineType(storage.DefaultStorageEngine, storageConfig.Dir)
storageEngine := resolveStorageEngineType(context.Background(), storage.DefaultStorageEngine, storageConfig)
switch storageEngine {
case enginepb.EngineTypePebble:
......
#! /usr/bin/env expect -f
#
source [file join [file dirname $argv0] common.tcl]
set storedir "encryption_store"
set keydir "$storedir/keys"
spawn /bin/bash
send "PS1=':''/# '\r"
eexpect ":/# "
start_test "Generate encryption keys."
send "mkdir -p $keydir\n"
send "$argv gen encryption-key -s 128 $keydir/aes-128.key\r"
eexpect "successfully created AES-128 key: $keydir/aes-128.key"
end_test
start_test "Start normal node with default engine."
send "$argv start-single-node --insecure --store=$storedir\r"
eexpect "storage engine: *pebble"
interrupt
eexpect "shutdown completed"
end_test
start_test "Restart normal node with non-default engine specified."
send "$argv start-single-node --insecure --store=$storedir --storage-engine=rocksdb\r"
eexpect "storage engine: *rocksdb"
interrupt
eexpect "shutdown completed"
end_test
start_test "Restart normal node; should resort to non-default engine."
send "$argv start-single-node --insecure --store=$storedir\r"
eexpect "storage engine: *rocksdb"
interrupt
eexpect "shutdown completed"
end_test
start_test "Restart normal node with default engine specified."
send "$argv start-single-node --insecure --store=$storedir --storage-engine=pebble\r"
eexpect "storage engine: *pebble"
interrupt
eexpect "shutdown completed"
end_test
start_test "Restart with AES-128."
send "$argv start-single-node --insecure --store=$storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain\r"
eexpect "storage engine: *pebble"
interrupt
eexpect "shutdown completed"
send "$argv debug encryption-status $storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain\r"
eexpect " \"Active\": true,\r\n \"Type\": \"AES128_CTR\","
end_test
start_test "Restart with AES-128 and specify non-default engine."
send "$argv start-single-node --insecure --store=$storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain --storage-engine=rocksdb\r"
eexpect "storage engine: *rocksdb"
interrupt
eexpect "shutdown completed"
send "$argv debug encryption-status $storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain\r"
eexpect " \"Active\": true,\r\n \"Type\": \"AES128_CTR\","
end_test
start_test "Restart with AES-128 and engine unspecified; should resolve to non-default engine."
send "$argv start-single-node --insecure --store=$storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain\r"
eexpect "storage engine: *rocksdb"
interrupt
eexpect "shutdown completed"
send "$argv debug encryption-status $storedir --enterprise-encryption=path=$storedir,key=$keydir/aes-128.key,old-key=plain\r"
eexpect " \"Active\": true,\r\n \"Type\": \"AES128_CTR\","
end_test
......@@ -53,7 +53,6 @@ import (
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/vfs"
opentracing "github.com/opentracing/opentracing-go"
"github.com/spf13/cobra"
"google.golang.org/grpc"
......@@ -415,12 +414,28 @@ func initTempStorageConfig(
// Checks if the passed-in engine type is default, and if so, resolves it to
// the storage engine last used to write to the store at dir (or rocksdb if
// a store wasn't found).
func resolveStorageEngineType(engineType enginepb.EngineType, dir string) enginepb.EngineType {
func resolveStorageEngineType(
ctx context.Context, engineType enginepb.EngineType, cfg base.StorageConfig,
) enginepb.EngineType {
if engineType == enginepb.EngineTypeDefault {
engineType = enginepb.EngineTypePebble
// Check if this storage directory was last written to by pebble. In that
// case, default to opening a Pebble engine.
if version, err := pebble.GetVersion(dir, vfs.Default); err == nil {
pebbleCfg := &storage.PebbleConfig{
StorageConfig: cfg,
Opts: storage.DefaultPebbleOptions(),
}
pebbleCfg.Opts.EnsureDefaults()
pebbleCfg.Opts.ReadOnly = true
// Resolve encrypted env options in pebbleCfg and populate pebbleCfg.Opts.FS
// if necessary (eg. encrypted-at-rest is enabled).
_, _, err := storage.ResolveEncryptedEnvOptions(pebbleCfg)
if err != nil {
log.Infof(ctx, "unable to setup encrypted env to resolve past engine type: %s", err)
return engineType
}
// Check if this storage directory was last written to by rocksdb. In that
// case, default to opening a RocksDB engine.
if version, err := pebble.GetVersion(cfg.Dir, pebbleCfg.Opts.FS); err == nil {
if strings.HasPrefix(version, "rocksdb") {
engineType = enginepb.EngineTypeRocksDB
}
......@@ -541,10 +556,21 @@ func runStart(cmd *cobra.Command, args []string, disableReplication bool) error
if serverCfg.Settings.ExternalIODir, err = initExternalIODir(ctx, serverCfg.Stores.Specs[0]); err != nil {
return err
}
// Build a minimal StorageConfig out of the first store's spec, with enough
// attributes to be able to read encrypted-at-rest store directories.
firstSpec := serverCfg.Stores.Specs[0]
firstStoreConfig := base.StorageConfig{
Attrs: firstSpec.Attributes,
Dir: firstSpec.Path,
Settings: serverCfg.Settings,
UseFileRegistry: firstSpec.UseFileRegistry,
ExtraOptions: firstSpec.ExtraOptions,
}
// If the storage engine is set to "default", check the engine type used in
// this store directory in a past run. If this check fails for any reason,
// use RocksDB as the default engine type.
serverCfg.StorageEngine = resolveStorageEngineType(serverCfg.StorageEngine, serverCfg.Stores.Specs[0].Path)
// use Pebble as the default engine type.
serverCfg.StorageEngine = resolveStorageEngineType(ctx, serverCfg.StorageEngine, firstStoreConfig)
// Next we initialize the target directory for temporary storage.
// If encryption at rest is enabled in any fashion, we'll want temp
......
......@@ -421,6 +421,44 @@ var _ Engine = &Pebble{}
// code does not depend on CCL code.
var NewEncryptedEnvFunc func(fs vfs.FS, fr *PebbleFileRegistry, dbDir string, readOnly bool, optionBytes []byte) (vfs.FS, EncryptionStatsHandler, error)
// ResolveEncryptedEnvOptions fills in cfg.Opts.FS with an encrypted vfs if this
// store has encryption-at-rest enabled. Also returns the associated file
// registry and EncryptionStatsHandler.
func ResolveEncryptedEnvOptions(
cfg *PebbleConfig,
) (*PebbleFileRegistry, EncryptionStatsHandler, error) {
fileRegistry := &PebbleFileRegistry{FS: cfg.Opts.FS, DBDir: cfg.Dir, ReadOnly: cfg.Opts.ReadOnly}
if cfg.UseFileRegistry {
if err := fileRegistry.Load(); err != nil {
return nil, nil, err
}
} else {
if err := fileRegistry.checkNoRegistryFile(); err != nil {
return nil, nil, fmt.Errorf("encryption was used on this store before, but no encryption flags " +
"specified. You need a CCL build and must fully specify the --enterprise-encryption flag")
}
fileRegistry = nil
}
var statsHandler EncryptionStatsHandler
if len(cfg.ExtraOptions) > 0 {
// Encryption is enabled.
if !cfg.UseFileRegistry {
return nil, nil, fmt.Errorf("file registry is needed to support encryption")
}
if NewEncryptedEnvFunc == nil {
return nil, nil, fmt.Errorf("encryption is enabled but no function to create the encrypted env")
}
var err error
cfg.Opts.FS, statsHandler, err =
NewEncryptedEnvFunc(cfg.Opts.FS, fileRegistry, cfg.Dir, cfg.Opts.ReadOnly, cfg.ExtraOptions)
if err != nil {
return nil, nil, err
}
}
return fileRegistry, statsHandler, nil
}
// NewPebble creates a new Pebble instance, at the specified path.
func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) {
// pebble.Open also calls EnsureDefaults, but only after doing a clone. Call
......@@ -454,34 +492,9 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) {
}
}
fileRegistry := &PebbleFileRegistry{FS: cfg.Opts.FS, DBDir: cfg.Dir, ReadOnly: cfg.Opts.ReadOnly}
if cfg.UseFileRegistry {
if err := fileRegistry.Load(); err != nil {
return nil, err
}
} else {
if err := fileRegistry.checkNoRegistryFile(); err != nil {
return nil, fmt.Errorf("encryption was used on this store before, but no encryption flags " +
"specified. You need a CCL build and must fully specify the --enterprise-encryption flag")
}
fileRegistry = nil
}
var statsHandler EncryptionStatsHandler
if len(cfg.ExtraOptions) > 0 {
// Encryption is enabled.
if !cfg.UseFileRegistry {
return nil, fmt.Errorf("file registry is needed to support encryption")
}
if NewEncryptedEnvFunc == nil {
return nil, fmt.Errorf("encryption is enabled but no function to create the encrypted env")
}
var err error
cfg.Opts.FS, statsHandler, err =
NewEncryptedEnvFunc(cfg.Opts.FS, fileRegistry, cfg.Dir, cfg.Opts.ReadOnly, cfg.ExtraOptions)
if err != nil {
return nil, err
}
fileRegistry, statsHandler, err := ResolveEncryptedEnvOptions(&cfg)
if err != nil {
return nil, err
}
// The context dance here is done so that we have a clean context without
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment