...
 
Commits (2)
  • Paul Bardea's avatar
    backupccl: load previous incremental manifests in parallel · c9b94255
    Paul Bardea authored
    Previously, incremental backups would fetch the manifests of the backups
    it is building off of serially. This would be a bottle-neck for small
    backups. This commit issues these requests in parallel, aiming to reduce
    the duration of incremental backups.
    
    Release note (performance improvement): reduce time needed to run a
    backup command when it is built on a lot of previous incremental
    backups.
    c9b94255
  • Paul Bardea's avatar
    Merge pull request #48772 from pbardea/backport20.1-47158 · dce38b15
    Paul Bardea authored
    release-20.1: backupccl: load previous incremental manifests in parallel
    dce38b15
......@@ -33,6 +33,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/stats"
"github.com/cockroachdb/cockroach/pkg/sql/types"
"github.com/cockroachdb/cockroach/pkg/storage/cloud"
"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/interval"
"github.com/cockroachdb/cockroach/pkg/util/log"
......@@ -394,6 +395,7 @@ func backupPlanHook(
var encryption *roachpb.FileEncryptionOptions
var prevBackups []BackupManifest
g := ctxgroup.WithContext(ctx)
if len(incrementalFrom) > 0 {
if encryptionPassphrase != nil {
exportStore, err := makeCloudStorage(ctx, incrementalFrom[0])
......@@ -410,20 +412,28 @@ func backupPlanHook(
}
}
prevBackups = make([]BackupManifest, len(incrementalFrom))
for i, uri := range incrementalFrom {
// TODO(lucy): We may want to upgrade the table descs to the newer
// foreign key representation here, in case there are backups from an
// older cluster. Keeping the descriptors as they are works for now
// since all we need to do is get the past backups' table/index spans,
// but it will be safer for future code to avoid having older-style
// descriptors around.
desc, err := ReadBackupManifestFromURI(
ctx, uri, makeCloudStorage, encryption,
)
if err != nil {
return errors.Wrapf(err, "failed to read backup from %q", uri)
}
prevBackups[i] = desc
for i := range incrementalFrom {
i := i
g.GoCtx(func(ctx context.Context) error {
// TODO(lucy): We may want to upgrade the table descs to the newer
// foreign key representation here, in case there are backups from an
// older cluster. Keeping the descriptors as they are works for now
// since all we need to do is get the past backups' table/index spans,
// but it will be safer for future code to avoid having older-style
// descriptors around.
uri := incrementalFrom[i]
desc, err := ReadBackupManifestFromURI(
ctx, uri, makeCloudStorage, encryption,
)
if err != nil {
return errors.Wrapf(err, "failed to read backup from %q", uri)
}
prevBackups[i] = desc
return nil
})
}
if err := g.Wait(); err != nil {
return err
}
} else {
exists, err := containsManifest(ctx, defaultStore)
......@@ -445,25 +455,33 @@ func backupPlanHook(
if err != nil {
return errors.Wrapf(err, "determining base for incremental backup")
}
prevBackups = make([]BackupManifest, 0, len(prev)+1)
prevBackups = make([]BackupManifest, len(prev)+1)
m, err := readBackupManifestFromStore(ctx, defaultStore, encryption)
if err != nil {
return errors.Wrap(err, "loading base backup manifest")
}
prevBackups = append(prevBackups, m)
prevBackups[0] = m
if m.DescriptorCoverage == tree.AllDescriptors &&
backupStmt.DescriptorCoverage != tree.AllDescriptors {
return errors.Errorf("cannot append a backup of specific tables or databases to a full-cluster backup")
}
for _, inc := range prev {
m, err := readBackupManifest(ctx, defaultStore, inc, encryption)
if err != nil {
return errors.Wrapf(err, "loading prior backup part manifest %q", inc)
}
prevBackups = append(prevBackups, m)
for i := range prev {
i := i
g.GoCtx(func(ctx context.Context) error {
inc := prev[i]
m, err := readBackupManifest(ctx, defaultStore, inc, encryption)
if err != nil {
return errors.Wrapf(err, "loading prior backup part manifest %q", inc)
}
prevBackups[i+1] = m
return nil
})
}
if err := g.Wait(); err != nil {
return err
}
// Pick a piece-specific suffix and update the destination path(s).
......