Backups: Add Support for Backup Init SQL Queries

go/cmd/vtbackup/cli/vtbackup.go

@@ -35,21 +35,24 @@ import (
 	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/mysql/replication"
+	"vitess.io/vitess/go/protoutil"
 	"vitess.io/vitess/go/stats"
 	"vitess.io/vitess/go/vt/dbconfigs"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/mysqlctl"
 	"vitess.io/vitess/go/vt/mysqlctl/backupstats"
 	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
-	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/topo"
 	"vitess.io/vitess/go/vt/topo/topoproto"
 	"vitess.io/vitess/go/vt/utils"
 	"vitess.io/vitess/go/vt/vterrors"
 	_ "vitess.io/vitess/go/vt/vttablet/grpctmclient"
 	"vitess.io/vitess/go/vt/vttablet/tmclient"
+
+	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 )
 
 const (
@@ -79,6 +82,10 @@ var (
 	allowFirstBackup    bool
 	restartBeforeBackup bool
 	upgradeSafe         bool
+	initSQLQueries      []string
+	initSQLTabletTypes  []topodatapb.TabletType
+	initSQLTimeout      time.Duration
+	initSQLFailOnError  bool
 
 	// vttablet-like flags
 	initDbNameOverride string
@@ -208,6 +215,10 @@ func init() {
 	utils.SetFlagBoolVar(Main.Flags(), &allowFirstBackup, "allow-first-backup", allowFirstBackup, "Allow this job to take the first backup of an existing shard.")
 	utils.SetFlagBoolVar(Main.Flags(), &restartBeforeBackup, "restart-before-backup", restartBeforeBackup, "Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs.")
 	Main.Flags().BoolVar(&upgradeSafe, "upgrade-safe", upgradeSafe, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
+	Main.Flags().StringSliceVar(&initSQLQueries, "init-backup-sql-queries", nil, "Queries to execute before initializing the backup")
+	Main.Flags().Var((*topoproto.TabletTypeListFlag)(&initSQLTabletTypes), "init-backup-tablet-types", "Tablet types used for the backup where the init SQL queries (--init-backup-sql-queries) will be executed before initializing the backup")
+	Main.Flags().DurationVar(&initSQLTimeout, "init-backup-sql-timeout", initSQLTimeout, "At what point should we time out the init SQL query (--init-backup-sql-queries) work and either fail the backup job (--init-backup-sql-fail-on-error) or continue on with the backup")
+	Main.Flags().BoolVar(&initSQLFailOnError, "init-backup-sql-fail-on-error", false, "Whether or not to fail the backup if the init SQL queries (--init-backup-sql-queries) fail, which includes if they fail to complete before the specified timeout (--init-backup-sql-timeout)")
 
 	// vttablet-like flags
 	utils.SetFlagStringVar(Main.Flags(), &initDbNameOverride, "init-db-name-override", initDbNameOverride, "(init parameter) override the name of the db used by vttablet")
@@ -394,9 +405,16 @@ func takeBackup(ctx, backgroundCtx context.Context, topoServer *topo.Server, bac
 		Keyspace:             initKeyspace,
 		Shard:                initShard,
 		TabletAlias:          topoproto.TabletAliasString(tabletAlias),
+		TabletType:           topodatapb.TabletType_REPLICA,
 		Stats:                backupstats.BackupStats(),
 		UpgradeSafe:          upgradeSafe,
 		MysqlShutdownTimeout: mysqlShutdownTimeout,
+		InitSQL: &tabletmanagerdatapb.BackupRequest_InitSQL{
+			Queries:     initSQLQueries,
+			TabletTypes: initSQLTabletTypes,
+			Timeout:     protoutil.DurationToProto(initSQLTimeout),
+			FailOnError: initSQLFailOnError,
+		},
 	}
 	// In initial_backup mode, just take a backup of this empty database.
 	if initialBackup {
@@ -489,6 +507,11 @@ func takeBackup(ctx, backgroundCtx context.Context, topoServer *topo.Server, bac
 		}
 	}
 
+	// Perform any requested pre backup initialization queries.
+	if err := mysqlctl.ExecuteBackupInitSQL(ctx, &backupParams); err != nil {
+		return vterrors.Wrap(err, "failed to execute backup init SQL queries")
+	}
+
 	// We have restored a backup. Now start replication.
 	if err := resetReplication(ctx, restorePos, mysqld); err != nil {
 		return fmt.Errorf("error resetting replication: %v", err)

go/cmd/vtctldclient/command/backups.go

@@ -30,6 +30,8 @@ import (
 	"vitess.io/vitess/go/vt/mysqlctl"
 	"vitess.io/vitess/go/vt/topo/topoproto"
 
+	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
 )
 
@@ -86,14 +88,33 @@ var backupOptions = struct {
 	IncrementalFromPos   string
 	UpgradeSafe          bool
 	MysqlShutdownTimeout time.Duration
+	InitSQLQueries       []string
+	InitSQLTabletTypes   []topodatapb.TabletType
+	InitSQLTimeout       time.Duration
+	InitSQLFailOnError   bool
 }{}
 
+func validateBackupOptions() error {
+	if len(backupOptions.InitSQLQueries) > 0 {
+		if len(backupOptions.InitSQLTabletTypes) == 0 {
+			return errors.New("backup init SQL queries provided but no tablet types on which to run them")
+		}
+		if backupOptions.InitSQLTimeout == 0 {
+			return errors.New("backup init SQL queries provided but no timeout provided -- this is dangerous and not allowed")
+		}
+	}
+	return nil
+}
+
 func commandBackup(cmd *cobra.Command, args []string) error {
 	tabletAlias, err := topoproto.ParseTabletAlias(cmd.Flags().Arg(0))
 	if err != nil {
 		return err
 	}
 
+	if err := validateBackupOptions(); err != nil {
+		return err
+	}
 	cli.FinishedParsing(cmd)
 
 	req := &vtctldatapb.BackupRequest{
@@ -103,6 +124,12 @@ func commandBackup(cmd *cobra.Command, args []string) error {
 		IncrementalFromPos:   backupOptions.IncrementalFromPos,
 		UpgradeSafe:          backupOptions.UpgradeSafe,
 		MysqlShutdownTimeout: protoutil.DurationToProto(backupOptions.MysqlShutdownTimeout),
+		InitSql: &tabletmanagerdatapb.BackupRequest_InitSQL{
+			Queries:     backupOptions.InitSQLQueries,
+			TabletTypes: backupOptions.InitSQLTabletTypes,
+			Timeout:     protoutil.DurationToProto(backupOptions.InitSQLTimeout),
+			FailOnError: backupOptions.InitSQLFailOnError,
+		},
 	}
 
 	if backupOptions.BackupEngine != "" {
@@ -141,6 +168,9 @@ func commandBackupShard(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if err := validateBackupOptions(); err != nil {
+		return err
+	}
 	cli.FinishedParsing(cmd)
 
 	stream, err := client.BackupShard(commandCtx, &vtctldatapb.BackupShardRequest{
@@ -151,6 +181,12 @@ func commandBackupShard(cmd *cobra.Command, args []string) error {
 		IncrementalFromPos:   backupShardOptions.IncrementalFromPos,
 		UpgradeSafe:          backupShardOptions.UpgradeSafe,
 		MysqlShutdownTimeout: protoutil.DurationToProto(backupShardOptions.MysqlShutdownTimeout),
+		InitSql: &tabletmanagerdatapb.BackupRequest_InitSQL{
+			Queries:     backupOptions.InitSQLQueries,
+			TabletTypes: backupOptions.InitSQLTabletTypes,
+			Timeout:     protoutil.DurationToProto(backupOptions.InitSQLTimeout),
+			FailOnError: backupOptions.InitSQLFailOnError,
+		},
 	})
 	if err != nil {
 		return err
@@ -300,13 +336,15 @@ func init() {
 
 	Backup.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
 	Backup.Flags().DurationVar(&backupOptions.MysqlShutdownTimeout, "mysql-shutdown-timeout", mysqlctl.DefaultShutdownTimeout, "Timeout to use when MySQL is being shut down.")
+	addInitSQLFlags(Backup)
 	Root.AddCommand(Backup)
 
 	BackupShard.Flags().BoolVar(&backupShardOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.")
 	BackupShard.Flags().Int32Var(&backupShardOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.")
 	BackupShard.Flags().StringVar(&backupShardOptions.IncrementalFromPos, "incremental-from-pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
 	BackupShard.Flags().BoolVar(&backupShardOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
 	BackupShard.Flags().DurationVar(&backupShardOptions.MysqlShutdownTimeout, "mysql-shutdown-timeout", mysqlctl.DefaultShutdownTimeout, "Timeout to use when MySQL is being shut down.")
+	addInitSQLFlags(BackupShard)
 	Root.AddCommand(BackupShard)
 
 	GetBackups.Flags().Uint32VarP(&getBackupsOptions.Limit, "limit", "l", 0, "Retrieve only the most recent N backups.")
@@ -322,3 +360,10 @@ func init() {
 	RestoreFromBackup.Flags().BoolVar(&restoreFromBackupOptions.DryRun, "dry-run", false, "Only validate restore steps, do not actually restore data")
 	Root.AddCommand(RestoreFromBackup)
 }
+
+func addInitSQLFlags(cmd *cobra.Command) {
+	cmd.Flags().StringSliceVar(&backupOptions.InitSQLQueries, "init-backup-sql-queries", nil, "Queries to execute before initializing the backup")
+	cmd.Flags().Var((*topoproto.TabletTypeListFlag)(&backupOptions.InitSQLTabletTypes), "init-backup-tablet-types", "Tablet types used for the backup where the init SQL queries (--init-backup-sql-queries) will be executed before initializing the backup")
+	cmd.Flags().DurationVar(&backupOptions.InitSQLTimeout, "init-backup-sql-timeout", backupOptions.InitSQLTimeout, "At what point should we time out the init SQL query (--init-backup-sql-queries) work and either fail the backup job (--init-backup-sql-fail-on-error) or continue on with the backup")
+	cmd.Flags().BoolVar(&backupOptions.InitSQLFailOnError, "init-backup-sql-fail-on-error", false, "Whether or not to fail the backup if the init SQL queries (--init-backup-sql-queries) fail, which includes if they fail to complete before the specified timeout (--init-backup-sql-timeout)")
+}