From ce355b2eef9435dcaf32434d676ff42e69cb295c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 5 Jun 2024 15:42:28 +1000 Subject: [PATCH 01/27] Fix gprestore hanging on copy in case the helper goes down at start --- helper/helper.go | 30 ++++++++++++++++++++++++++++++ restore/data.go | 5 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/helper/helper.go b/helper/helper.go index 3d0dd14b3..0e4138049 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -256,10 +256,40 @@ func DoCleanup() { for pipeName, _ := range pipesMap { log("Removing pipe %s", pipeName) + + /* + * In case restore helper encountered an error, it is possible that the pipe file has been already opened for + * reading by another process, but nothing has been written to the pipe yet. In this case, the reader process + * may hang forever after the pipe is removed. To fix this problem, the pipe file is opened and closed + * to generate an EOF before it is deleted. + * + * Also, it is possible, that after EOF is issued (releasing the current reader process), but before the pipe is + * removed, a new reader process can start reading the pipe. To avoid such situation, we create a special file + * for the pipe (with '_removal_flag' suffix), and delete it only after all pipe shenanigans are done. + * Reader side should check that this file doesn't exist before accessing the pipe file. + */ + var pipeRemovalFlagFilename string + if *restoreAgent { + pipeRemovalFlagFilename = fmt.Sprintf("%s_removal_flag", pipeName) + fileHandlePipeRemovalFlag, err := utils.OpenFileForWrite(pipeRemovalFlagFilename) + if err == nil { + fileHandlePipeRemovalFlag.Close() + } + + // Open and close the pipe file to generate an EOF. + fileHandlePipe, err := os.OpenFile(pipeName, os.O_WRONLY|unix.O_NONBLOCK, os.ModeNamedPipe) + if err == nil { + fileHandlePipe.Close() + } + } + err = deletePipe(pipeName) if err != nil { log("Encountered error removing pipe %s: %v", pipeName, err) } + if *restoreAgent { + utils.RemoveFileIfExists(pipeRemovalFlagFilename) + } } skipFiles, _ := filepath.Glob(fmt.Sprintf("%s_skip_*", *pipeFile)) diff --git a/restore/data.go b/restore/data.go index 0926dade8..007547ea1 100644 --- a/restore/data.go +++ b/restore/data.go @@ -37,7 +37,10 @@ func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, ta if singleDataFile || resizeCluster { //helper.go handles compression, so we don't want to set it here customPipeThroughCommand = "cat -" - checkPipeExistsCommand = fmt.Sprintf("(timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) && ", destinationToRead, destinationToRead) + // Before accessing the pipe file, we need to check that the "_removal_flag" file is not present. + // For more details, refer to comments in helper's DoCleanup() function. + pipeRemovalFlagFilename := fmt.Sprintf("%s_removal_flag", destinationToRead) + checkPipeExistsCommand = fmt.Sprintf("(( test ! -e \"%s\" && (timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) ) || (echo \"Pipe is in progress of removal %s\">&2; exit 1) )&& ", pipeRemovalFlagFilename, destinationToRead, destinationToRead, destinationToRead) } else if MustGetFlagString(options.PLUGIN_CONFIG) != "" { readFromDestinationCommand = fmt.Sprintf("%s restore_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) } From d8a2d22a8b50728c460cd3a89bd4bae654cbeb3b Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 5 Jun 2024 17:36:18 +1000 Subject: [PATCH 02/27] Fix spaces in shell cmd for COPY query --- restore/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/restore/data.go b/restore/data.go index 007547ea1..441f26b48 100644 --- a/restore/data.go +++ b/restore/data.go @@ -40,7 +40,7 @@ func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, ta // Before accessing the pipe file, we need to check that the "_removal_flag" file is not present. // For more details, refer to comments in helper's DoCleanup() function. pipeRemovalFlagFilename := fmt.Sprintf("%s_removal_flag", destinationToRead) - checkPipeExistsCommand = fmt.Sprintf("(( test ! -e \"%s\" && (timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) ) || (echo \"Pipe is in progress of removal %s\">&2; exit 1) )&& ", pipeRemovalFlagFilename, destinationToRead, destinationToRead, destinationToRead) + checkPipeExistsCommand = fmt.Sprintf("((test ! -e \"%s\" && (timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1))) || (echo \"Pipe is in progress of removal %s\">&2; exit 1)) && ", pipeRemovalFlagFilename, destinationToRead, destinationToRead, destinationToRead) } else if MustGetFlagString(options.PLUGIN_CONFIG) != "" { readFromDestinationCommand = fmt.Sprintf("%s restore_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) } From 6384cd0847b2fcd9036cf6079ded179ad9735188 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 5 Jun 2024 17:44:57 +1000 Subject: [PATCH 03/27] Fix format issues --- helper/helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helper/helper.go b/helper/helper.go index 0e4138049..220cc6b74 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -264,8 +264,8 @@ func DoCleanup() { * to generate an EOF before it is deleted. * * Also, it is possible, that after EOF is issued (releasing the current reader process), but before the pipe is - * removed, a new reader process can start reading the pipe. To avoid such situation, we create a special file - * for the pipe (with '_removal_flag' suffix), and delete it only after all pipe shenanigans are done. + * removed, a new reader process can start reading the pipe. To avoid such situation, we create a special file + * for the pipe (with '_removal_flag' suffix), and delete it only after all pipe shenanigans are done. * Reader side should check that this file doesn't exist before accessing the pipe file. */ var pipeRemovalFlagFilename string From acbc6c87272561be9645bbcab85d840a5c1bfd3c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 5 Jun 2024 18:33:06 +1000 Subject: [PATCH 04/27] Fix unit test --- restore/data_test.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/restore/data_test.go b/restore/data_test.go index 394c3891d..f646677c2 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -56,7 +56,7 @@ var _ = Describe("restore/data tests", func() { Expect(err).ShouldNot(HaveOccurred()) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) @@ -130,7 +130,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" @@ -141,7 +141,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" @@ -151,7 +151,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" @@ -161,7 +161,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" @@ -175,7 +175,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -190,7 +190,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -204,7 +204,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -215,7 +215,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -244,7 +244,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -254,7 +254,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -263,7 +263,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -272,7 +272,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) @@ -285,7 +285,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -299,7 +299,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" @@ -312,7 +312,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -322,7 +322,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", From e1d993be41f715a7254c10953f2af0060f1ba0ed Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 7 Jun 2024 13:58:42 +1000 Subject: [PATCH 05/27] Fix end-2-end test --- end_to_end/end_to_end_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 556371c1e..a9335abe8 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -2302,7 +2302,7 @@ LANGUAGE plpgsql NO SQL;`) output, err := gprestoreCmd.CombinedOutput() Expect(err).To(HaveOccurred()) Expect(string(output)).To(ContainSubstring(`Error loading data into table public.t1`)) - Expect(string(output)).To(ContainSubstring(`Error loading data into table public.t2`)) + Expect(string(output)).To(SatisfyAny(ContainSubstring(`Error loading data into table public.t2`), ContainSubstring(`Expected to restore 1000000 rows to table public.t2, but restored 0 instead`))) assertArtifactsCleaned(restoreConn, "20240502095933") testhelper.AssertQueryRuns(restoreConn, "DROP TABLE t0; DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE t4;") }) From db9a617f80dc6e2b04dad355e1cb0d391a8e4cc9 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 7 Jun 2024 14:10:38 +1000 Subject: [PATCH 06/27] Fix style --- helper/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helper/helper.go b/helper/helper.go index 220cc6b74..4c6be117f 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -282,11 +282,12 @@ func DoCleanup() { fileHandlePipe.Close() } } - + err = deletePipe(pipeName) if err != nil { log("Encountered error removing pipe %s: %v", pipeName, err) } + if *restoreAgent { utils.RemoveFileIfExists(pipeRemovalFlagFilename) } From f992b463b3a80557802c70ea755c603ffd0c7e63 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 7 Jun 2024 15:51:03 +1000 Subject: [PATCH 07/27] Fix case for single data file --- helper/restore_helper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helper/restore_helper.go b/helper/restore_helper.go index f2e7d6640..02f462b93 100644 --- a/helper/restore_helper.go +++ b/helper/restore_helper.go @@ -126,6 +126,8 @@ func doRestoreAgent() error { } } + preloadCreatedPipes(oidList, *copyQueue) + if *singleDataFile { contentToRestore := *content segmentTOC = make(map[int]*toc.SegmentTOC) @@ -173,8 +175,6 @@ func doRestoreAgent() error { } } - preloadCreatedPipes(oidList, *copyQueue) - var currentPipe string for i, oid := range oidList { if wasTerminated { @@ -493,7 +493,7 @@ func getSubsetFlag(fileToRead string, pluginConfig *utils.PluginConfig) bool { return false } // Helper's option does not allow to use subset - if !*isFiltered || *onErrorContinue { + if !*isFiltered || *onErrorContinue { return false } // Restore subset and compression does not allow together From 05eb55fb3efbeb85b743b86a94a699a923c1ef1e Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sun, 9 Jun 2024 17:18:08 +1000 Subject: [PATCH 08/27] Add test --- end_to_end/plugin_test.go | 38 +++++++++++++++++++++++++++++++++++++ plugins/example_plugin.bash | 4 ++++ 2 files changed, 42 insertions(+) diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index eedc8308a..5a434d55e 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" path "path/filepath" + "time" "github.com/greenplum-db/gp-common-go-libs/cluster" "github.com/greenplum-db/gp-common-go-libs/dbconn" @@ -14,6 +15,7 @@ import ( "github.com/greenplum-db/gpbackup/testutils" "github.com/greenplum-db/gpbackup/utils" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) func copyPluginToAllHosts(conn *dbconn.DBConn, pluginPath string) { @@ -306,6 +308,42 @@ var _ = Describe("End to End plugin tests", func() { Skip("This test is only needed for the most recent backup versions") } }) + It("Will not hang if gpbackup and gprestore runs with single-data-file and plugin fails at init for restore", func(ctx SpecContext) { + pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) + copyPluginToAllHosts(backupConn, pluginExecutablePath) + + testhelper.AssertQueryRuns(backupConn, "CREATE TABLE t0(a int);") + testhelper.AssertQueryRuns(backupConn, "INSERT INTO t0 SELECT i FROM generate_series(1, 10)i;") + defer testhelper.AssertQueryRuns(restoreConn, "DROP TABLE t0;") + + timestamp := gpbackup(gpbackupPath, backupHelperPath, + "--single-data-file", + "--plugin-config", pluginConfigPath) + + backupCluster.GenerateAndExecuteCommand( + "Instruct plugin to fail", + cluster.ON_HOSTS, + func(contentID int) string { + return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_FAIL") + }) + + gprestoreCmd := exec.Command(gprestorePath, + "--timestamp", timestamp, + "--redirect-db", "restoredb", + "--plugin-config", pluginConfigPath) + + _, err := gprestoreCmd.CombinedOutput() + Expect(err).To(HaveOccurred()) + + backupCluster.GenerateAndExecuteCommand( + "Unset plugin failure", + cluster.ON_HOSTS, + func(contentID int) string { + return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_FAIL") + }) + + assertArtifactsCleaned(restoreConn, timestamp) + }, SpecTimeout(time.Second*30)) It("runs gpbackup and gprestore with plugin, single-data-file, and no-compression", func() { pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) copyPluginToAllHosts(backupConn, pluginExecutablePath) diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index fc594e28f..2600f020c 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -76,6 +76,10 @@ restore_data() { echo "restore_data $1 $2" >> /tmp/plugin_out.txt filename=`basename "$2"` timestamp_dir=`basename $(dirname "$2")` + if [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then + sleep 3 + echo 'error' >&2 + fi timestamp_day_dir=${timestamp_dir%??????} cat /tmp/plugin_dest/$timestamp_day_dir/$timestamp_dir/$filename } From 066a01b3b75504dbc03c4391d839313c97e7fa23 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sun, 9 Jun 2024 20:36:45 +1000 Subject: [PATCH 09/27] Rework solution --- helper/helper.go | 22 +++++++--------------- restore/data.go | 5 +---- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/helper/helper.go b/helper/helper.go index 4c6be117f..a8f52af3d 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -264,21 +264,17 @@ func DoCleanup() { * to generate an EOF before it is deleted. * * Also, it is possible, that after EOF is issued (releasing the current reader process), but before the pipe is - * removed, a new reader process can start reading the pipe. To avoid such situation, we create a special file - * for the pipe (with '_removal_flag' suffix), and delete it only after all pipe shenanigans are done. - * Reader side should check that this file doesn't exist before accessing the pipe file. + * removed, a new reader process can start reading the pipe. To avoid such situation, we rename the file before + * closing it. */ - var pipeRemovalFlagFilename string if *restoreAgent { - pipeRemovalFlagFilename = fmt.Sprintf("%s_removal_flag", pipeName) - fileHandlePipeRemovalFlag, err := utils.OpenFileForWrite(pipeRemovalFlagFilename) - if err == nil { - fileHandlePipeRemovalFlag.Close() - } - - // Open and close the pipe file to generate an EOF. fileHandlePipe, err := os.OpenFile(pipeName, os.O_WRONLY|unix.O_NONBLOCK, os.ModeNamedPipe) if err == nil { + newPipeName := fmt.Sprintf("%s_rm", pipeName) + err = os.Rename(pipeName, newPipeName) + if err == nil { + pipeName = newPipeName + } fileHandlePipe.Close() } } @@ -287,10 +283,6 @@ func DoCleanup() { if err != nil { log("Encountered error removing pipe %s: %v", pipeName, err) } - - if *restoreAgent { - utils.RemoveFileIfExists(pipeRemovalFlagFilename) - } } skipFiles, _ := filepath.Glob(fmt.Sprintf("%s_skip_*", *pipeFile)) diff --git a/restore/data.go b/restore/data.go index 441f26b48..0926dade8 100644 --- a/restore/data.go +++ b/restore/data.go @@ -37,10 +37,7 @@ func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, ta if singleDataFile || resizeCluster { //helper.go handles compression, so we don't want to set it here customPipeThroughCommand = "cat -" - // Before accessing the pipe file, we need to check that the "_removal_flag" file is not present. - // For more details, refer to comments in helper's DoCleanup() function. - pipeRemovalFlagFilename := fmt.Sprintf("%s_removal_flag", destinationToRead) - checkPipeExistsCommand = fmt.Sprintf("((test ! -e \"%s\" && (timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1))) || (echo \"Pipe is in progress of removal %s\">&2; exit 1)) && ", pipeRemovalFlagFilename, destinationToRead, destinationToRead, destinationToRead) + checkPipeExistsCommand = fmt.Sprintf("(timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) && ", destinationToRead, destinationToRead) } else if MustGetFlagString(options.PLUGIN_CONFIG) != "" { readFromDestinationCommand = fmt.Sprintf("%s restore_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) } From 32132038eeaf756a5ba3ceec37bcd1f593fb0cbe Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sun, 9 Jun 2024 20:37:16 +1000 Subject: [PATCH 10/27] Revert "Fix unit test" This reverts commit acbc6c87272561be9645bbcab85d840a5c1bfd3c. --- restore/data_test.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/restore/data_test.go b/restore/data_test.go index f646677c2..394c3891d 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -56,7 +56,7 @@ var _ = Describe("restore/data tests", func() { Expect(err).ShouldNot(HaveOccurred()) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) @@ -130,7 +130,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" @@ -141,7 +141,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" @@ -151,7 +151,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" @@ -161,7 +161,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" @@ -175,7 +175,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -190,7 +190,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -204,7 +204,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -215,7 +215,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -244,7 +244,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -254,7 +254,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -263,7 +263,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -272,7 +272,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) @@ -285,7 +285,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -299,7 +299,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" @@ -312,7 +312,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -322,7 +322,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '((test ! -e \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456_removal_flag\" && (timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1))) || (echo \"Pipe is in progress of removal /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", From 3598ed802f108e4074ad973f97aadb510294c126 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Sun, 9 Jun 2024 22:31:38 +1000 Subject: [PATCH 11/27] Remove unrelated style change done by goimports --- helper/restore_helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/restore_helper.go b/helper/restore_helper.go index 02f462b93..0eaca8b64 100644 --- a/helper/restore_helper.go +++ b/helper/restore_helper.go @@ -493,7 +493,7 @@ func getSubsetFlag(fileToRead string, pluginConfig *utils.PluginConfig) bool { return false } // Helper's option does not allow to use subset - if !*isFiltered || *onErrorContinue { + if !*isFiltered || *onErrorContinue { return false } // Restore subset and compression does not allow together From 99d717da8d7722b8bbe9a7a54113b35a777f83f5 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 10 Jun 2024 09:44:45 +1000 Subject: [PATCH 12/27] Fix test --- end_to_end/plugin_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index 5a434d55e..bc1059daa 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -314,7 +314,7 @@ var _ = Describe("End to End plugin tests", func() { testhelper.AssertQueryRuns(backupConn, "CREATE TABLE t0(a int);") testhelper.AssertQueryRuns(backupConn, "INSERT INTO t0 SELECT i FROM generate_series(1, 10)i;") - defer testhelper.AssertQueryRuns(restoreConn, "DROP TABLE t0;") + defer testhelper.AssertQueryRuns(backupConn, "DROP TABLE t0;") timestamp := gpbackup(gpbackupPath, backupHelperPath, "--single-data-file", From d305ae96d79362e189b257640ca9838b972dc657 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 10 Jun 2024 10:50:17 +1000 Subject: [PATCH 13/27] Improve and simplify solution --- helper/helper.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/helper/helper.go b/helper/helper.go index a8f52af3d..50ff6559f 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -264,18 +264,13 @@ func DoCleanup() { * to generate an EOF before it is deleted. * * Also, it is possible, that after EOF is issued (releasing the current reader process), but before the pipe is - * removed, a new reader process can start reading the pipe. To avoid such situation, we rename the file before - * closing it. + * removed, a new reader process can start reading the pipe. To avoid such situation, we close the file handle + * after we have removed the file. */ if *restoreAgent { fileHandlePipe, err := os.OpenFile(pipeName, os.O_WRONLY|unix.O_NONBLOCK, os.ModeNamedPipe) if err == nil { - newPipeName := fmt.Sprintf("%s_rm", pipeName) - err = os.Rename(pipeName, newPipeName) - if err == nil { - pipeName = newPipeName - } - fileHandlePipe.Close() + defer fileHandlePipe.Close() } } From 473216c2683a580a7dfe2bc752c73a205db1b213 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 11 Jun 2024 17:20:50 +1000 Subject: [PATCH 14/27] Fix the backup helper --- helper/helper.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/helper/helper.go b/helper/helper.go index 50ff6559f..994751beb 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -266,12 +266,21 @@ func DoCleanup() { * Also, it is possible, that after EOF is issued (releasing the current reader process), but before the pipe is * removed, a new reader process can start reading the pipe. To avoid such situation, we close the file handle * after we have removed the file. + * + * Similar problem can happen with backup helper. If it finishes its work right after start, before opening the + * pipe for reading, the writing side will stay hanging on the other pipe's end. So, we also open and close the + * pipe to release the writing side. */ if *restoreAgent { fileHandlePipe, err := os.OpenFile(pipeName, os.O_WRONLY|unix.O_NONBLOCK, os.ModeNamedPipe) if err == nil { defer fileHandlePipe.Close() } + } else if *backupAgent { + fileHandlePipe, err := os.OpenFile(pipeName, os.O_RDONLY|unix.O_NONBLOCK, os.ModeNamedPipe) + if err == nil { + defer fileHandlePipe.Close() + } } err = deletePipe(pipeName) From 1c4337fd61ed881e1ded8c7538c516cf3e0b95e3 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 18 Jun 2024 12:05:55 +1000 Subject: [PATCH 15/27] Update solution to avoid partial restore --- end_to_end/end_to_end_suite_test.go | 2 +- restore/data.go | 8 +++++- restore/data_test.go | 39 ++++++++++++++++------------- restore/global_variables.go | 4 +++ 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index a9335abe8..556371c1e 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -2302,7 +2302,7 @@ LANGUAGE plpgsql NO SQL;`) output, err := gprestoreCmd.CombinedOutput() Expect(err).To(HaveOccurred()) Expect(string(output)).To(ContainSubstring(`Error loading data into table public.t1`)) - Expect(string(output)).To(SatisfyAny(ContainSubstring(`Error loading data into table public.t2`), ContainSubstring(`Expected to restore 1000000 rows to table public.t2, but restored 0 instead`))) + Expect(string(output)).To(ContainSubstring(`Error loading data into table public.t2`)) assertArtifactsCleaned(restoreConn, "20240502095933") testhelper.AssertQueryRuns(restoreConn, "DROP TABLE t0; DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE t4;") }) diff --git a/restore/data.go b/restore/data.go index 0926dade8..f5345d5b5 100644 --- a/restore/data.go +++ b/restore/data.go @@ -33,16 +33,22 @@ func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, ta customPipeThroughCommand := utils.GetPipeThroughProgram().InputCommand origSize, destSize, resizeCluster := GetResizeClusterInfo() checkPipeExistsCommand := "" + checkHelperErrorFileCommand := "" if singleDataFile || resizeCluster { //helper.go handles compression, so we don't want to set it here customPipeThroughCommand = "cat -" checkPipeExistsCommand = fmt.Sprintf("(timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) && ", destinationToRead, destinationToRead) + // If, by the end of reading from the pipe, the restore helper error file exists, it means restore helper faced + // an error during table restore, and we can't rely on COPY results (even if GPDB reported Ok status), + // as restore helper could push into the pipe less data than expected (or even no data at all). + helperErrorFileName := fmt.Sprintf("%s_error", globalFPInfo.GetSegmentPipePathForCopyCommand()) + checkHelperErrorFileCommand = fmt.Sprintf(" && test ! -e \"%s\"", helperErrorFileName) } else if MustGetFlagString(options.PLUGIN_CONFIG) != "" { readFromDestinationCommand = fmt.Sprintf("%s restore_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) } - copyCommand = fmt.Sprintf("PROGRAM '%s%s %s | %s'", checkPipeExistsCommand, readFromDestinationCommand, destinationToRead, customPipeThroughCommand) + copyCommand = fmt.Sprintf("PROGRAM '%s%s %s | %s%s'", checkPipeExistsCommand, readFromDestinationCommand, destinationToRead, customPipeThroughCommand, checkHelperErrorFileCommand) query := fmt.Sprintf("COPY %s%s FROM %s WITH CSV DELIMITER '%s' ON SEGMENT;", tableName, tableAttributes, copyCommand, tableDelim) diff --git a/restore/data_test.go b/restore/data_test.go index 394c3891d..3bc598d5b 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -10,6 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/greenplum-db/gp-common-go-libs/cluster" "github.com/greenplum-db/gpbackup/backup" + "github.com/greenplum-db/gpbackup/filepath" "github.com/greenplum-db/gpbackup/history" "github.com/greenplum-db/gpbackup/options" "github.com/greenplum-db/gpbackup/restore" @@ -21,6 +22,10 @@ import ( ) var _ = Describe("restore/data tests", func() { + var fpInfo filepath.FilePathInfo + BeforeEach(func() { + fpInfo = restore.GetFPInfo() + }) Describe("CopyTableIn", func() { BeforeEach(func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "cat", OutputCommand: "cat -", InputCommand: "cat -", Extension: ""}) @@ -56,7 +61,7 @@ var _ = Describe("restore/data tests", func() { Expect(err).ShouldNot(HaveOccurred()) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) @@ -130,7 +135,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" @@ -141,7 +146,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" @@ -151,7 +156,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" @@ -161,7 +166,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" @@ -175,7 +180,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -190,7 +195,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -204,7 +209,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -215,7 +220,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -244,7 +249,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -254,7 +259,7 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -263,7 +268,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -272,7 +277,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) @@ -285,7 +290,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -299,7 +304,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" @@ -312,7 +317,7 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -322,7 +327,7 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", diff --git a/restore/global_variables.go b/restore/global_variables.go index 92196867d..0e2de5b80 100644 --- a/restore/global_variables.go +++ b/restore/global_variables.go @@ -84,6 +84,10 @@ func SetFPInfo(fpInfo filepath.FilePathInfo) { globalFPInfo = fpInfo } +func GetFPInfo() filepath.FilePathInfo { + return globalFPInfo +} + func SetPluginConfig(config *utils.PluginConfig) { pluginConfig = config } From 21f198476ee2da2ac12df326f8f483eaf2515728 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 20 Jun 2024 10:17:54 +1000 Subject: [PATCH 16/27] Fix code style in unit test --- restore/data_test.go | 102 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 17 deletions(-) diff --git a/restore/data_test.go b/restore/data_test.go index 3bc598d5b..657eba14c 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -61,7 +61,11 @@ var _ = Describe("restore/data tests", func() { Expect(err).ShouldNot(HaveOccurred()) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) @@ -135,7 +139,11 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" @@ -146,7 +154,11 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" @@ -156,7 +168,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" @@ -166,7 +182,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" @@ -180,7 +200,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -195,7 +219,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -209,7 +237,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) @@ -220,7 +252,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -249,7 +285,11 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -259,7 +299,11 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -268,7 +312,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) @@ -277,7 +325,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) @@ -290,7 +342,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -304,7 +360,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" @@ -317,7 +377,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" @@ -327,7 +391,11 @@ var _ = Describe("restore/data tests", func() { Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM '(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || (echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ + "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ + "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ + "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ + "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", From c0fd30af9c45be8b9abcae94f8ace02c15fd4757 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 20 Jun 2024 14:33:32 +1000 Subject: [PATCH 17/27] Update changes in unit test --- restore/data_test.go | 139 ++++++++++++------------------------------- 1 file changed, 37 insertions(+), 102 deletions(-) diff --git a/restore/data_test.go b/restore/data_test.go index 657eba14c..5cfb1b880 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -23,6 +23,9 @@ import ( var _ = Describe("restore/data tests", func() { var fpInfo filepath.FilePathInfo + copyFromPipeQueryTemplate := "COPY public.foo(i,j) FROM PROGRAM " + + "'(timeout 300 bash -c \"while [ ! -p \"%s\" ]; do sleep 1; done\" || (echo \"Pipe not found %s\">&2; exit 1)) && " + + "cat %s | cat - && test ! -e \"%s_error\"' WITH CSV DELIMITER ',' ON SEGMENT;" BeforeEach(func() { fpInfo = restore.GetFPInfo() }) @@ -61,13 +64,9 @@ var _ = Describe("restore/data tests", func() { Expect(err).ShouldNot(HaveOccurred()) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) - mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) Expect(err).ShouldNot(HaveOccurred()) @@ -139,14 +138,10 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) @@ -154,42 +149,30 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) Expect(err).ShouldNot(HaveOccurred()) @@ -200,15 +183,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) @@ -219,15 +198,11 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) @@ -237,26 +212,19 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(20))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -264,7 +232,6 @@ var _ = Describe("restore/data tests", func() { Where: "COPY foo, line 1: \"5\"", } mock.ExpectExec(execStr).WillReturnError(pgErr) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).To(HaveOccurred()) @@ -285,13 +252,9 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with gzip compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) - mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) @@ -299,39 +262,27 @@ var _ = Describe("restore/data tests", func() { }) It("will restore a table from its own file with zstd compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) - mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from its own file without compression", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) - mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will restore a table from a single data file", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) - mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) Expect(err).ShouldNot(HaveOccurred()) @@ -342,14 +293,10 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) @@ -360,14 +307,10 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) @@ -377,25 +320,18 @@ var _ = Describe("restore/data tests", func() { _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} restore.SetPluginConfig(&pluginConfig) - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).ShouldNot(HaveOccurred()) Expect(numRowsRestored).Should(Equal(int64(10))) }) It("will output expected error string from COPY ON SEGMENT failure", func() { - execStr := regexp.QuoteMeta(fmt.Sprintf("COPY public.foo(i,j) FROM PROGRAM "+ - "'(timeout 300 bash -c \"while [ ! -p \"/backups/20170101/20170101010101/gpbackup__20170101010101_3456\" ]; do sleep 1; done\" || "+ - "(echo \"Pipe not found /backups/20170101/20170101010101/gpbackup__20170101010101_3456\">&2; exit 1)) && "+ - "cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat - && test ! -e \"%s_error\"' "+ - "WITH CSV DELIMITER ',' ON SEGMENT;", fpInfo.GetSegmentPipePathForCopyCommand())) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + execStr := regexp.QuoteMeta(fmt.Sprintf(copyFromPipeQueryTemplate, filename, filename, filename, fpInfo.GetSegmentPipePathForCopyCommand())) pgErr := &pgconn.PgError{ Severity: "ERROR", Code: "22P04", @@ -403,7 +339,6 @@ var _ = Describe("restore/data tests", func() { Where: "COPY foo, line 1: \"5\"", } mock.ExpectExec(execStr).WillReturnError(pgErr) - filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) Expect(err).To(HaveOccurred()) From f6c13e600249057b78b62b4a9e101f8a268c4c86 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 24 Jun 2024 13:53:33 +1000 Subject: [PATCH 18/27] Fix hanging in case the restore helper fails to get oid list from a file --- restore/data.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/restore/data.go b/restore/data.go index f5345d5b5..9cc4b6e92 100644 --- a/restore/data.go +++ b/restore/data.go @@ -9,6 +9,7 @@ import ( "fmt" "sync" "sync/atomic" + "time" "github.com/greenplum-db/gp-common-go-libs/cluster" "github.com/greenplum-db/gp-common-go-libs/dbconn" @@ -226,6 +227,27 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Make sure it's called to release resources even if no errors + // Launch a checker that polls if the restore helper has ended with an error. + // It is our 'Ultima ratio regum' - in case restore helper couldn't read a file with the oid list, it is not aware + // about the pipes, pre-created by the gprestore, and it can't close them. So we cancel all pending COPY commands + // from here after giving a chance to the restore helper to close pipes on its own. + if backupConfig.SingleDataFile || resizeCluster { + go func() { + for { + time.Sleep(5 * time.Second) + remoteOutput := globalCluster.GenerateAndExecuteCommand("Checking gpbackup_helper agent failure", cluster.ON_SEGMENTS, func(contentID int) string { + helperErrorFileName := fmt.Sprintf("%s_error", fpInfo.GetSegmentPipeFilePath(contentID)) + return fmt.Sprintf("! ls %s", helperErrorFileName) + }) + if remoteOutput.NumErrors != 0 { + // the delay below is to give the restore helper a chance to close all pipes, if it can... + time.Sleep(5 * time.Second) + cancel() + } + } + }() + } + for i := 0; i < connectionPool.NumConns; i++ { workerPool.Add(1) go func(whichConn int) { From bac9ce9f591216ccff61913c950a72af7eeacf7e Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Thu, 27 Jun 2024 10:45:28 +1000 Subject: [PATCH 19/27] Add more logging --- restore/data.go | 1 + 1 file changed, 1 insertion(+) diff --git a/restore/data.go b/restore/data.go index 9cc4b6e92..18bedf793 100644 --- a/restore/data.go +++ b/restore/data.go @@ -240,6 +240,7 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co return fmt.Sprintf("! ls %s", helperErrorFileName) }) if remoteOutput.NumErrors != 0 { + gplog.Error("gpbackup_helper failed to start on some segments") // the delay below is to give the restore helper a chance to close all pipes, if it can... time.Sleep(5 * time.Second) cancel() From ecb4eafe35f87fd8a4e05f5e316c19b1be9d73ec Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 1 Jul 2024 11:07:46 +1000 Subject: [PATCH 20/27] Fix test after merge --- plugins/example_plugin.bash | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index 64659a628..35a2ebbdd 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -76,13 +76,12 @@ restore_data() { echo "restore_data $1 $2" >> /tmp/plugin_out.txt filename=`basename "$2"` timestamp_dir=`basename $(dirname "$2")` - if [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then - sleep 3 - echo 'error' >&2 - fi timestamp_day_dir=${timestamp_dir%??????} if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 + elif [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then + pkill "restore-agent" + exit 1 elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then exit 1 fi From 7dd829e65ae14a29c901d7544819fcc77f9a51ba Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 22 Jul 2024 09:42:10 +1000 Subject: [PATCH 21/27] Update delta in the example plugin --- plugins/example_plugin.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index 35a2ebbdd..25df1221c 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -80,7 +80,7 @@ restore_data() { if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 elif [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then - pkill "restore-agent" + pkill "gpbackup_helper" exit 1 elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then exit 1 From 9384554383531179615982b740d670fdd03af560 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Mon, 5 Aug 2024 16:29:52 +1000 Subject: [PATCH 22/27] Fix comments and spaces --- plugins/example_plugin.bash | 4 ++-- restore/data.go | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index 3756e7cea..5989c395d 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -80,8 +80,8 @@ restore_data() { if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 elif [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then - pkill "gpbackup_helper" - exit 1 + pkill "gpbackup_helper" + exit 1 elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then exit 1 fi diff --git a/restore/data.go b/restore/data.go index 50599d064..ca47a073e 100644 --- a/restore/data.go +++ b/restore/data.go @@ -259,10 +259,8 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Make sure it's called to release resources even if no errors - // Launch a checker that polls if the restore helper has ended with an error. - // It is our 'Ultima ratio regum' - in case restore helper couldn't read a file with the oid list, it is not aware - // about the pipes, pre-created by the gprestore, and it can't close them. So we cancel all pending COPY commands - // from here after giving a chance to the restore helper to close pipes on its own. + // Launch a checker that polls if the restore helper has ended with an error. It will cancel all pending + // COPY commands that could be hanging on pipes, that the restore helper didn't close before it died. if backupConfig.SingleDataFile || resizeCluster { go func() { for { From 7102051e2e86837443027430533ed419032721c7 Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 7 Aug 2024 14:23:32 +1000 Subject: [PATCH 23/27] Add execution context into backup Plus add checker goroutine into backup. --- backup/data.go | 36 ++++++++++++++++++++++++++++++------ backup/data_test.go | 19 ++++++++++--------- restore/data.go | 15 +-------------- utils/agent_remote.go | 16 ++++++++++++++++ 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/backup/data.go b/backup/data.go index e3d4c37d4..c06debb7a 100644 --- a/backup/data.go +++ b/backup/data.go @@ -5,6 +5,7 @@ package backup */ import ( + "context" "errors" "fmt" "strings" @@ -62,7 +63,7 @@ type BackupProgressCounters struct { ProgressBar utils.ProgressBar } -func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite string, connNum int) (int64, error) { +func CopyTableOut(queryContext context.Context, connectionPool *dbconn.DBConn, table Table, destinationToWrite string, connNum int) (int64, error) { if wasTerminated { return -1, nil } @@ -112,7 +113,7 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite } else { utils.LogProgress(`%sExecuting "%s" on master`, workerInfo, query) } - result, err := connectionPool.Exec(query, connNum) + result, err := connectionPool.ExecContext(queryContext, query, connNum) if err != nil { return 0, err } @@ -121,7 +122,7 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite return numRows, nil } -func BackupSingleTableData(table Table, rowsCopiedMap map[uint32]int64, counters *BackupProgressCounters, whichConn int) error { +func BackupSingleTableData(queryContext context.Context, table Table, rowsCopiedMap map[uint32]int64, counters *BackupProgressCounters, whichConn int) error { workerInfo := "" if gplog.GetVerbosity() >= gplog.LOGVERBOSE { workerInfo = fmt.Sprintf("Worker %d: ", whichConn) @@ -137,7 +138,7 @@ func BackupSingleTableData(table Table, rowsCopiedMap map[uint32]int64, counters } else { destinationToWrite = globalFPInfo.GetTableBackupFilePathForCopyCommand(table.Oid, utils.GetPipeThroughProgram().Extension, false) } - rowsCopied, err := CopyTableOut(connectionPool, table, destinationToWrite, whichConn) + rowsCopied, err := CopyTableOut(queryContext, connectionPool, table, destinationToWrite, whichConn) if err != nil { return err } @@ -181,6 +182,15 @@ func BackupDataForAllTables(tables []Table) []map[uint32]int64 { tasks <- table } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // Make sure it's called to release resources even if no errors + + // Launch a checker that polls if the backup helper has ended with an error. It will cancel all pending + // COPY commands that could be hanging on pipes, that the backup helper didn't close before it died. + if MustGetFlagBool(options.SINGLE_DATA_FILE) { + utils.StartHelperChecker(globalCluster, globalFPInfo, cancel) + } + /* * Worker 0 is a special database connection that * 1) Exports the database snapshot if the feature is supported @@ -212,8 +222,15 @@ func BackupDataForAllTables(tables []Table) []map[uint32]int64 { * transaction commits and the locks are released. */ for table := range tasks { + // Check if any error occurred in any other goroutines: + select { + case <-ctx.Done(): + return // Error somewhere, terminate + default: // Default is must to avoid blocking + } if wasTerminated || isErroredBackup.Load() { counters.ProgressBar.(*pb.ProgressBar).NotPrint = true + cancel() return } if backupSnapshot != "" && connectionPool.Tx[whichConn] == nil { @@ -261,7 +278,7 @@ func BackupDataForAllTables(tables []Table) []map[uint32]int64 { break } } - err = BackupSingleTableData(table, rowsCopiedMaps[whichConn], &counters, whichConn) + err = BackupSingleTableData(ctx, table, rowsCopiedMaps[whichConn], &counters, whichConn) if err != nil { // if copy isn't working, skip remaining backups, and let downstream panic // handling deal with it @@ -294,14 +311,21 @@ func BackupDataForAllTables(tables []Table) []map[uint32]int64 { }() for _, table := range tables { for { + // Check if any error occurred in any other goroutines: + select { + case <-ctx.Done(): + return // Error somewhere, terminate + default: // Default is must to avoid blocking + } if wasTerminated || isErroredBackup.Load() { + cancel() return } state, _ := oidMap.Load(table.Oid) if state.(int) == Unknown { time.Sleep(time.Millisecond * 50) } else if state.(int) == Deferred { - err := BackupSingleTableData(table, rowsCopiedMaps[0], &counters, 0) + err := BackupSingleTableData(ctx, table, rowsCopiedMaps[0], &counters, 0) if err != nil { isErroredBackup.Store(true) gplog.Fatal(err, "") diff --git a/backup/data_test.go b/backup/data_test.go index 2743d1328..3fd8fe038 100644 --- a/backup/data_test.go +++ b/backup/data_test.go @@ -1,6 +1,7 @@ package backup_test import ( + "context" "fmt" "regexp" @@ -79,7 +80,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -92,7 +93,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -102,7 +103,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -115,7 +116,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -125,7 +126,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -138,7 +139,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -148,7 +149,7 @@ var _ = Describe("backup/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := backup.CopyTableOut(connectionPool, testTable, filename, defaultConnNum) + _, err := backup.CopyTableOut(context.Background(), connectionPool, testTable, filename, defaultConnNum) Expect(err).ShouldNot(HaveOccurred()) }) @@ -178,7 +179,7 @@ var _ = Describe("backup/data tests", func() { backupFile := fmt.Sprintf("/gpbackup__20170101010101_pipe_(.*)_%d", testTable.Oid) copyCmd := fmt.Sprintf(copyFmtStr, backupFile) mock.ExpectExec(copyCmd).WillReturnResult(sqlmock.NewResult(0, 10)) - err := backup.BackupSingleTableData(testTable, rowsCopiedMap, &counters, 0) + err := backup.BackupSingleTableData(context.Background(), testTable, rowsCopiedMap, &counters, 0) Expect(err).ShouldNot(HaveOccurred()) Expect(rowsCopiedMap[0]).To(Equal(int64(10))) @@ -190,7 +191,7 @@ var _ = Describe("backup/data tests", func() { backupFile := fmt.Sprintf("/backups/20170101/20170101010101/gpbackup__20170101010101_%d", testTable.Oid) copyCmd := fmt.Sprintf(copyFmtStr, backupFile) mock.ExpectExec(copyCmd).WillReturnResult(sqlmock.NewResult(0, 10)) - err := backup.BackupSingleTableData(testTable, rowsCopiedMap, &counters, 0) + err := backup.BackupSingleTableData(context.Background(), testTable, rowsCopiedMap, &counters, 0) Expect(err).ShouldNot(HaveOccurred()) Expect(rowsCopiedMap[0]).To(Equal(int64(10))) diff --git a/restore/data.go b/restore/data.go index ca47a073e..2396d7360 100644 --- a/restore/data.go +++ b/restore/data.go @@ -9,7 +9,6 @@ import ( "fmt" "sync" "sync/atomic" - "time" "github.com/greenplum-db/gp-common-go-libs/cluster" "github.com/greenplum-db/gp-common-go-libs/dbconn" @@ -262,19 +261,7 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co // Launch a checker that polls if the restore helper has ended with an error. It will cancel all pending // COPY commands that could be hanging on pipes, that the restore helper didn't close before it died. if backupConfig.SingleDataFile || resizeCluster { - go func() { - for { - time.Sleep(5 * time.Second) - remoteOutput := globalCluster.GenerateAndExecuteCommand("Checking gpbackup_helper agent failure", cluster.ON_SEGMENTS, func(contentID int) string { - helperErrorFileName := fmt.Sprintf("%s_error", fpInfo.GetSegmentPipeFilePath(contentID)) - return fmt.Sprintf("! ls %s", helperErrorFileName) - }) - if remoteOutput.NumErrors != 0 { - gplog.Error("gpbackup_helper failed to start on some segments") - cancel() - } - } - }() + utils.StartHelperChecker(globalCluster, globalFPInfo, cancel) } for i := 0; i < connectionPool.NumConns; i++ { diff --git a/utils/agent_remote.go b/utils/agent_remote.go index af43f42d4..3ee401afd 100644 --- a/utils/agent_remote.go +++ b/utils/agent_remote.go @@ -361,3 +361,19 @@ func CreateSkipFileOnSegments(oid string, tableName string, c *cluster.Cluster, return fmt.Sprintf("Could not create skip file %s_skip_%s on segments", fpInfo.GetSegmentPipeFilePath(contentID), oid) }) } + +func StartHelperChecker(cl *cluster.Cluster, fpInfo filepath.FilePathInfo, cancel func()) { + go func() { + for { + time.Sleep(5 * time.Second) + remoteOutput := cl.GenerateAndExecuteCommand("Checking gpbackup_helper agent failure", cluster.ON_SEGMENTS, func(contentID int) string { + helperErrorFileName := fmt.Sprintf("%s_error", fpInfo.GetSegmentPipeFilePath(contentID)) + return fmt.Sprintf("! ls %s", helperErrorFileName) + }) + if remoteOutput.NumErrors != 0 { + gplog.Error("gpbackup_helper failed to start on some segments") + cancel() + } + } + }() +} From be1b0c6ef048fc0affc25aee643c35f7e22b373f Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Wed, 7 Aug 2024 17:27:42 +1000 Subject: [PATCH 24/27] Update test --- end_to_end/plugin_test.go | 4 ++-- plugins/example_plugin.bash | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index d953e3060..b173df68c 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -340,7 +340,7 @@ var _ = Describe("End to End plugin tests", func() { "Instruct plugin to fail", cluster.ON_HOSTS, func(contentID int) string { - return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_FAIL") + return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_KILL_HELPER") }) gprestoreCmd := exec.Command(gprestorePath, @@ -355,7 +355,7 @@ var _ = Describe("End to End plugin tests", func() { "Unset plugin failure", cluster.ON_HOSTS, func(contentID int) string { - return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_FAIL") + return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_KILL_HELPER") }) assertArtifactsCleaned(timestamp) diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index 5989c395d..626ee8097 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -79,9 +79,8 @@ restore_data() { timestamp_day_dir=${timestamp_dir%??????} if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 - elif [ -e "/tmp/GPBACKUP_PLUGIN_FAIL" ] ; then + elif [ -e "/tmp/GPBACKUP_PLUGIN_KILL_HELPER" ] ; then pkill "gpbackup_helper" - exit 1 elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then exit 1 fi From 30d89e22f7e9262e947eb7a14f936195a3c0023c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 9 Aug 2024 10:22:20 +1000 Subject: [PATCH 25/27] Update the test --- end_to_end/plugin_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index b173df68c..d5b9ce65c 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -324,7 +324,7 @@ var _ = Describe("End to End plugin tests", func() { Skip("This test is only needed for the most recent backup versions") } }) - It("Will not hang if gpbackup and gprestore runs with single-data-file and plugin fails at init for restore", func(ctx SpecContext) { + It("Will not hang if gpbackup and gprestore runs with single-data-file and the helper goes down at its start", func(ctx SpecContext) { copyPluginToAllHosts(backupConn, examplePluginExec) testhelper.AssertQueryRuns(backupConn, "CREATE TABLE t0(a int);") @@ -337,12 +337,19 @@ var _ = Describe("End to End plugin tests", func() { timestamp := getBackupTimestamp(string(output)) backupCluster.GenerateAndExecuteCommand( - "Instruct plugin to fail", + "Instruct plugin to kill the helper", cluster.ON_HOSTS, func(contentID int) string { return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_KILL_HELPER") }) + defer backupCluster.GenerateAndExecuteCommand( + "Unset plugin instruction", + cluster.ON_HOSTS, + func(contentID int) string { + return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_KILL_HELPER") + }) + gprestoreCmd := exec.Command(gprestorePath, "--timestamp", timestamp, "--redirect-db", "restoredb", @@ -351,13 +358,6 @@ var _ = Describe("End to End plugin tests", func() { _, err := gprestoreCmd.CombinedOutput() Expect(err).To(HaveOccurred()) - backupCluster.GenerateAndExecuteCommand( - "Unset plugin failure", - cluster.ON_HOSTS, - func(contentID int) string { - return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_KILL_HELPER") - }) - assertArtifactsCleaned(timestamp) }, SpecTimeout(time.Second*30)) It("runs gpbackup and gprestore with plugin, single-data-file, and no-compression", func() { From 465ce05035fb088a48be8a7c693adf3eb0c8981e Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Fri, 9 Aug 2024 17:06:30 +1000 Subject: [PATCH 26/27] Simplify test and plugin code --- end_to_end/plugin_test.go | 6 +++--- plugins/example_plugin.bash | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index d5b9ce65c..eac2ce2da 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -337,17 +337,17 @@ var _ = Describe("End to End plugin tests", func() { timestamp := getBackupTimestamp(string(output)) backupCluster.GenerateAndExecuteCommand( - "Instruct plugin to kill the helper", + "Instruct plugin to fail", cluster.ON_HOSTS, func(contentID int) string { - return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_KILL_HELPER") + return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_DIE") }) defer backupCluster.GenerateAndExecuteCommand( "Unset plugin instruction", cluster.ON_HOSTS, func(contentID int) string { - return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_KILL_HELPER") + return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_DIE") }) gprestoreCmd := exec.Command(gprestorePath, diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index 626ee8097..b2e193d3f 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -79,8 +79,6 @@ restore_data() { timestamp_day_dir=${timestamp_dir%??????} if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 - elif [ -e "/tmp/GPBACKUP_PLUGIN_KILL_HELPER" ] ; then - pkill "gpbackup_helper" elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then exit 1 fi From 405a815e21a14d28b8311a6829519f996845e43c Mon Sep 17 00:00:00 2001 From: Roman Eskin Date: Tue, 13 Aug 2024 12:16:05 +1000 Subject: [PATCH 27/27] Add test with resize cluster --- end_to_end/plugin_test.go | 41 ++++++++++++++++++ .../resources/9-segment-db-with-plugin.tar.gz | Bin 0 -> 6832 bytes plugins/example_plugin.bash | 6 ++- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 end_to_end/resources/9-segment-db-with-plugin.tar.gz diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index eac2ce2da..5e04483c5 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -360,6 +360,47 @@ var _ = Describe("End to End plugin tests", func() { assertArtifactsCleaned(timestamp) }, SpecTimeout(time.Second*30)) + It("Will not hang if gprestore runs with cluster resize and the helper goes down on one of the tables", func(ctx SpecContext) { + copyPluginToAllHosts(backupConn, examplePluginExec) + + pluginBackupDirectory := `/tmp/plugin_dest` + os.Mkdir(pluginBackupDirectory, 0777) + command := exec.Command("tar", "-xzf", fmt.Sprintf("resources/%s.tar.gz", "9-segment-db-with-plugin"), "-C", pluginBackupDirectory) + mustRunCommand(command) + + backupCluster.GenerateAndExecuteCommand( + "Instruct plugin to fail", + cluster.ON_HOSTS, + func(contentID int) string { + return fmt.Sprintf("touch /tmp/GPBACKUP_PLUGIN_DIE") + }) + + defer backupCluster.GenerateAndExecuteCommand( + "Unset plugin instruction", + cluster.ON_HOSTS, + func(contentID int) string { + return fmt.Sprintf("rm /tmp/GPBACKUP_PLUGIN_DIE") + }) + + timestamp := "20240812201233" + + gprestoreCmd := exec.Command(gprestorePath, + "--resize-cluster", + "--timestamp", timestamp, + "--redirect-db", "restoredb", + "--plugin-config", examplePluginTestConfig) + + // instruct plugin to die only before restoring the last table + gprestoreCmd.Env = os.Environ() + gprestoreCmd.Env = append(gprestoreCmd.Env, "GPBACKUP_PLUGIN_DIE_ON_OID=16392") + + _, err := gprestoreCmd.CombinedOutput() + Expect(err).To(HaveOccurred()) + + assertArtifactsCleaned(timestamp) + + os.RemoveAll(pluginBackupDirectory) + }, SpecTimeout(time.Second*30)) It("runs gpbackup and gprestore with plugin, single-data-file, and no-compression", func() { copyPluginToAllHosts(backupConn, examplePluginExec) diff --git a/end_to_end/resources/9-segment-db-with-plugin.tar.gz b/end_to_end/resources/9-segment-db-with-plugin.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..9ecebe887ef32e2e7539765cb3a8087d7c055bf0 GIT binary patch literal 6832 zcmV;h8c*dPiwFP!000001MFLSSX0-Q*RQtTf`ZCkY1q25B8Y1qs}OiVg-jghEXPieL;Xpz=^a$(%@; zf{%)R!kzhs^-q#>*4k^Iwf1lS);@d3!gA4^3kntj0jK#y6L413Kc{378jHu_F?bZp z3cij;qp-L|2;AIvZPB8bXpk9&Kr9N5WCn(D!{4a)w)_7_#lq4GWramqqpjv{AiPKZ z30T5H{^vuc<$vmn6$)*I!F<5s@NW5s_0SlM)k6N~Mc$nM;K%?bD``G21OwdsF(@wwz9^A=KO9ctOf z(DSX@1`;Q?CpBJQtzS$8f9m_|QPG8pibNls(NYuRU!yq5M;)Vmbz~tndMECg2cpDPNuHPdM_OtJl}Ln+~ntr zy-|9ld0FcQ?v{RU)xWR&59QasB7*|Gs#yBrA0@fSXp1%s+=rsdw<-JkqkC>U6g|p ztIJh>l`7N^ZaEd)$_Fw@!$ic8B-`SC=hKgWJ0=>xK!v)A#!0<}1PM-RQw*6{4hc0* zvxqH2arMrF#iIHFd-2+WA&V!QcvT7FiWEq<%-f>Q;^R5X|19ut^&0$J<3A++@ze2- znZ!SO0sr$M)9^2ME$Kmg<^;tTf4NktJKpKl{$IWuN{beP}kG>~iw z&mHq)!cZ6!Fz@ACUq;2`OuM=1@w3 z#+(ghmpA#HxzheOpZ)heRgTCph!kSAgLCDOer)}Plf^qZ{oz7&8v!b&mszvKqfZ<# zMVwugoYGhrd}TH}yFU`9k8qfZ!GqlBbIzEerhJOx!R4~icDM1fd{|c%*(j-djuku- z$!kzNNHdLeH4xzWL^BWgV#bKpag=(*k=?v!eUey1OJ3}OUSg}+*kUuEPqC>V zejb!MWGuL|4N@jKE?c!a1^N#=(;Kw5|;Ijk5}4j7aZYg@u~#WwyN<))#N z*<*ydmS45#116UDZnd@S)D_3KIBc~&J+RU_y7<(I70#I8uC-!qSvl~$93>dnhv5@6 zA$lZ5Wt^>pA)9eILTFil-KQ%?7I2hi*t_}}@>=ZH4iI`^dHCZWy%v+AM^G?9XOC+L zHGZF=?5yZoN`Oz_0q(FRIixwYA8Zbg=IRgTwRS%fwexaE44I13RESSe+}9Qx6qLp5 z$b*LSN7%<5qQjxC_Pj<8^Fe`3NwTflB)`eX)w!<#y|g9c1*lyeHNF42#VzM9o;dPt zZ@J~+u!eo+Si5to<&9=AZaV_u7Lp-WyAfkiiFx7@Q^wy5pIkVnTKX5qII;VZe{z1` zCbPglX0G`khbO$s{|oq^7nxE2>%jH@lAp$_QkRD0OsLO-R~P$jbKD*p-(ZcN$X90G zD%;T`dHw~V9ABq$Lyk3*&R~j8Wtcka+QLk|C))xpwH14qL*i_OzlO^CQ2v}N=?ZoV z&!7Z2^i`w)Yk$KmGb%#6fN%BlnYWY{uDIl`Asi=Q@}J zGi2>w!EJrg_79bMjhk`K^HpxGy9H-bVcrx-`CJ{P9KzYT~lp2RG8h`$Hdlbf}3ew%~UY7<;tfLSqwg`UkL4|D30YohgXG_oadIB ziU|dA-nk60+EgN^zbce|!5>L8v4tyxlB>dz0#tcNZA)&8*OhwP&!wuub6z`y zc||{ND>26o>a}k60%-%5q+C{AF z*E;OlkWJV@gEIr#nnK>I6e^(jBpraVDKJm#-zS{8JaDL$%X~mqTKZ&l1NqY33Mn57 zy(J?0=Z^FUYCEb!`!9_I9T&-x$oc9p+eEyiPHn|&PWz!-9lsvNfAHRF)dK@bT{3#j z7NodBqIzh0SSb)STtrnuX{KQF9civb2!2;9dq});EiPIQUJ1nBK(*JE15cu=noKNt z#j+uIO^`(lg%>-|RfPoW9YY)142UDCP&$Q|aG>`rKcfR|PAD!?E#39)#95=cTloJj z@5%ok^!?vV{{OxH$1UK0K4gLa7x@1@`2WMc|9=htqpTM2KOeHd{|o&8w)g)t(I3VJ znSo4@X%W3Qbap-_-|=JLgnuj={WAZf(OAp^{^vuq=zs@;70P9YgA8^!Di3Smhrd@X-`C>|JJdVd(YM*t04K(FPpXGK!umbz-AQD!=8;L%ck%8>r8s)h zXaL308)5Y>b~EFPd86r{C-hy-W*Mb7SQ+O%tvaHD&=12j)Me2+xg2GHO(Ce2@HPJVTAK#-{}BnsRYcQVOQ^H>ywyFH!0 zlS*?!Aj1N20az^4CIE#G#1ZgJQ~=5v!wSTs*tj5T0*)1gwMPC4=>OyaC*1>H9v=1% zaBnt7Mn!-TtcXw|k^_Q~NV7jt2#yR2Q2_&0NG)Yp-hnd$H017mAaDxU-h8=F=RB&Qr$FDFal0E-EkY>W{-%B2?*t~ z5F|PR3A0^zATug(>RL35!wzFko9Snoyv_j)*gcrMMr_pjl!@SmgX~~-)Yn>{22A_Tn)kDmMezt^1q!7ku`K4b>|GY`CY|9AZ46FOp58D8$&)*;Rm5${UK)5WoBzGa zHQ)9LqvgpVQF5*& z&KX@Pd*p&BaM6zYcA^1IWN=RZ$)$XaPE8(4pCef^Q}``bDDf|}Xpt9X#} zR>dz+j(J$+Ef#Zs%7aL;&2cdPF7QNczKvIWM^qnt3hpqmH4Ci~X|3bk1N?rt3cze0a*k8U9yTR(2}h(G_awP&U(L`Hu#OlP zd8zF472QvrqRFCoB24PT15Voq)@eyO^iGCerCgF6Pc~+xJ1`VjMY;$|(BnapNBQ4e z0w3C>b(BGg{>@${Cpi98<72^0eBSLsn|*pR85Rj^U5lWXRhy>b{&5Q_=%xDjxiaH_ zbU&hh*S*Z^GvIy{xp^W;Z>&ku?&X=LtSFT6HY)r=gf<(vC-hVOHJsm9k4}pj_r2iE zP=LI2hX{qrC0}Qd?ll66yvNdZHQoF~5*@$gteNe_S`L19^Bo`U=N$Q+@$~-RlG2Dj z|Kj01*im@#mzXr=Qzu;oXNz6$pACt z+^gq*tQOAy&5O*a|HD5*zgYjLrb8P1`)yd=^wC!<*~Lw)Co1S?UJvbVk(zemJJ%$@ zVt8dm3!|sm@H^LNz~YQi1<2^3jRzUROYxBuse`WV-XrSmM-;mwws%)c&iNY3h8-eo z;kEZ3{&S$O1QpX5a4#d}0?ez+3elN`TMXTVMlQ@9pfHcJ{q8YmU0XD%#NgnyPW_CL zRA|}xrgVz&vCFw?_3~}T(FfsXjB^t71yZ1U7$vw5WVfMV#w{H{jWv_9+X!xryRqZi zvW{Bu<448o1w>#l@=Vh)VG~?5L)~(6WBv1v-NdLxXVptwRA-sIlDC=0#bmG zEs>lae5GvMz?rJ)t%Aje{5C1Bca-JTz(t2`qW@uknBhAlm}-qP{;1}Ef^}7zEqi8% zbLn~2=Kh_aO;)efPWhe9l7{%hK`GE3Oh{#-{^5R5WKf{F=by}YY9-W`5w>TF=#2 zNW4yMdH2OjLrMe9K`K6lcTCPHStYIOaEO+w9^JGeHgd)mKfLf;wbxv%1!jSN!d##K zA)sEV{}=dwp5*2K|9-aQrv&Lq-P@E2HH#99JNS3fRjfIu>t9zmT0W)QedpK>=*U*v zl+|1nCavQ)W|~x*N2x}x3KPEIcNo|V?dWOx^*hJDaS@EzJq6jQ^0VT|@a$~YJy~jJ zhVAJTb1kW=u%mh-T-GP4oVMDY)@?-#Sdwb%@Oyk*M$mF`Uc|i|8%bm8Mr=(4jL|D? zVh-4!4&_sfyVD&Awp?;%{*~JkM(2;*fVJh%{PkoLfKayp>yqVw>BJ&W78jrYl{5~2UA}_vuwkCLm67^ zEia4HKhz30u~(hm%3dQLE#VTAVSk8q!-qBmJ=NgB$OV6 zuA|`<#(%eWH9u|>QT%Si3I2dl#i0n2wc{jCB5~N#ZH2@FVu1r6S~;DC@oNW7WxXX0!VlXXD6pQ?22H=f_~ zcs|~HGammbe+90_d*AHdxU?UB`oPYIznvXi`|;N83lDyB{mVDr2LI%byKg;o^Y#bd zzWdbR<+~qzeswQAzI)$|Z|Lt&+w0pDy2j{f&=4IyidmpZ5qA-~V6p_+R4xcd`A~0et@--~X>L z|F_=t|7G!ieE+`|;QSxX|5fq-*Bbxp%)kGu;QU`LK>r{8|0?nSQ=k9G^*^-$wj?lPlfS+&F}xtjsLQS>wjtij{i9RR~Y|Kzx`wUzZT&5kK=!p z{lC_0|F`Dte~kat0^I)t_y4G{{(rsW|NQ!YT>n!OaQ#27|F1Crx90DE%&-4daR0Ab zfb)Mi|5suC?^^Bu;`(3w{zsj_^}o3OceVZhLf2>>>yst-{$mg}{-XW=6uHyw&W`^o zzW-AXOxqpmHg!VZqQO9F#2Z2l$eA+|A&x$w@t^MSRceH6kF5dO+u7S=`-xqA{_#B! z`E*Qu>KIg{i7#N~8V-*`NH4=7bw=VqRp2PJs6T-J8(hYIOPRCXv}QCY?z**%4xBbs zqOUHmL~eJpMOyG3zLwE~^QKxT?fz!e5X7k-(B78Uqp*+JKh{K0XPR_Zs2yw8q+6&V zs-kU56G5G6qHXNy2CrFf(QB_zQ`f17AkM~KxwX6=vDr1N!_!YLHRubf zY`kr?S7;*1-k=q^W3t>}FUzZukA^eU$@ZVi4e>Mg-fnl-L{D~ip4(;z%jv;+b9(f8 zrFwWiW!s9En4)6C$yB%Wuwat%?Le1Ph%1m+DDAJvORtRo+gq`6-=V zO}xD#uh!O<;tiDVBtR!reL* zZo!rFC1hnqZnY|R%$4Krm6f@AI^4zT-BMawDYxfBSDNeCSlx$wLMsR@Bgmg!uKN>J z)EYt2vp6gq=pkG)jQDtEIp%sgc~m-ztX!7W&!;Ix{lB<6IbFJV1E&2YJg75#-0{s3Le*bH=_&@Z2t{<+#adP}$R=cX4YyTa5 z|EES=^eJ^bJDQO5FhmXYfRc0kv6E-{bC#sgn!s?WPdsvo9uG)@Vy%!&Z!{!_)DK{| z%o34qv!ykaeH~0GJ_dHWkBsPHv!yo8-dNELRqM74GK?&HL_i=Vm2M_0rK!rzPL_?* z5!v*bCe#~)${kta<`cgGG5RNnU0K7OOmt^NY|F_BNlMsg0Oiny%Nvc+kY$m4WTVl5 z@-Z!^VYtrNGRamH_**{R3JyS@t!be48r$d3o!@z((dqWpMv`ONfVRZt!4^G&Qncy5 zINlf8@wqgn9r(Y&Oz9ExiseZnIiW|?i1?>XM2cc=O}O&|DZZBgEY^HZ+qO;i#JNBc z1}zy|c1VR0#PO8taI3|RMl>LqfjZzaUbyTUO2)1~(Zl$v-N+6t(1O<>jJltByLUAT@vgC(0 z0rEhGe{Cr(F1Xn!$P(>Eh5^qaSEB;!N{&?Nw#n`}DG>Ka3^#JZOxl+ueulVXl4c+Z zJs|KL1?fQ2P1C0)*pUI5hb$R-5SB?2v1{A>-WFU9g3#A3rz{UHjvH91IQfE&=Ua#H zNX>$Za#k#6WXxSJOZ1SH3K-a?2^{m{qmVias6hgWWqED85&K>`KZ0g!^H zyg+l_Wi85Gjm4f`dOchX)de25?i&Y0beM6Q4(7xoi$8kZPo7Ls4BlK&s4+!x^0B2{ zU0YCvQ}MBPQmIA2MkY{yp^q(TC5INBo^W+tuK&XIU#t0l zIkR<1j(hljN?TE9{Xb1Z|F0H^XFRfe9^Vv4sqeLn+m(1OEF2p85zV_YBBaipn8ao* zOfM!UQ^Hl<#n=RoE$}|i&~@zNBnmrXVOnZa9;}7?qetAo@o8*3^RRn-Qdl{{Ey`lM z1z{mQH(X|u#|~7HAn~e72@1kO`b7kpLnThxR3E3IY945g1xesJUn0U=9F%&G;_uwk zSDM`L1e=zEZRD^?i8LAjQVWPky9dNh}<-3P-9GfPrN5xnJgACW@X! z1b(1!1B;UO8EI;T2W}LKRgFQC7dxBxXvNvY?MvLfO*h4jVDu literal 0 HcmV?d00001 diff --git a/plugins/example_plugin.bash b/plugins/example_plugin.bash index b2e193d3f..302d5d6c8 100755 --- a/plugins/example_plugin.bash +++ b/plugins/example_plugin.bash @@ -79,7 +79,11 @@ restore_data() { timestamp_day_dir=${timestamp_dir%??????} if [ -e "/tmp/GPBACKUP_PLUGIN_LOG_TO_STDERR" ] ; then echo 'Some plugin warning' >&2 - elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" ] ; then + elif [ -e "/tmp/GPBACKUP_PLUGIN_DIE" -a "$GPBACKUP_PLUGIN_DIE_ON_OID" = "" ] ; then + exit 1 + elif [[ -e "/tmp/GPBACKUP_PLUGIN_DIE" && "$filename" == *"$GPBACKUP_PLUGIN_DIE_ON_OID"* ]] ; then + # sleep a while for test purposes - to let gprestore start COPY commands + sleep 5 exit 1 fi cat /tmp/plugin_dest/$timestamp_day_dir/$timestamp_dir/$filename