Merge pull request #36 from arenadata/1.29.2-sync

Sync 1.29.2 changes

backup/backup.go

@@ -419,9 +419,8 @@ func DoTeardown() {
 			backupReport.ConstructBackupParamsString()
 
 			history.WriteConfigFile(&backupReport.BackupConfig, configFilename)
-			if backupReport.BackupConfig.EndTime == "" {
-				backupReport.BackupConfig.EndTime = history.CurrentTimestamp()
-			}
+			// We always want to override the initial end time set by the call to StoreBackupHistory
+			backupReport.BackupConfig.EndTime = history.CurrentTimestamp()
 			endtime, _ := time.ParseInLocation("20060102150405", backupReport.BackupConfig.EndTime, operating.System.Local)
 			backupReport.WriteBackupReportFile(reportFilename, globalFPInfo.Timestamp, endtime, objectCounts, errMsg)
 			report.EmailReport(globalCluster, globalFPInfo.Timestamp, reportFilename, "gpbackup", !backupFailed)
@@ -477,10 +476,10 @@ func DoCleanup(backupFailed bool) {
 			utils.CleanUpHelperFilesOnAllHosts(globalCluster, globalFPInfo)
 		}
 
-		// The gpbackup_history entry is written to the DB with an "In Progress" status very early
-		// on.  If we get to cleanup and the backup succeeded, mark it as a success, otherwise mark
-		// it as a failure. Between our signal handler and recovering panics, there should be no
-		// way for gpbackup to exit that leaves the entry in the initial status.
+		// The gpbackup_history entry is written to the DB with an "In Progress" status and a preliminary EndTime value
+		// very early on.  If we get to cleanup and the backup succeeded, mark it as a success, otherwise mark it as a
+		// failure; in either case, update the end time to the actual value. Between our signal handler and recovering
+		// panics, there should be no way for gpbackup to exit that leaves the entry in the initial status.
 
 		if !MustGetFlagBool(options.NO_HISTORY) {
 			var statusString string
@@ -494,7 +493,7 @@ func DoCleanup(backupFailed bool) {
 			if err != nil {
 				gplog.Error(fmt.Sprintf("Unable to update history database.  Error: %v", err))
 			} else {
-				_, err := historyDB.Exec(fmt.Sprintf("UPDATE backups SET status='%s' WHERE timestamp='%s'", statusString, globalFPInfo.Timestamp))
+				_, err := historyDB.Exec(fmt.Sprintf("UPDATE backups SET status='%s', end_time='%s' WHERE timestamp='%s'", statusString, backupReport.BackupConfig.EndTime, globalFPInfo.Timestamp))
 				historyDB.Close()
 				if err != nil {
 					gplog.Error(fmt.Sprintf("Unable to update history database.  Error: %v", err))

backup/predata_relations.go

@@ -207,7 +207,12 @@ func printColumnDefinitions(metadataFile *utils.FileWithByteCount, columnDefs []
 		}
 		if column.HasDefault {
 			if column.AttGenerated != "" {
-				line += fmt.Sprintf(" GENERATED ALWAYS AS %s %s", column.DefaultVal, column.AttGenerated)
+				// Unlike most keywords, GENERATED cannot be applied to a column that inherits from a parent table,
+				// even if the specified generation expression is identical to that of the column it inherits,
+				// so we skip printing it in that case.
+				if !column.IsInherited {
+					line += fmt.Sprintf(" GENERATED ALWAYS AS %s %s", column.DefaultVal, column.AttGenerated)
+				}
 			} else {
 				line += fmt.Sprintf(" DEFAULT %s", column.DefaultVal)
 			}

backup/predata_relations_tables_test.go

@@ -59,6 +59,7 @@ ENCODING 'UTF-8';`)
 		colStorageType := backup.ColumnDefinition{Oid: 0, Num: 1, Name: "i", Type: "integer", StatTarget: -1, StorageType: "PLAIN"}
 		colWithCollation := backup.ColumnDefinition{Oid: 0, Num: 1, Name: "c", Type: "character (8)", StatTarget: -1, Collation: "public.some_coll"}
 		rowTwoGenerated := backup.ColumnDefinition{Oid: 0, Num: 2, Name: "j", HasDefault: true, Type: "integer", StatTarget: -1, DefaultVal: "(i * 2)", AttGenerated: "STORED"}
+		rowTwoGeneratedInherited := backup.ColumnDefinition{Oid: 0, Num: 2, Name: "j", HasDefault: true, Type: "integer", StatTarget: -1, DefaultVal: "(i * 2)", AttGenerated: "STORED", IsInherited: true}
 
 		Context("No special table attributes", func() {
 			It("prints a CREATE TABLE OF type block with one attribute", func() {
@@ -188,6 +189,15 @@ ALTER TABLE ONLY public.tablename ALTER COLUMN i SET (n_distinct=1);`)
 				testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TABLE public.tablename (
 	i integer,
 	j integer GENERATED ALWAYS AS (i * 2) STORED
+) DISTRIBUTED RANDOMLY;`)
+			})
+			It("prints a CREATE TABLE block that omits the generated attribute if it inherits from another table", func() {
+				col := []backup.ColumnDefinition{rowOne, rowTwoGeneratedInherited}
+				testTable.ColumnDefs = col
+				backup.PrintRegularTableCreateStatement(backupfile, tocfile, testTable)
+				testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TABLE public.tablename (
+	i integer,
+	j integer
 ) DISTRIBUTED RANDOMLY;`)
 			})
 		})

backup/queries_table_defs.go

@@ -224,6 +224,7 @@ type ColumnDefinition struct {
 	SecurityLabelProvider string
 	SecurityLabel         string
 	AttGenerated          string
+	IsInherited           bool
 }
 
 var storageTypeCodes = map[string]string{
@@ -348,7 +349,8 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi
 		coalesce(array_to_string(ARRAY(SELECT option_name || ' ' || quote_literal(option_value) FROM pg_options_to_table(attfdwoptions) ORDER BY option_name), ', '), '') AS fdwoptions,
 		CASE WHEN a.attcollation <> t.typcollation THEN quote_ident(cn.nspname) || '.' || quote_ident(coll.collname) ELSE '' END AS collation,
 		coalesce(sec.provider,'') AS securitylabelprovider,
-		coalesce(sec.label,'') AS securitylabel
+		coalesce(sec.label,'') AS securitylabel,
+		(a.attinhcount > 0) AS isinherited
 	FROM pg_catalog.pg_attribute a
 		JOIN pg_class c ON a.attrelid = c.oid
 		JOIN pg_namespace n ON c.relnamespace = n.oid

ci/gpbackup-generated.yml

@@ -12,7 +12,7 @@
 ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2023-05-23 09:49:00.253325
+## Generated by gen_pipeline.py at: 2023-07-17 15:50:51.409088
 ## Template file: gpbackup-tpl.yml
 ## Pipeline Name: gpbackup
 ## Nightly Trigger: True
@@ -255,13 +255,13 @@ resources:
 - name: ubuntu-debian-image
   type: registry-image
   source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu20.04-build
     tag: latest
 
 - name: ubuntu-debian-test-image
   type: registry-image
   source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu20.04-test
     tag: latest
 
 ##############################################
@@ -326,9 +326,6 @@ resources:
     branch: ((dp/ccp-git-branch))
     private_key: ((gp-concourse-cluster-provisioner-git-key))
     uri: ((dp/ccp-git-remote))
-  # issue with Ubuntu versioning is breaking our tests. pin for now
-  version:
-    ref: fe40dce2883df9d836f59f92950b93ec142ba8e5
 
 ##############################################
 
@@ -339,7 +336,7 @@ resources:
   source:
     bucket: ((dp/prod/gcs-ci-bucket))
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64((dp/prod/rc-build-type-gcs)).tar.gz
+    regexp: server/published/gpdb6/server-rc-(.*)-ubuntu20.04_x86_64((dp/prod/rc-build-type-gcs)).debug.tar.gz
 
 - name: bin_gpdb_7x_rhel8
   type: gcs
@@ -1067,7 +1064,7 @@ jobs:
       <<: *ccp_default_params
       vars:
         <<: *ccp_default_vars
-        PLATFORM: ubuntu18.04
+        PLATFORM: ubuntu20.04
         default_image_user: root
   - task: gen_cluster
     params:

ci/gpbackup-release-generated.yml

@@ -12,7 +12,7 @@
 ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2023-05-23 09:49:00.272419
+## Generated by gen_pipeline.py at: 2023-07-17 15:50:51.450573
 ## Template file: gpbackup-tpl.yml
 ## Pipeline Name: gpbackup-release
 ## Nightly Trigger: True
@@ -248,13 +248,13 @@ resources:
 - name: ubuntu-debian-image
   type: registry-image
   source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu20.04-build
     tag: latest
 
 - name: ubuntu-debian-test-image
   type: registry-image
   source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu20.04-test
     tag: latest
 
 ##############################################
@@ -325,9 +325,6 @@ resources:
     branch: ((dp/ccp-git-branch))
     private_key: ((gp-concourse-cluster-provisioner-git-key))
     uri: ((dp/ccp-git-remote))
-  # issue with Ubuntu versioning is breaking our tests. pin for now
-  version:
-    ref: fe40dce2883df9d836f59f92950b93ec142ba8e5
 
 ##############################################
 
@@ -338,7 +335,7 @@ resources:
   source:
     bucket: ((dp/prod/gcs-ci-bucket))
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64((dp/prod/rc-build-type-gcs)).tar.gz
+    regexp: server/published/gpdb6/server-rc-(.*)-ubuntu20.04_x86_64((dp/prod/rc-build-type-gcs)).debug.tar.gz
 
 - name: bin_gpdb_7x_rhel8
   type: gcs
@@ -1191,7 +1188,7 @@ jobs:
       <<: *ccp_default_params
       vars:
         <<: *ccp_default_vars
-        PLATFORM: ubuntu18.04
+        PLATFORM: ubuntu20.04
         default_image_user: root
   - task: gen_cluster
     params:

ci/pivnet_release/metadata.yml

@@ -50,7 +50,7 @@ file_groups:
       file_version: <TILE_RELEASE_VERSION>
     - file: file://greenplum_backup_restore-<TILE_RELEASE_VERSION>-gp6-ubuntu-amd64.gppkg
       description:
-      upload_as: Backup and Restore for GP 6 on Ubuntu 18.04
+      upload_as: Backup and Restore for GP 6 on Ubuntu 20.04
       file_type: Software
       file_version: <TILE_RELEASE_VERSION>
     - file: file://greenplum_backup_restore-<TILE_RELEASE_VERSION>-rhel8.tar.gz
@@ -70,7 +70,7 @@ file_groups:
       file_version: <TILE_RELEASE_VERSION>
     - file: file://greenplum_backup_restore-<TILE_RELEASE_VERSION>-ubuntu.tar.gz
       description:
-      upload_as: Backup and Restore for GP 6 on Ubuntu 18.04 compressed tarball
+      upload_as: Backup and Restore for GP 6 on Ubuntu 20.04 compressed tarball
       file_type: Software
       file_version: <TILE_RELEASE_VERSION>
 

ci/regression/regression_pipeline.yml

@@ -1,12 +1,26 @@
 # USAGE: fly -t dp set-pipeline  -p regression -c ~/workspace/gpbackup/ci/regression/regression_pipeline.yml -v gpbackup-git-branch=BRANCH_NAME
 ---
 groups:
-- name: Regression
+- name: all
+  jobs: 
+  - build_binaries
+  - build_gppkgs
+  - regdb-gpdb6
+  - regdb-gpdb7
+  - regdb-gpdb6-to-gpdb7-backup
+  - regdb-gpdb6-to-gpdb7-restore
+
+- name: regression
   jobs:
   - build_binaries
   - build_gppkgs
-  - regdb-GPDB6
-  - regdb-GPDB7
+  - regdb-gpdb6
+  - regdb-gpdb7
+
+- name: migration
+  jobs: 
+  - regdb-gpdb6-to-gpdb7-backup
+  - regdb-gpdb6-to-gpdb7-restore
 
 resource_types:
 - name: terraform
@@ -162,20 +176,28 @@ resources:
     json_key: ((dp/dev/gcp_svc_acct_key))
     regexp: open_source_license_VMware_Greenplum_Backup_and_Restore_(.*)_.*.txt
 
-- name: icw_dump_GPDB6
+- name: icw_dump_gpdb6
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-intermediates-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    versioned_file: 6X_STABLE_without_asserts/icw_planner_centos6_dump/dump.sql.xz
+    versioned_file: 6X_STABLE_without_asserts/icw_planner_rocky8_dump/dump.sql.xz
 
-- name: icw_dump_GPDB7
+- name: icw_dump_gpdb7
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-intermediates-prod
     json_key: ((concourse-gcs-resources-service-account-key))
     versioned_file: gpdb_main/icw_planner_rhel8_dump/dump.sql.xz
 
+- name: gpdb6-migration-backup
+  type: gcs
+  icon: google
+  source:
+    bucket: ((dp/dev/gcs-ci-bucket))
+    json_key: ((dp/dev/gcp_svc_acct_key))
+    versioned_file: gpbackup/intermediates/migration-backups/gpdb6/migration-backup.tar.gz
+
 - name: terraform
   type: terraform
   source:
@@ -350,19 +372,19 @@ jobs:
     file: gpbackup/ci/tasks/gpbackup-tools-versions.yml
   - in_parallel:
     - do: # RHEL8
-      - task: build-ddboost-RHEL8
+      - task: build-ddboost-rhel8
         image: rocky8-gpdb6-image
         file: gpbackup/ci/tasks/build-ddboost.yml
         input_mapping:
           bin_gpdb: bin_gpdb_6x_rhel8
-      - task: tar-binaries-RHEL8
+      - task: tar-binaries-rhel8
         image: rocky8-gpdb6-image
         file: gpbackup/ci/tasks/build-os-tars.yml
         input_mapping:
           gpbackup-go-components: gpbackup-go-components-rhel8
         output_mapping:
           gpbackup_tar: gpbackup_tar_rhel8
-      - task: build_gppkgs-RHEL8
+      - task: build_gppkgs-rhel8
         image: rocky8-gpdb6-image
         file: gpbackup/ci/tasks/build-gppkg.yml
         input_mapping:
@@ -394,7 +416,7 @@ jobs:
     params:
       file: gppkgs/gpbackup-gppkgs.tar.gz
 
-- name: regdb-GPDB6
+- name: regdb-gpdb6
   plan:
   - in_parallel:
     - get: weekly-trigger
@@ -411,7 +433,7 @@ jobs:
       resource: gpdb6_src
     - get: gppkgs
     - get: icw_dump
-      resource: icw_dump_GPDB6
+      resource: icw_dump_gpdb6
     - get: terraform.d
       params:
         unpack: true
@@ -442,7 +464,7 @@ jobs:
   on_failure:
     *slack_alert
 
-- name: regdb-GPDB7
+- name: regdb-gpdb7
   plan:
   - in_parallel:
     - get: weekly-trigger
@@ -460,7 +482,7 @@ jobs:
     - get: gppkgs
     - get: diffdb_src
     - get: icw_dump
-      resource: icw_dump_GPDB7
+      resource: icw_dump_gpdb7
     - get: terraform.d
       params:
         unpack: true
@@ -490,3 +512,53 @@ jobs:
     <<: *ccp_destroy_nvme
   on_failure:
     *slack_alert
+
+- name: regdb-gpdb6-to-gpdb7-backup
+  plan:
+  - in_parallel:
+    - get: weekly-trigger
+      trigger: true
+    - get: rocky8-gpdb6-image
+    - get: gpbackup
+      trigger: true
+      passed: [build_gppkgs]
+    - get: bin_gpdb_6x_rhel8
+      resource: 
+    - get: gpdb_src
+      resource: gpdb6_src
+    - get: gppkgs
+    - get: icw_dump
+      resource: icw_dump_gpdb6
+  - task: icw-migr-backup
+    image: rocky8-gpdb6-image
+    file: gpbackup/ci/tasks/icw-migr-backup.yml
+    input_mapping:
+      bin_gpdb: bin_gpdb_6x_rhel8
+  - put: gpdb6-migration-backup
+    params:
+        file: migration-artifacts/migration-backup.tar.gz
+  on_failure:
+    *slack_alert
+
+- name: regdb-gpdb6-to-gpdb7-restore
+  plan:
+  - in_parallel:
+    - get: rocky8-gpdb7-image
+    - get: gp-pkg
+    - get: gpbackup
+    - get: bin_gpdb_7x_rhel8
+      resource: 
+    - get: gpdb_src
+      resource: gpdb_main_src
+    - get: gppkgs
+    - get: migration-backup
+      trigger: true
+      resource: gpdb6-migration-backup 
+      passed: [regdb-gpdb6-to-gpdb7-backup]
+  - task: icw-migr-restore
+    image: rocky8-gpdb7-image
+    file: gpbackup/ci/tasks/icw-migr-restore.yml
+    input_mapping:
+      bin_gpdb: bin_gpdb_7x_rhel8
+  on_failure:
+    *slack_alert