Restore a backup in-place, for restoring full to update a db with another full db without downtime on data

.github/workflows/build.yaml

@@ -42,14 +42,14 @@ jobs:
 #          key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
 
       - name: Install golang dependencies
-        run: go mod download -x
+        run: go mod download
 #        if: |
 #          steps.cache-golang.outputs.cache-hit != 'true'
 
       - name: Build clickhouse-backup binary
         id: make-race
         env:
-          GOROOT: ${{ env.GOROOT_1_24_X64 }}
+          GOROOT: ${{ env.GOROOT_1_25_X64 }}
         run: |
           make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup
           make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips 
@@ -125,8 +125,7 @@ jobs:
 
       - name: Install python venv
         run: |
-          set -x
-          (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv)
+          (dpkg -l | grep venv) > /dev/null || (apt-get update && apt-get install -y python3-venv)
           python3 -m venv ~/venv/qa
 
       - name: Cache python
@@ -138,7 +137,6 @@ jobs:
 
       - name: Install python dependencies
         run: |
-          set -x
           ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt
         if: |
           steps.cache-python.outputs.cache-hit != 'true'
@@ -155,10 +153,10 @@ jobs:
           # don't change it to avoid not working CI/CD
           RUN_TESTS: "*"
         run: |
-          set -xe
+          set -e
           export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup
 
-          docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull
+          docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull --quiet
 
           chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
           source ~/venv/qa/bin/activate
@@ -170,7 +168,7 @@ jobs:
           sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances
       - name: Format testflows coverage
         env:
-          GOROOT: ${{ env.GOROOT_1_24_X64 }}
+          GOROOT: ${{ env.GOROOT_1_25_X64 }}
         run: |
           sudo chmod -Rv a+rw test/testflows/_coverage_/
           ls -la test/testflows/_coverage_
@@ -252,7 +250,7 @@ jobs:
       - name: Running integration tests
         env:
           RUN_PARALLEL: 4
-          GOROOT: ${{ env.GOROOT_1_24_X64 }}
+          GOROOT: ${{ env.GOROOT_1_25_X64 }}
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
           # options for advanced debug CI/CD
           # RUN_TESTS: "^TestSkipDisk$"
@@ -277,9 +275,11 @@ jobs:
           QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }}
           QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }}
         run: |
-          set -xe
+          set -e
           echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}"
           echo "GCS_TESTS=${GCS_TESTS}"
+          echo "Go version: $(go version)"
+          echo "Testing ClickHouse ${CLICKHOUSE_VERSION}"
 
           chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
 
@@ -302,7 +302,7 @@ jobs:
           pids=()
           project_ids=()
           for ((i = 0; i < RUN_PARALLEL; i++)); do
-            docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d &
+            docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress=quiet up -d &
             pids+=($!)
             project_ids+=("project${i}")
           done
@@ -315,9 +315,9 @@ jobs:
                 echo "$pid docker compose up successful"
             else
                 echo "=== docker ${project_id} state ==="
-                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet ps -a
                 echo "=== docker ${project_id} logs ==="
-                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet logs
                 echo "$pid the docker compose up failed."
                 exit 1  # Exit with an error code if any command fails
             fi
@@ -329,16 +329,16 @@ jobs:
           if [[ "0" != "${TEST_FAILED}" ]]; then
             for project_id in "${project_ids[@]}"; do
                 echo "=== docker ${project_id} state ==="
-                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet ps -a
                 echo "=== docker ${project_id} logs ==="
-                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
+                docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet logs
             done
             exit 1
           fi      
 
       - name: Format integration coverage
         env:
-          GOROOT: ${{ env.GOROOT_1_24_X64 }}
+          GOROOT: ${{ env.GOROOT_1_25_X64 }}
         run: |
           sudo chmod -Rv a+rw test/integration/_coverage_/
           ls -la test/integration/_coverage_ 

ChangeLog.md

@@ -11,6 +11,7 @@ IMPROVEMENTS
 BUG FIXES
 - fix restore schema on cluster for VIEW, fix [1199](https://github.com/Altinity/clickhouse-backup/issues/1199).
 - disable free space check when using `--hardlink-exists-files`, fix [1198](https://github.com/Altinity/clickhouse-backup/issues/1198)
+- add chmod 0640 for `--hardlink-exists-files` during `download` and `restore_remote`, fix [1164](https://github.com/Altinity/clickhouse-backup/issues/1164)
 
 # v2.6.33
 BUG FIXES
@@ -614,7 +615,7 @@ BUG FIXES
 - fixed restore for object disk frozen_metadata.txt, fix [752](https://github.com/Altinity/clickhouse-backup/issues/752)
 - fixed more corner cases for `check_parts_columns: true`, fix [747](https://github.com/Altinity/clickhouse-backup/issues/747)
 - fixed applying macros to s3 endpoint in object disk during restoring embedded backups, fix [750](https://github.com/Altinity/clickhouse-backup/issues/750)
-- rewrote `GCS` clients pool, set default `GCS_CLIENT_POOL_SIZE` as `max(upload_concurrency, download_concurrency) * 3` to avoid stuck, fix [753](https://github.com/Altinity/clickhouse-backup/pull/753), thanks @minguyen-jumptrading
+- rewrote `GCS` clients pool, set default `GCS_CLIENT_POOL_SIZE` as `max(upload_concurrency, download_concurrency) * 3` to avoid stuck, fix [753](https://github.com/Altinity/clickhouse-backup/pull/753), thanks @minguyen9988
 
 # v2.4.1
 IMPROVEMENTS

cmd/clickhouse-backup/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	stdlog "log"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -437,6 +438,10 @@ func main() {
 			UsageText: "clickhouse-backup restore  [-t, --tables=<db>.<table>] [-m, --restore-database-mapping=<originDB>:<targetDB>[,<...>]] [--tm, --restore-table-mapping=<originTable>:<targetTable>[,<...>]] [--partitions=<partitions_names>] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--named-collections] [--resume] <backup_name>",
 			Action: func(c *cli.Context) error {
 				b := backup.NewBackuper(config.GetConfigFromCli(c))
+				// Override config with CLI flag if provided
+				if c.Bool("restore-in-place") {
+					b.SetRestoreInPlace(true)
+				}
 				return b.Restore(c.Args().First(), c.String("tables"), c.StringSlice("restore-database-mapping"), c.StringSlice("restore-table-mapping"), c.StringSlice("partitions"), c.StringSlice("skip-projections"), c.Bool("schema"), c.Bool("data"), c.Bool("drop"), c.Bool("ignore-dependencies"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("named-collections"), c.Bool("named-collections-only"), c.Bool("resume"), c.Bool("restore-schema-as-attach"), c.Bool("replicated-copy-to-detached"), version, c.Int("command-id"))
 			},
 			Flags: append(cliapp.Flags,
@@ -536,6 +541,11 @@ func main() {
 					Hidden: false,
 					Usage:  "Copy data to detached folder for Replicated*MergeTree tables but skip ATTACH PART step",
 				},
+				cli.BoolFlag{
+					Name:   "restore-in-place",
+					Hidden: false,
+					Usage:  "Perform in-place restore by comparing backup parts with current database parts. Only downloads differential parts instead of full restore. Requires --data flag.",
+				},
 			),
 		},
 		{
@@ -544,7 +554,21 @@ func main() {
 			UsageText: "clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=<db>.<table>] [-m, --restore-database-mapping=<originDB>:<targetDB>[,<...>]] [--tm, --restore-table-mapping=<originTable>:<targetTable>[,<...>]] [--partitions=<partitions_names>] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--named-collections] [--resumable] <backup_name>",
 			Action: func(c *cli.Context) error {
 				b := backup.NewBackuper(config.GetConfigFromCli(c))
-				return b.RestoreFromRemote(c.Args().First(), c.String("tables"), c.StringSlice("restore-database-mapping"), c.StringSlice("restore-table-mapping"), c.StringSlice("partitions"), c.StringSlice("skip-projections"), c.Bool("schema"), c.Bool("d"), c.Bool("rm"), c.Bool("i"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("named-collections"), c.Bool("named-collections-only"), c.Bool("resume"), c.Bool("restore-schema-as-attach"), c.Bool("replicated-copy-to-detached"), c.Bool("hardlink-exists-files"), version, c.Int("command-id"))
+				// Override config with CLI flag if provided
+				if c.Bool("restore-in-place") {
+					b.SetRestoreInPlace(true)
+				}
+				// CI/CD Diagnostic: Log function signature validation for Go 1.25 + ClickHouse 23.8 compatibility
+				log.Debug().Fields(map[string]interface{}{
+					"operation":               "restore_remote_cli_validation",
+					"go_version":             runtime.Version(),
+					"hardlink_exists_files":  c.Bool("hardlink-exists-files"),
+					"drop_if_schema_changed": c.Bool("drop-if-schema-changed"),
+					"function_signature":     "RestoreFromRemote",
+					"parameter_count":        "expected_22_parameters",
+					"issue_diagnosis":        "hardcoded_false_should_be_cli_flag",
+				}).Msg("diagnosing CI Build/Test (1.25, 23.8) function signature mismatch")
+				return b.RestoreFromRemote(c.Args().First(), c.String("tables"), c.StringSlice("restore-database-mapping"), c.StringSlice("restore-table-mapping"), c.StringSlice("partitions"), c.StringSlice("skip-projections"), c.Bool("schema"), c.Bool("d"), c.Bool("rm"), c.Bool("i"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("named-collections"), c.Bool("named-collections-only"), c.Bool("resume"), c.Bool("restore-schema-as-attach"), c.Bool("replicated-copy-to-detached"), c.Bool("hardlink-exists-files"), c.Bool("drop-if-schema-changed"), version, c.Int("command-id"))
 			},
 			Flags: append(cliapp.Flags,
 				cli.StringFlag{
@@ -643,6 +667,16 @@ func main() {
 					Hidden: false,
 					Usage:  "Create hardlinks for existing files instead of downloading",
 				},
+				cli.BoolFlag{
+					Name:   "restore-in-place",
+					Hidden: false,
+					Usage:  "Perform in-place restore by comparing backup parts with current database parts. Only downloads differential parts instead of full restore. Requires --data flag.",
+				},
+				cli.BoolFlag{
+					Name:   "drop-if-schema-changed",
+					Hidden: false,
+					Usage:  "Drop and recreate tables when schema changes are detected during in-place restore. Only available with --restore-in-place flag.",
+				},
 			),
 		},
 		{

pkg/backup/backuper.go

@@ -6,16 +6,17 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"github.com/Altinity/clickhouse-backup/v2/pkg/common"
-	"github.com/Altinity/clickhouse-backup/v2/pkg/metadata"
-	"github.com/Altinity/clickhouse-backup/v2/pkg/utils"
-	"github.com/eapache/go-resiliency/retrier"
 	"net/url"
 	"os"
 	"path"
 	"regexp"
 	"strings"
 
+	"github.com/Altinity/clickhouse-backup/v2/pkg/common"
+	"github.com/Altinity/clickhouse-backup/v2/pkg/metadata"
+	"github.com/Altinity/clickhouse-backup/v2/pkg/utils"
+	"github.com/eapache/go-resiliency/retrier"
+
 	"github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse"
 	"github.com/Altinity/clickhouse-backup/v2/pkg/config"
 	"github.com/Altinity/clickhouse-backup/v2/pkg/resumable"
@@ -64,6 +65,11 @@ func NewBackuper(cfg *config.Config, opts ...BackuperOpt) *Backuper {
 	return b
 }
 
+// SetRestoreInPlace sets the RestoreInPlace flag in the configuration
+func (b *Backuper) SetRestoreInPlace(value bool) {
+	b.cfg.General.RestoreInPlace = value
+}
+
 // Classify need to log retries
 func (b *Backuper) Classify(err error) retrier.Action {
 	if err == nil {

pkg/backup/create.go

@@ -303,6 +303,10 @@ func (b *Backuper) createBackupLocal(ctx context.Context, backupName, diffFromRe
 
 	var backupDataSize, backupObjectDiskSize, backupMetadataSize uint64
 	var metaMutex sync.Mutex
+
+	// Note: The Parts field in metadata will contain the parts that were backed up,
+	// which represents the database state at backup time for in-place restore planning
+
 	createBackupWorkingGroup, createCtx := errgroup.WithContext(ctx)
 	createBackupWorkingGroup.SetLimit(max(b.cfg.ClickHouse.MaxConnections, 1))
 
@@ -320,6 +324,7 @@ func (b *Backuper) createBackupLocal(ctx context.Context, backupName, diffFromRe
 			var disksToPartsMap map[string][]metadata.Part
 			var checksums map[string]uint64
 			var addTableToBackupErr error
+
 			if doBackupData && table.BackupType == clickhouse.ShardBackupFull {
 				logger.Debug().Msg("begin data backup")
 				shadowBackupUUID := strings.ReplaceAll(uuid.New().String(), "-", "")

pkg/backup/download.go

@@ -1288,6 +1288,9 @@ func (b *Backuper) makePartHardlinks(exists, new string) error {
 				return err
 			}
 		}
+		if err = os.Chmod(newF, 0640); err != nil {
+			return err
+		}
 		return nil
 	}); walkErr != nil {
 		log.Warn().Msgf("Link recursively %s -> %s return error: %v", new, exists, walkErr)