Compress dump files to reduce size fo the files

Author: Rub21

images/backup-restore/start.sh

@@ -35,21 +35,22 @@ cloudStorageOps() {
 }
 
 backupDB() {
-	local LOCAL_BACKUP_FILE=${BACKUP_CLOUD_FILE}.dump
-	local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.dump"
-	if [ "$SET_DATE_AT_NAME" == "true" ]; then
-		local CURRENT_DATE=$(date '+%Y%m%d-%H%M')
-		LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump"
-		CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump"
-	fi
+    local LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}.dump"
+    local LOCAL_BACKUP_FILE_GZIP="${BACKUP_CLOUD_FILE}.dump.gz"
+    local CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}.dump.gz"
 
-	# Backup database with max compression
-	echo "Backing up DB ${POSTGRES_DB} into ${LOCAL_BACKUP_FILE}"
-	# pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} ${POSTGRES_DB} | gzip -9 >${LOCAL_BACKUP_FILE}
-	pg_dump -h ${POSTGRES_HOST} -U ${POSTGRES_USER} -Fc -f ${LOCAL_BACKUP_FILE} ${POSTGRES_DB}
+    if [ "$SET_DATE_AT_NAME" == "true" ]; then
+        local CURRENT_DATE
+        CURRENT_DATE=$(date '+%Y%m%d-%H%M')
+        LOCAL_BACKUP_FILE="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump"
+        LOCAL_BACKUP_FILE_GZIP="${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump.gz"
+        CLOUD_BACKUP_FILE="${BACKUP_CLOUD_FOLDER}/${BACKUP_CLOUD_FILE}-${CURRENT_DATE}.dump.gz"
+    fi
 
-	# Handle cloud storage based on the provider
-	cloudStorageOps "${LOCAL_BACKUP_FILE}" "${CLOUD_BACKUP_FILE}"
+    # Backup database with pg_dump custom format (-Fc) + gzip
+    echo "Backing up DB ${POSTGRES_DB} into ${LOCAL_BACKUP_FILE_GZIP}"
+    pg_dump -h "${POSTGRES_HOST}" -U "${POSTGRES_USER}" -Fc "${POSTGRES_DB}" | gzip -9 > "${LOCAL_BACKUP_FILE}.gz"
+    cloudStorageOps "${LOCAL_BACKUP_FILE_GZIP}" "${CLOUD_BACKUP_FILE}"
 }
 
 restoreDB() {

images/planet-dump/start.sh

@@ -23,23 +23,50 @@ if [ "$OVERWRITE_PLANET_FILE" == "true" ]; then
 	cloud_planetPBFFile=planet/planet-latest.osm.pbf
 fi
 
+# ===============================
+# Download db .dump file 
+# ===============================
 # ===============================
 # Download db .dump file 
 # ===============================
 download_dump_file() {
-	echo "Downloading db .dump file from cloud..."
-	if [ "$CLOUDPROVIDER" == "aws" ]; then
-		if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then
-			temp_txt="$VOLUME_DIR/tmp_dump_url.txt"
-			aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt"
-			first_line=$(head -n 1 "$temp_txt")
-			aws s3 cp "$first_line" "$dumpFile"
-		else
-			aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile"
-		fi
-	elif [ "$CLOUDPROVIDER" == "gcp" ]; then
-		gsutil cp "$DUMP_CLOUD_URL" "$dumpFile"
-	fi
+    echo "Downloading db .dump file from cloud..."
+    if [ "$CLOUDPROVIDER" == "aws" ]; then
+        if [[ "$DUMP_CLOUD_URL" == *.txt ]]; then
+            # Download the .txt file containing the URL
+            temp_txt="$VOLUME_DIR/tmp_dump_url.txt"
+            aws s3 cp "$DUMP_CLOUD_URL" "$temp_txt"
+
+            # Get the first line (S3 URL to the .dump or .dump.gz file)
+            first_line=$(head -n 1 "$temp_txt")
+            echo "Found dump URL in txt: $first_line"
+
+            aws s3 cp "$first_line" "$dumpFile"
+
+            # Check if it's compressed (.gz) and decompress
+            if [[ "$first_line" == *.gz ]]; then
+                echo "Decompressing gzip file..."
+                gunzip -f "$dumpFile"
+                dumpFile="${dumpFile%.gz}"
+            fi
+        else
+            aws s3 cp "$DUMP_CLOUD_URL" "$dumpFile"
+            # If it's compressed, decompress
+            if [[ "$DUMP_CLOUD_URL" == *.gz ]]; then
+                echo "Decompressing gzip file..."
+                gunzip -f "$dumpFile"
+                dumpFile="${dumpFile%.gz}"
+            fi
+        fi
+
+    elif [ "$CLOUDPROVIDER" == "gcp" ]; then
+        gsutil cp "$DUMP_CLOUD_URL" "$dumpFile"
+    else
+        echo "Unsupported CLOUDPROVIDER: $CLOUDPROVIDER"
+        exit 1
+    fi
+
+    echo "Dump file ready at: $dumpFile"
 }