-
Notifications
You must be signed in to change notification settings - Fork 11
/
Copy pathgenerate
241 lines (190 loc) · 9.96 KB
/
generate
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
#!/usr/bin/env bash
#
# Backups one or more sites.
#
# `captaincore backup generate <site>`
#
# [--skip-remote]
# Skips saving to Restic backup repo
#
# [--skip-db]
# Skips database backup
#
if [ ${#@} -ne 1 ]; then
echo -e "${COLOR_RED}Error:${COLOR_NORMAL} Requires a <site>"
exit
fi
while read config; do
if [[ "$config" == "Error:"* ]]; then
continue
fi
declare "$config"
done <<< "$(php ${CAPTAINCORE_PATH}/lib/local-scripts/configs.php fetch)"
site=$1
run_command() {
runtime_start=$( date +%s )
if [[ $SKIP_REMOTE == true ]]; then
echo "Skipping remote"
fi
# Extract environment
if [[ "$site" == *"-staging"* ]]; then
environment=staging
fi
if [[ "$site" == *"-production"* ]]; then
environment=production
fi
if [[ "$site" != *"-"* ]]; then
environment=production
fi
# Load site configs
while read site_configs; do if [[ $site_configs == "" ]]; then continue; fi; declare "$site_configs"; done <<< "$(captaincore site get $site --bash --captain-id=$CAPTAIN_ID)"
# Site found, start the backup
if [[ $domain == "" ]]; then
echo "Error: $site missing domain. Skipping backup."
exit
fi
# Append trailing slash if home_directory exist
if [ "$home_directory" != "" ]; then
home_directory="${home_directory}/"
fi
# Define Rclone config file
rclone_config_file="$path/${site}_${site_id}/rclone.conf"
if [ ! -f "$rclone_config_file" ]; then
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
fi
# Lookup rclone
remote_check=$( rclone config show $environment --config="$rclone_config_file" )
remote_backup_check=$( rclone config show backup --config="$rclone_config_file" )
if [[ $remote_check == *"Couldn't find type of fs"* || $remote_backup_check == *"Couldn't find type of fs"* ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Generating rclone configs for $site"
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
fi
# Captures FTP errors in $ftp_output and file listing to log file
ftp_output=$( { rclone lsd ${environment}:$home_directory --config="$rclone_config_file" ; } 2>&1 )
ftp_search_for_wordpress=$( echo "$ftp_output" | perl -wnE'say for /wp-admin/g' )
# Handle FTP errors
if [[ $ftp_search_for_wordpress != "wp-admin"* ]]; then
wordpress_not_found=true
fi
if [[ "$backup_mode" == "direct" ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Begin incremental backup ${site}-${environment} to Rclone"
if [[ $SKIP_DB != true ]] && [[ $wordpress_not_found != true ]]; then
# Database backup
captaincore ssh ${site}-${environment} --script="db-backup" --captain-id=$CAPTAIN_ID
if [[ "$provider" == "wpengine" ]]; then
rclone sync ${environment}:_wpeprivate/database-backup.sql backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file" --fast-list
fi
if [[ "$provider" == "kinsta" ]]; then
rclone sync ${environment}:private/database-backup.sql backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file" --fast-list
fi
if [[ "$provider" == "rocketdotnet" ]]; then
rclone sync ${environment}:tmp/database-backup.sql backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file" --fast-list
fi
fi
# Backup WordPress site to Rclone backup
if [[ "$wp_content" != "wp-content" ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Backing up ${site}-${environment} alternative wp-content location ($wp_content)"
rclone sync ${environment}:$home_directory backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="+ $wp_content/**" --filter="- wp-content/**" --filter="- content/**" --fast-list --config="$rclone_config_file"
else
rclone sync ${environment}:$home_directory backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="- content/**" --fast-list --config="$rclone_config_file"
fi
# attempt to unmount
if [[ "$OSTYPE" == "linux-gnu" ]]; then
fusermount -u $path/${site}_${site_id}/${environment}/backup/
elif [[ "$OSTYPE" == "darwin"* ]]; then
umount $path/${site}_${site_id}/${environment}/backup/
fi
mkdir -p $path/${site}_${site_id}/${environment}/backup/
cd $path/${site}_${site_id}/${environment}/
# if files found, purge backup
if [[ "$( ls backup )" != "" ]]; then
rm -rf backup/
mkdir backup/
fi
echo "Mounting ${environment} to backup/"
rclone mount backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ backup --config="$rclone_config_file" --daemon --read-only --transfers 40
echo "rclone mount backup:${rclone_backup#*:}/${site}_${site_id}/${environment}/backup/ backup --config="$rclone_config_file" --daemon --read-only --transfers 40"
echo "changing to backup/"
cd "backup"
mount_point=$(pwd)
if [[ "$mount_point/" != "$path/${site}_${site_id}/${environment}/backup/" ]]; then
cd $path/${site}_${site_id}/${environment}/
if [[ "$OSTYPE" == "linux-gnu" ]]; then
fusermount -u backup/
elif [[ "$OSTYPE" == "darwin"* ]]; then
umount backup/
fi
echo "Backup cancel. Rclone mounting error."
exit
fi
if [[ $( restic snapshots --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key" ) == "" ]]; then
echo "Generating restic repo for $site"
restic init --quiet --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key"
fi
if [[ "$wp_content" != "wp-content" ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Backing up ${site}-${environment} alternative wp-content location ($wp_content)"
restic backup . --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key" --exclude="wp-content/" --exclude="content/*" --exclude="!$wp_content" --exclude-file="${CAPTAINCORE_PATH}/lib/restic-excludes" --no-scan --ignore-inode --read-concurrency=3
else
restic backup . --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key" --exclude-file="${CAPTAINCORE_PATH}/lib/restic-excludes" --no-scan --ignore-inode --read-concurrency=3
fi
cd $path/${site}_${site_id}/${environment}/
sleep 7
if [[ "$OSTYPE" == "linux-gnu" ]]; then
fusermount -u backup/
elif [[ "$OSTYPE" == "darwin"* ]]; then
umount backup/
fi
captaincore backup list-generate ${site}-${environment} --captain-id=$CAPTAIN_ID
captaincore backup get-generate ${site}-${environment} --captain-id=$CAPTAIN_ID
cd $path/${site}_${site_id}/${environment}/backups/
runtime_end=$( date +%s )
echo "$runtime_start $runtime_end" >> runtime
exit
fi
# Incremental backup locally with rclone
echo "$(date +'%Y-%m-%d %H:%M') Begin incremental backup ${site}-${environment} to local"
if [[ $SKIP_DB != true ]] && [[ $wordpress_not_found != true ]]; then
# Database backup
captaincore ssh ${site}-${environment} --script="db-backup" --captain-id=$CAPTAIN_ID
if [[ "$provider" == "wpengine" ]]; then
rclone sync ${environment}:_wpeprivate/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
fi
if [[ "$provider" == "kinsta" ]]; then
rclone sync ${environment}:private/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
fi
if [[ "$provider" == "rocketdotnet" ]]; then
rclone sync ${environment}:tmp/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
fi
fi
mkdir -p $path/${site}_${site_id}/${environment}/backup/
# Backup site locally
if [[ "$wp_content" != "wp-content" ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Backing up ${site}-${environment} alternative wp-content location ($wp_content)"
rclone sync ${environment}:$home_directory $path/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="+ $wp_content/**" --filter="- wp-content/**" --filter="- content/**" --config="$rclone_config_file"
else
rclone sync ${environment}:$home_directory $path/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="- content/**" --config="$rclone_config_file"
fi
# Incremental backup upload to Restic
if [[ $SKIP_REMOTE != true ]]; then
echo "$(date +'%Y-%m-%d %H:%M') Storing $site to backup archive"
if [[ $( restic snapshots --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key" ) == "" ]]; then
echo "Generating restic repo for $site"
restic init --quiet --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key"
fi
cd $path/${site}_${site_id}/${environment}/backup/
restic backup . --quiet --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --exclude-file="${CAPTAINCORE_PATH}/lib/restic-excludes" --password-file="${CAPTAINCORE_PATH}/data/restic.key"
captaincore backup list-generate ${site}-${environment} --captain-id=$CAPTAIN_ID
captaincore backup get-generate ${site}-${environment} --captain-id=$CAPTAIN_ID
fi
captaincore usage-update ${site}-${environment} --captain-id=$CAPTAIN_ID
if [ -f "${path}/process-${process_id}-progress.log" ]; then
echo -n "." >> ${path}/process-${process_id}-progress.log
fi
if [ ! -d "$path/${site}_${site_id}/${environment}/backups/" ]; then
mkdir -p "$path/${site}_${site_id}/${environment}/backups"
fi
cd $path/${site}_${site_id}/${environment}/backups/
runtime_end=$( date +%s )
echo "$runtime_start $runtime_end" >> runtime
}
run_command