1
|
|
|
<?php |
2
|
|
|
use League\Flysystem\Config; |
3
|
|
|
use League\Flysystem\Filesystem; |
4
|
|
|
use League\Flysystem\Util; |
5
|
|
|
use League\Flysystem\Adapter\Local; |
6
|
|
|
|
7
|
|
|
class Xcloner_File_System{ |
|
|
|
|
8
|
|
|
|
9
|
|
|
private $excluded_files = ""; |
10
|
|
|
private $excluded_files_by_default = array("administrator/backups", "wp-content/backups"); |
11
|
|
|
private $included_files_handler = "backup_files.csv"; |
12
|
|
|
private $temp_dir_handler = ".dir"; |
13
|
|
|
public $filesystem; |
14
|
|
|
public $tmp_filesystem; |
15
|
|
|
public $storage_filesystem; |
16
|
|
|
private $xcloner_settings_append; |
|
|
|
|
17
|
|
|
private $xcloner_container; |
18
|
|
|
private $diff_timestamp_start = ""; |
19
|
|
|
|
20
|
|
|
private $logger; |
21
|
|
|
private $start_adapter; |
22
|
|
|
private $tmp_adapter; |
23
|
|
|
private $storage_adapter; |
24
|
|
|
private $xcloner_requirements; |
|
|
|
|
25
|
|
|
private $xcloner_settings; |
26
|
|
|
private $start_filesystem; |
27
|
|
|
private $tmp_filesystem_append; |
28
|
|
|
private $storage_filesystem_append; |
29
|
|
|
|
30
|
|
|
private $files_counter; |
31
|
|
|
private $files_size; |
32
|
|
|
private $last_logged_file; |
33
|
|
|
private $folders_to_process_per_session = 25; |
34
|
|
|
private $backup_archive_extensions = array("tar", "tgz", "tar.gz", "gz", "csv"); |
35
|
|
|
private $backup_name_tags = array('[time]', '[hostname]', '[domain]'); |
36
|
|
|
|
37
|
|
|
public function __construct(Xcloner $xcloner_container, $hash = "") |
|
|
|
|
38
|
|
|
{ |
39
|
|
|
$this->xcloner_container = $xcloner_container; |
40
|
|
|
|
41
|
|
|
$this->logger = $xcloner_container->get_xcloner_logger()->withName("xcloner_file_system"); |
42
|
|
|
$this->xcloner_settings = $xcloner_container->get_xcloner_settings(); |
43
|
|
|
|
44
|
|
|
try{ |
45
|
|
|
|
46
|
|
|
$this->start_adapter = new Local($this->xcloner_settings->get_xcloner_start_path(),LOCK_EX, 'SKIP_LINKS'); |
47
|
|
|
$this->start_filesystem = new Filesystem($this->start_adapter, new Config([ |
48
|
|
|
'disable_asserts' => true, |
49
|
|
|
])); |
50
|
|
|
|
51
|
|
|
$this->tmp_adapter = new Local($this->xcloner_settings->get_xcloner_tmp_path(),LOCK_EX, 'SKIP_LINKS'); |
52
|
|
|
$this->tmp_filesystem = new Filesystem($this->tmp_adapter, new Config([ |
53
|
|
|
'disable_asserts' => true, |
54
|
|
|
])); |
55
|
|
|
$adapter = new Local($this->xcloner_settings->get_xcloner_tmp_path(),LOCK_EX|FILE_APPEND, 'SKIP_LINKS'); |
56
|
|
|
$this->tmp_filesystem_append = new Filesystem($adapter, new Config([ |
57
|
|
|
'disable_asserts' => true, |
58
|
|
|
])); |
59
|
|
|
|
60
|
|
|
$adapter = new Local($this->xcloner_settings->get_xcloner_store_path(),LOCK_EX, 'SKIP_LINKS'); |
61
|
|
|
$this->storage_filesystem = new Filesystem($adapter, new Config([ |
62
|
|
|
'disable_asserts' => true, |
63
|
|
|
])); |
64
|
|
|
|
65
|
|
|
$this->storage_adapter = new Local($this->xcloner_settings->get_xcloner_store_path(),FILE_APPEND, 'SKIP_LINKS'); |
66
|
|
|
$this->storage_filesystem_append = new Filesystem($this->storage_adapter, new Config([ |
67
|
|
|
'disable_asserts' => true, |
68
|
|
|
])); |
69
|
|
|
}catch(Exception $e){ |
70
|
|
|
$this->logger->error("Filesystem Initialization Error: ".$e->getMessage()); |
71
|
|
|
} |
72
|
|
|
|
73
|
|
|
|
74
|
|
|
if($value = get_option('xcloner_directories_to_scan_per_request')) |
75
|
|
|
$this->folders_to_process_per_session = $value; |
76
|
|
|
|
77
|
|
|
} |
78
|
|
|
|
79
|
|
|
public function set_diff_timestamp_start($timestamp = "") |
80
|
|
|
{ |
81
|
|
|
if($timestamp) |
82
|
|
|
{ |
83
|
|
|
$this->logger->info(sprintf("Setting Differential Timestamp To %s", date("Y-m-d", $timestamp)), array("FILESYSTEM", "DIFF")); |
84
|
|
|
$this->diff_timestamp_start = $timestamp; |
85
|
|
|
} |
86
|
|
|
} |
87
|
|
|
|
88
|
|
|
public function get_diff_timestamp_start() |
89
|
|
|
{ |
90
|
|
|
return $this->diff_timestamp_start; |
91
|
|
|
} |
92
|
|
|
|
93
|
|
|
private function get_xcloner_container() |
94
|
|
|
{ |
95
|
|
|
return $this->xcloner_container; |
96
|
|
|
} |
97
|
|
|
|
98
|
|
|
|
99
|
|
|
public function set_hash($hash) |
100
|
|
|
{ |
101
|
|
|
$this->xcloner_settings->set_hash($hash); |
102
|
|
|
} |
103
|
|
|
|
104
|
|
|
public function get_hash($hash) |
|
|
|
|
105
|
|
|
{ |
106
|
|
|
$this->xcloner_settings->get_hash(); |
107
|
|
|
} |
108
|
|
|
|
109
|
|
|
public function get_tmp_filesystem() |
110
|
|
|
{ |
111
|
|
|
return $this->tmp_filesystem; |
112
|
|
|
} |
113
|
|
|
|
114
|
|
|
public function get_storage_filesystem($remote_storage_selection = "") |
115
|
|
|
{ |
116
|
|
|
if($remote_storage_selection != "") |
117
|
|
|
{ |
118
|
|
|
$remote_storage = $this->get_xcloner_container()->get_xcloner_remote_storage(); |
119
|
|
|
$method = "get_".$remote_storage_selection."_filesystem"; |
120
|
|
|
|
121
|
|
|
if(!method_exists($remote_storage, $method)) |
122
|
|
|
return false; |
123
|
|
|
|
124
|
|
|
list($adapter, $filesystem) = $remote_storage->$method(); |
|
|
|
|
125
|
|
|
|
126
|
|
|
return $filesystem; |
127
|
|
|
} |
128
|
|
|
|
129
|
|
|
return $this->storage_filesystem; |
130
|
|
|
} |
131
|
|
|
|
132
|
|
|
public function get_tmp_filesystem_adapter() |
133
|
|
|
{ |
134
|
|
|
return $this->tmp_adapter; |
135
|
|
|
} |
136
|
|
|
|
137
|
|
|
public function get_tmp_filesystem_append() |
138
|
|
|
{ |
139
|
|
|
return $this->tmp_filesystem_append; |
140
|
|
|
} |
141
|
|
|
|
142
|
|
|
public function get_start_adapter() |
143
|
|
|
{ |
144
|
|
|
return $this->start_adapter; |
145
|
|
|
} |
146
|
|
|
|
147
|
|
|
public function get_start_filesystem() |
148
|
|
|
{ |
149
|
|
|
return $this->start_filesystem; |
150
|
|
|
} |
151
|
|
|
|
152
|
|
|
public function get_logger() |
153
|
|
|
{ |
154
|
|
|
return $this->logger; |
155
|
|
|
} |
156
|
|
|
|
157
|
|
|
public function get_start_path_file_info($file) |
158
|
|
|
{ |
159
|
|
|
$info= $this->getMetadataFull('start_adapter', $file); |
160
|
|
|
return $this->start_filesystem->normalizeFileInfo($info); |
161
|
|
|
} |
162
|
|
|
|
163
|
|
|
public function get_storage_path_file_info($file) |
164
|
|
|
{ |
165
|
|
|
return $this->getMetadataFull('storage_adapter', $file); |
166
|
|
|
} |
167
|
|
|
|
168
|
|
|
public function get_included_files_handler($metadata = 0) |
169
|
|
|
{ |
170
|
|
|
$path = $this->included_files_handler; |
171
|
|
|
if(!$metadata) |
172
|
|
|
return $path; |
173
|
|
|
|
174
|
|
|
$spl_info = $this->getMetadataFull('tmp_adapter', $path); |
175
|
|
|
return $spl_info; |
176
|
|
|
|
177
|
|
|
} |
178
|
|
|
|
179
|
|
|
public function get_temp_dir_handler() |
180
|
|
|
{ |
181
|
|
|
return $this->temp_dir_handler; |
182
|
|
|
} |
183
|
|
|
|
184
|
|
|
public function get_latest_backup() |
185
|
|
|
{ |
186
|
|
|
$files = $this->get_backup_archives_list(); |
187
|
|
|
|
188
|
|
|
if(is_array($files)) |
189
|
|
|
$this->sort_by($files, "timestamp","desc"); |
190
|
|
|
|
191
|
|
|
$new_list = array(); |
192
|
|
|
|
193
|
|
View Code Duplication |
foreach($files as $key=>$file) |
|
|
|
|
194
|
|
|
if(!isset($file['parent'])) |
195
|
|
|
$new_list[] = ($files[$key]); |
196
|
|
|
|
197
|
|
|
if(isset($new_list[0])) |
198
|
|
|
return $new_list[0]; |
199
|
|
|
} |
200
|
|
|
|
201
|
|
|
public function get_latest_backups() |
202
|
|
|
{ |
203
|
|
|
$files = $this->get_backup_archives_list(); |
204
|
|
|
|
205
|
|
|
if(is_array($files)) |
206
|
|
|
$this->sort_by($files, "timestamp","desc"); |
207
|
|
|
|
208
|
|
|
$new_list = array(); |
209
|
|
|
|
210
|
|
View Code Duplication |
foreach($files as $key=>$file) |
|
|
|
|
211
|
|
|
if(!isset($file['parent'])) |
212
|
|
|
$new_list[] = ($files[$key]); |
213
|
|
|
|
214
|
|
|
return $new_list; |
215
|
|
|
} |
216
|
|
|
|
217
|
|
|
public function get_storage_usage() |
218
|
|
|
{ |
219
|
|
|
$files = $this->get_backup_archives_list(); |
220
|
|
|
$total = 0; |
221
|
|
|
|
222
|
|
|
if(is_array($files)) |
223
|
|
|
foreach($files as $file) |
224
|
|
|
$total += $file['size']; |
225
|
|
|
|
226
|
|
|
return $total; |
227
|
|
|
} |
228
|
|
|
|
229
|
|
|
public function is_part($backup_name) |
230
|
|
|
{ |
231
|
|
|
if(stristr($backup_name, "-part")) |
232
|
|
|
return true; |
233
|
|
|
|
234
|
|
|
return false; |
235
|
|
|
} |
236
|
|
|
|
237
|
|
|
public function is_multipart($backup_name) |
238
|
|
|
{ |
239
|
|
|
if(stristr($backup_name, "-multipart")) |
240
|
|
|
return true; |
241
|
|
|
|
242
|
|
|
return false; |
243
|
|
|
} |
244
|
|
|
|
245
|
|
View Code Duplication |
public function get_backup_size($backup_name) |
|
|
|
|
246
|
|
|
{ |
247
|
|
|
$backup_size = $this->get_storage_filesystem()->getSize($backup_name); |
248
|
|
|
if($this->is_multipart($backup_name)) |
249
|
|
|
{ |
250
|
|
|
$backup_parts = $this->get_multipart_files($backup_name); |
251
|
|
|
foreach($backup_parts as $part_file) |
252
|
|
|
$backup_size += $this->get_storage_filesystem()->getSize($part_file); |
253
|
|
|
} |
254
|
|
|
|
255
|
|
|
return $backup_size; |
256
|
|
|
} |
257
|
|
|
|
258
|
|
View Code Duplication |
public function get_multipart_files($backup_name, $storage_selection = "") |
|
|
|
|
259
|
|
|
{ |
260
|
|
|
$files = array(); |
261
|
|
|
|
262
|
|
|
if($this->is_multipart($backup_name)) |
263
|
|
|
{ |
264
|
|
|
$lines = explode(PHP_EOL, $this->get_storage_filesystem($storage_selection)->read($backup_name)); |
265
|
|
|
foreach($lines as $line) |
266
|
|
|
{ |
267
|
|
|
if($line) |
268
|
|
|
{ |
269
|
|
|
$data = str_getcsv($line); |
270
|
|
|
$files[] = $data[0]; |
271
|
|
|
} |
272
|
|
|
} |
273
|
|
|
} |
274
|
|
|
|
275
|
|
|
return $files; |
276
|
|
|
} |
277
|
|
|
|
278
|
|
|
public function delete_backup_by_name($backup_name, $storage_selection = "") |
279
|
|
|
{ |
280
|
|
|
if($this->is_multipart($backup_name)) |
281
|
|
|
{ |
282
|
|
|
$lines = explode(PHP_EOL, $this->get_storage_filesystem($storage_selection)->read($backup_name)); |
283
|
|
|
foreach($lines as $line) |
284
|
|
|
{ |
285
|
|
|
if($line) |
286
|
|
|
{ |
287
|
|
|
$data = str_getcsv($line); |
288
|
|
|
$this->get_storage_filesystem($storage_selection)->delete($data[0]); |
289
|
|
|
} |
290
|
|
|
} |
291
|
|
|
} |
292
|
|
|
|
293
|
|
|
if($this->get_storage_filesystem($storage_selection)->delete($backup_name)) |
294
|
|
|
$return = true; |
295
|
|
|
else |
296
|
|
|
$return = false; |
297
|
|
|
|
298
|
|
|
return $return; |
299
|
|
|
} |
300
|
|
|
|
301
|
|
|
public function getMetadataFull($adapter = "storage_adapter" , $path) |
302
|
|
|
{ |
303
|
|
|
$location = $this->$adapter->applyPathPrefix($path); |
304
|
|
|
$spl_info = new SplFileInfo($location); |
305
|
|
|
|
306
|
|
|
return ($spl_info); |
307
|
|
|
} |
308
|
|
|
|
309
|
|
|
|
310
|
|
|
public function get_backup_archives_list($storage_selection = "") |
311
|
|
|
{ |
312
|
|
|
$list = array(); |
313
|
|
|
|
314
|
|
|
|
315
|
|
|
if(method_exists($this->get_storage_filesystem($storage_selection), "listContents")) |
316
|
|
|
$list = $this->get_storage_filesystem($storage_selection)->listContents(); |
317
|
|
|
|
318
|
|
|
|
319
|
|
|
$backup_files = array(); |
320
|
|
|
$parents = array(); |
321
|
|
|
|
322
|
|
View Code Duplication |
foreach($list as $file_info) |
|
|
|
|
323
|
|
|
{ |
324
|
|
|
if(isset($file_info['extension']) and $file_info['extension'] == "csv") |
|
|
|
|
325
|
|
|
{ |
326
|
|
|
$data = array(); |
|
|
|
|
327
|
|
|
|
328
|
|
|
$lines = explode(PHP_EOL, $this->get_storage_filesystem($storage_selection)->read($file_info['path'])); |
329
|
|
|
foreach($lines as $line) |
330
|
|
|
if($line) |
331
|
|
|
{ |
332
|
|
|
$data = str_getcsv($line); |
333
|
|
|
if(is_array($data)){ |
334
|
|
|
$parents[$data[0]] = $file_info['basename']; |
335
|
|
|
$file_info['childs'][] = $data; |
336
|
|
|
$file_info['size'] += $data[2]; |
337
|
|
|
} |
338
|
|
|
} |
339
|
|
|
|
340
|
|
|
} |
341
|
|
|
|
342
|
|
|
if($file_info['type'] == 'file' and isset($file_info['extension']) and in_array($file_info['extension'], $this->backup_archive_extensions)) |
|
|
|
|
343
|
|
|
$backup_files[$file_info['path']] = $file_info; |
344
|
|
|
} |
345
|
|
|
|
346
|
|
|
foreach($backup_files as $key=>$file_info) |
347
|
|
|
{ |
348
|
|
|
if(!isset($backup_files[$key]['timestamp'])) |
|
|
|
|
349
|
|
|
{ |
350
|
|
|
//$backup_files[$key]['timestamp'] = $this->get_storage_filesystem($storage_selection)->getTimestamp($file_info['path']); |
|
|
|
|
351
|
|
|
} |
352
|
|
|
|
353
|
|
|
if(isset($parents[$file_info['basename']])) |
354
|
|
|
$backup_files[$key]['parent'] = $parents[$file_info['basename']]; |
355
|
|
|
} |
356
|
|
|
|
357
|
|
|
return $backup_files; |
358
|
|
|
} |
359
|
|
|
|
360
|
|
|
public function start_file_recursion($init = 0) |
361
|
|
|
{ |
362
|
|
|
if($init) |
363
|
|
|
{ |
364
|
|
|
$this->logger->info(sprintf(__("Starting the filesystem scanner on root folder %s"), $this->xcloner_settings->get_xcloner_start_path())); |
365
|
|
|
$this->do_system_init(); |
366
|
|
|
} |
367
|
|
|
|
368
|
|
|
if($this->tmp_filesystem->has($this->get_temp_dir_handler())){ |
369
|
|
|
//.dir exists, we presume we have files to iterate |
370
|
|
|
$content = $this->tmp_filesystem->read($this->get_temp_dir_handler()); |
371
|
|
|
$files = array_filter(explode("\n", $content)); |
372
|
|
|
$this->tmp_filesystem->delete($this->get_temp_dir_handler()); |
373
|
|
|
|
374
|
|
|
$counter = 0; |
375
|
|
|
foreach($files as $file) |
376
|
|
|
{ |
377
|
|
|
if($counter < $this->folders_to_process_per_session){ |
378
|
|
|
$this->build_files_list($file); |
379
|
|
|
$counter++; |
380
|
|
|
}else{ |
381
|
|
|
$this->tmp_filesystem_append->write($this->get_temp_dir_handler(), $file."\n"); |
382
|
|
|
} |
383
|
|
|
} |
384
|
|
|
}else{ |
385
|
|
|
$this->build_files_list(); |
386
|
|
|
} |
387
|
|
|
|
388
|
|
|
if($this->scan_finished()) |
389
|
|
|
{ |
390
|
|
|
$metadata_dumpfile = $this->get_tmp_filesystem()->getMetadata("index.html"); |
391
|
|
|
$this->store_file($metadata_dumpfile, 'tmp_filesystem'); |
392
|
|
|
$this->files_counter++; |
393
|
|
|
|
394
|
|
|
//adding included dump file to the included files list |
395
|
|
|
if($this->get_tmp_filesystem()->has($this->get_included_files_handler())) |
|
|
|
|
396
|
|
|
{ |
397
|
|
|
$metadata_dumpfile = $this->get_tmp_filesystem()->getMetadata($this->get_included_files_handler()); |
|
|
|
|
398
|
|
|
$this->store_file($metadata_dumpfile, 'tmp_filesystem'); |
399
|
|
|
$this->files_counter++; |
400
|
|
|
} |
401
|
|
|
|
402
|
|
|
//adding a default index.html to the temp xcloner folder |
403
|
|
|
if(!$this->get_tmp_filesystem()->has("index.html")) |
404
|
|
|
{ |
405
|
|
|
$this->get_tmp_filesystem()->write("index.html",""); |
406
|
|
|
} |
407
|
|
|
|
408
|
|
|
//adding the default log file |
409
|
|
|
if($this->get_tmp_filesystem()->has($this->xcloner_settings->get_logger_filename(1))) |
410
|
|
|
{ |
411
|
|
|
$metadata_dumpfile = $this->get_tmp_filesystem()->getMetadata($this->xcloner_settings->get_logger_filename(1)); |
412
|
|
|
$this->store_file($metadata_dumpfile, 'tmp_filesystem'); |
413
|
|
|
$this->files_counter++; |
414
|
|
|
} |
415
|
|
|
|
416
|
|
|
return false; |
417
|
|
|
} |
418
|
|
|
|
419
|
|
|
return true; |
420
|
|
|
} |
421
|
|
|
|
422
|
|
|
public function get_backup_attachments() |
423
|
|
|
{ |
424
|
|
|
$return = array(); |
425
|
|
|
$return[] = $this->xcloner_settings->get_xcloner_tmp_path().DS.$this->get_included_files_handler(); |
426
|
|
|
if($this->xcloner_settings->get_xcloner_option('xcloner_enable_log')) |
427
|
|
|
$return[] = $this->xcloner_settings->get_xcloner_tmp_path().DS.$this->xcloner_settings->get_logger_filename(1); |
428
|
|
|
|
429
|
|
|
return $return; |
430
|
|
|
} |
431
|
|
|
|
432
|
|
|
public function remove_tmp_filesystem() |
433
|
|
|
{ |
434
|
|
|
//delete the temporary folder |
435
|
|
|
$this->logger->info(sprintf("Deleting the temporary storage folder %s", $this->xcloner_settings->get_xcloner_tmp_path())); |
436
|
|
|
|
437
|
|
|
$contents = $this->get_tmp_filesystem()->listContents(); |
438
|
|
|
|
439
|
|
|
if(is_array($contents)) |
440
|
|
|
foreach($contents as $file_info) |
441
|
|
|
$this->get_tmp_filesystem()->delete($file_info['path']); |
442
|
|
|
|
443
|
|
|
@rmdir($this->xcloner_settings->get_xcloner_tmp_path()); |
|
|
|
|
444
|
|
|
|
445
|
|
|
return; |
446
|
|
|
} |
447
|
|
|
|
448
|
|
|
public function cleanup_tmp_directories() |
449
|
|
|
{ |
450
|
|
|
$adapter = new Local($this->xcloner_settings->get_xcloner_tmp_path(false),LOCK_EX|FILE_APPEND, 'SKIP_LINKS'); |
451
|
|
|
$tmp_filesystem = new Filesystem($adapter, new Config([ |
452
|
|
|
'disable_asserts' => true, |
453
|
|
|
])); |
454
|
|
|
|
455
|
|
|
$contents = $tmp_filesystem->listContents(); |
456
|
|
|
|
457
|
|
|
foreach($contents as $file) |
458
|
|
|
{ |
459
|
|
View Code Duplication |
if(preg_match("/.xcloner-(.*)/",$file['path'])) |
|
|
|
|
460
|
|
|
{ |
461
|
|
|
$tmp_filesystem->deleteDir($file['path']); |
462
|
|
|
$this->logger->info(sprintf("Delete temporary directory %s", $file['path'])); |
463
|
|
|
} |
464
|
|
|
} |
465
|
|
|
|
466
|
|
|
return true; |
467
|
|
|
} |
468
|
|
|
|
469
|
|
|
private function do_system_init() |
470
|
|
|
{ |
471
|
|
|
$this->files_counter = 0; |
472
|
|
|
|
473
|
|
|
if(!$this->storage_filesystem->has("index.html")) |
474
|
|
|
$this->storage_filesystem->write("index.html",""); |
475
|
|
|
|
476
|
|
|
if(!$this->tmp_filesystem->has("index.html")) |
477
|
|
|
$this->tmp_filesystem->write("index.html",""); |
478
|
|
|
|
479
|
|
|
if($this->tmp_filesystem->has($this->get_included_files_handler())) |
|
|
|
|
480
|
|
|
$this->tmp_filesystem->delete($this->get_included_files_handler()); |
|
|
|
|
481
|
|
|
|
482
|
|
|
if($this->tmp_filesystem->has($this->get_temp_dir_handler())) |
483
|
|
|
$this->tmp_filesystem->delete($this->get_temp_dir_handler()); |
484
|
|
|
} |
485
|
|
|
|
486
|
|
|
public function get_scanned_files_num() |
487
|
|
|
{ |
488
|
|
|
return $this->files_counter; |
489
|
|
|
} |
490
|
|
|
|
491
|
|
|
public function get_scanned_files_total_size() |
492
|
|
|
{ |
493
|
|
|
return $this->files_size; |
494
|
|
|
} |
495
|
|
|
|
496
|
|
|
public function last_logged_file() |
497
|
|
|
{ |
498
|
|
|
return $this->last_logged_file; |
499
|
|
|
} |
500
|
|
|
|
501
|
|
|
public function set_excluded_files($excluded_files = array()) |
502
|
|
|
{ |
503
|
|
|
if(!is_array($excluded_files)) |
504
|
|
|
$excluded_files = array(); |
505
|
|
|
|
506
|
|
|
$this->excluded_files = array_merge($excluded_files, $this->excluded_files_by_default); |
|
|
|
|
507
|
|
|
|
508
|
|
|
return $this->excluded_files; |
509
|
|
|
} |
510
|
|
|
|
511
|
|
|
public function get_excluded_files() |
512
|
|
|
{ |
513
|
|
|
return $this->excluded_files_by_default; |
514
|
|
|
} |
515
|
|
|
|
516
|
|
|
public function list_directory($path) |
517
|
|
|
{ |
518
|
|
|
return $this->start_filesystem->listContents($path); |
519
|
|
|
} |
520
|
|
|
|
521
|
|
|
public function build_files_list($folder = "") |
522
|
|
|
{ |
523
|
|
|
$this->logger->debug(sprintf(("Building the files system list"))); |
524
|
|
|
|
525
|
|
|
//if we start with the root folder(empty value), we initializa the file system |
526
|
|
|
if(!$folder){ |
|
|
|
|
527
|
|
|
|
528
|
|
|
} |
529
|
|
|
|
530
|
|
|
try{ |
531
|
|
|
|
532
|
|
|
$files = $this->start_filesystem->listContents($folder); |
533
|
|
|
foreach($files as $file) |
534
|
|
|
{ |
535
|
|
|
if(!is_readable($this->xcloner_settings->get_xcloner_start_path().DS.$file['path'])) |
536
|
|
|
{ |
537
|
|
|
$this->logger->info(sprintf(__("Excluding %s from the filesystem list, file not readable"), $file['path']), array("FILESYSTEM SCAN","NOT READABLE")); |
538
|
|
|
} |
539
|
|
|
elseif(!$matching_pattern = $this->is_excluded($file) ){ |
540
|
|
|
$this->logger->info(sprintf(__("Adding %s to the filesystem list"), $file['path']), array("FILESYSTEM SCAN","INCLUDE")); |
541
|
|
|
$file['visibility'] = $this->start_filesystem->getVisibility($file['path']); |
542
|
|
|
if($this->store_file($file)) |
543
|
|
|
{ |
544
|
|
|
$this->files_counter++; |
545
|
|
|
} |
546
|
|
|
if(isset($file['size'])) |
547
|
|
|
$this->files_size += $file['size']; |
548
|
|
|
|
549
|
|
|
}else{ |
550
|
|
|
$this->logger->info(sprintf(__("Excluding %s from the filesystem list, matching pattern %s"), $file['path'], $matching_pattern), array("FILESYSTEM SCAN","EXCLUDE")); |
551
|
|
|
} |
552
|
|
|
} |
553
|
|
|
|
554
|
|
|
}catch(Exception $e){ |
555
|
|
|
|
556
|
|
|
$this->logger->error($e->getMessage()); |
557
|
|
|
|
558
|
|
|
} |
559
|
|
|
|
560
|
|
|
} |
561
|
|
|
|
562
|
|
|
public function estimate_read_write_time() |
563
|
|
|
{ |
564
|
|
|
$tmp_file = ".xcloner".substr(md5(time()), 0, 5); |
565
|
|
|
|
566
|
|
|
$start_time = microtime(); |
567
|
|
|
|
568
|
|
|
$data = str_repeat(rand(0,9), 1024*1024); //write 1MB data |
569
|
|
|
|
570
|
|
|
try{ |
571
|
|
|
$this->tmp_filesystem->write($tmp_file, $data); |
572
|
|
|
|
573
|
|
|
$end_time = microtime() - $start_time; |
574
|
|
|
|
575
|
|
|
$return['writing_time'] = $end_time; |
|
|
|
|
576
|
|
|
|
577
|
|
|
$return['reading_time'] = $this->estimate_reading_time($tmp_file); |
578
|
|
|
|
579
|
|
|
$this->tmp_filesystem->delete($tmp_file); |
580
|
|
|
|
581
|
|
|
}catch(Exception $e){ |
582
|
|
|
|
583
|
|
|
$this->logger->error($e->getMessage()); |
584
|
|
|
|
585
|
|
|
} |
586
|
|
|
|
587
|
|
|
return $return; |
|
|
|
|
588
|
|
|
} |
589
|
|
|
|
590
|
|
|
public function backup_storage_cleanup() |
591
|
|
|
{ |
592
|
|
|
$this->logger->info(sprintf(("Cleaning the backup storage on matching rules"))); |
593
|
|
|
|
594
|
|
|
$_storage_size = 0; |
595
|
|
|
$_backup_files_list = array(); |
596
|
|
|
|
597
|
|
|
//rule date limit |
598
|
|
|
$current_timestamp = strtotime("-".$this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_days')." days"); |
599
|
|
|
|
600
|
|
|
$files = $this->storage_filesystem->listContents(); |
601
|
|
|
|
602
|
|
|
if(is_array($files)) |
603
|
|
|
foreach($files as $file) |
604
|
|
|
{ |
605
|
|
|
if(isset($file['extension']) and in_array($file['extension'], $this->backup_archive_extensions)) |
|
|
|
|
606
|
|
|
{ |
607
|
|
|
$_storage_size += $file['size']; //bytes |
608
|
|
|
$_backup_files_list[] = $file; |
609
|
|
|
} |
610
|
|
|
} |
611
|
|
|
|
612
|
|
|
|
613
|
|
|
$this->sort_by($_backup_files_list, "timestamp","asc"); |
614
|
|
|
|
615
|
|
|
$_backups_counter = sizeof($_backup_files_list); |
616
|
|
|
|
617
|
|
|
foreach($_backup_files_list as $file) |
618
|
|
|
{ |
619
|
|
|
//processing rule folder capacity |
620
|
|
|
if($this->xcloner_settings->get_xcloner_option('xcloner_cleanup_capacity_limit') && |
621
|
|
|
$_storage_size >= ($set_storage_limit = 1024*1024*$this->xcloner_settings->get_xcloner_option('xcloner_cleanup_capacity_limit'))) //bytes |
622
|
|
|
{ |
623
|
|
|
$this->storage_filesystem->delete($file['path']); |
624
|
|
|
$_storage_size -= $file['size']; |
625
|
|
|
$this->logger->info("Deleting backup ".$file['path']." matching rule", array("STORAGE SIZE LIMIT", $_storage_size." >= ".$set_storage_limit)); |
626
|
|
|
} |
627
|
|
|
|
628
|
|
|
//processing rule days limit |
629
|
|
View Code Duplication |
if($this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_days') && $current_timestamp >= $file['timestamp']) |
|
|
|
|
630
|
|
|
{ |
631
|
|
|
$this->storage_filesystem->delete($file['path']); |
632
|
|
|
$this->logger->info("Deleting backup ".$file['path']." matching rule", array("RETENTION LIMIT TIMESTAMP", $file['timestamp']." =< ".$this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_days'))); |
633
|
|
|
} |
634
|
|
|
|
635
|
|
|
//processing backup countert limit |
636
|
|
View Code Duplication |
if($this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_archives') && $_backups_counter >= $this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_archives')) |
|
|
|
|
637
|
|
|
{ |
638
|
|
|
$this->storage_filesystem->delete($file['path']); |
639
|
|
|
$_backups_counter--; |
640
|
|
|
$this->logger->info("Deleting backup ".$file['path']." matching rule", array("BACKUP QUANTITY LIMIT", $_backups_counter." >= ".$this->xcloner_settings->get_xcloner_option('xcloner_cleanup_retention_limit_archives'))); |
641
|
|
|
} |
642
|
|
|
|
643
|
|
|
|
644
|
|
|
} |
645
|
|
|
|
646
|
|
|
} |
647
|
|
|
|
648
|
|
|
public function estimate_reading_time($tmp_file) |
649
|
|
|
{ |
650
|
|
|
$this->logger->debug(sprintf(("Estimating file system reading time"))); |
651
|
|
|
|
652
|
|
|
$start_time = microtime(); |
653
|
|
|
|
654
|
|
|
$this->tmp_filesystem->read($tmp_file); |
655
|
|
|
|
656
|
|
|
$end_time = microtime() - $start_time; |
657
|
|
|
|
658
|
|
|
return $end_time; |
659
|
|
|
|
660
|
|
|
} |
661
|
|
|
|
662
|
|
|
public function process_backup_name($name = "", $max_length=100) |
663
|
|
|
{ |
664
|
|
|
if(!$name) |
665
|
|
|
$name = $this->xcloner_settings->get_default_backup_name(); |
666
|
|
|
|
667
|
|
|
foreach($this->backup_name_tags as $tag) |
668
|
|
|
{ |
669
|
|
|
if($tag == '[time]') |
670
|
|
|
$name = str_replace($tag, date("Y-m-d_H-i"),$name); |
671
|
|
|
elseif($tag == '[hostname]') |
672
|
|
|
$name = str_replace($tag, gethostname() ,$name); |
673
|
|
|
elseif($tag == '[domain]') |
674
|
|
|
{ |
675
|
|
|
$domain = parse_url(admin_url(), PHP_URL_HOST); |
676
|
|
|
$name = str_replace($tag, $domain ,$name); |
677
|
|
|
} |
678
|
|
|
} |
679
|
|
|
|
680
|
|
|
if($max_length) |
681
|
|
|
$name = substr($name, 0, $max_length); |
682
|
|
|
|
683
|
|
|
return $name; |
684
|
|
|
} |
685
|
|
|
|
686
|
|
|
public function sort_by( &$array, $field, $direction = 'asc') |
687
|
|
|
{ |
688
|
|
|
if(strtolower($direction) == "desc" || $direction == SORT_DESC) |
689
|
|
|
$direction = SORT_DESC; |
690
|
|
|
else |
691
|
|
|
$direction = SORT_ASC; |
692
|
|
|
|
693
|
|
|
$array = $this->array_orderby($array, $field, $direction); |
694
|
|
|
|
695
|
|
|
return true; |
696
|
|
|
} |
697
|
|
|
|
698
|
|
|
private function array_orderby() |
699
|
|
|
{ |
700
|
|
|
$args = func_get_args(); |
701
|
|
|
$data = array_shift($args); |
702
|
|
|
|
703
|
|
|
foreach ($args as $n => $field) { |
704
|
|
|
if (is_string($field)) { |
705
|
|
|
$tmp = array(); |
706
|
|
|
foreach ($data as $key => $row) |
707
|
|
|
{ |
708
|
|
|
if(is_array($row)) |
709
|
|
|
$tmp[$key] = $row[$field]; |
710
|
|
|
else |
711
|
|
|
$tmp[$key] = $row->$field; |
712
|
|
|
} |
713
|
|
|
$args[$n] = $tmp; |
714
|
|
|
} |
715
|
|
|
} |
716
|
|
|
$args[] = &$data; |
717
|
|
|
|
718
|
|
|
call_user_func_array('array_multisort', $args); |
719
|
|
|
|
720
|
|
|
return array_pop($args); |
721
|
|
|
} |
722
|
|
|
|
723
|
|
|
private function check_file_diff_time($file) |
724
|
|
|
{ |
725
|
|
|
if($this->get_diff_timestamp_start() != "") |
726
|
|
|
{ |
727
|
|
|
$fileMeta = $this->getMetadataFull("start_adapter", $file['path']); |
728
|
|
|
$timestamp = $fileMeta->getMTime(); |
729
|
|
|
if($timestamp < $fileMeta->getCTime()) |
730
|
|
|
{ |
731
|
|
|
$timestamp = $fileMeta->getCTime(); |
732
|
|
|
} |
733
|
|
|
|
734
|
|
|
if($timestamp <= $this->get_diff_timestamp_start()) |
735
|
|
|
{ |
736
|
|
|
return " file DIFF timestamp ".$timestamp." < ". $this->diff_timestamp_start; |
737
|
|
|
} |
738
|
|
|
} |
739
|
|
|
|
740
|
|
|
return false; |
741
|
|
|
} |
742
|
|
|
|
743
|
|
|
public function is_excluded($file) |
744
|
|
|
{ |
745
|
|
|
$this->logger->debug(sprintf(("Checking if %s is excluded"), $file['path'])); |
746
|
|
|
|
747
|
|
|
if($xcloner_exclude_files_larger_than_mb = $this->xcloner_settings->get_xcloner_option('xcloner_exclude_files_larger_than_mb')) |
748
|
|
|
{ |
749
|
|
|
if(isset($file['size']) && $file['size'] > $this->calc_to_bytes($xcloner_exclude_files_larger_than_mb)) |
750
|
|
|
return "> ".$xcloner_exclude_files_larger_than_mb."MB"; |
751
|
|
|
} |
752
|
|
|
|
753
|
|
|
if(!sizeof($this->excluded_files)) |
754
|
|
|
$this->set_excluded_files(); |
755
|
|
|
|
756
|
|
|
if(is_array($this->excluded_files)) |
757
|
|
|
foreach($this->excluded_files as $excluded_file_pattern) |
758
|
|
|
{ |
759
|
|
|
if($excluded_file_pattern == "/") |
760
|
|
|
$needle = "$"; |
761
|
|
|
else |
762
|
|
|
$needle = "$".$excluded_file_pattern; |
763
|
|
|
|
764
|
|
|
if(strstr("$".$file['path'], $needle)){ |
765
|
|
|
return $excluded_file_pattern; |
766
|
|
|
} |
767
|
|
|
} |
768
|
|
|
|
769
|
|
|
if( $regex = $this->is_excluded_regex($file)) |
770
|
|
|
return $regex; |
771
|
|
|
|
772
|
|
|
if($file['type'] == "file") |
773
|
|
|
{ |
774
|
|
|
$check_file_diff_timestamp = $this->check_file_diff_time($file); |
775
|
|
|
if($check_file_diff_timestamp) |
|
|
|
|
776
|
|
|
{ |
777
|
|
|
return $check_file_diff_timestamp; |
778
|
|
|
} |
779
|
|
|
} |
780
|
|
|
|
781
|
|
|
return false; |
782
|
|
|
} |
783
|
|
|
|
784
|
|
|
/*REGEX examples |
785
|
|
|
* |
786
|
|
|
* exclude all except .php file |
787
|
|
|
* PATTERN: ^(.*)\.(.+)$(?<!(php)) |
788
|
|
|
* |
789
|
|
|
* exclude all except .php and .txt |
790
|
|
|
* PATTERN: ^(.*)\.(.+)$(?<!(php|txt))"; |
791
|
|
|
* |
792
|
|
|
* exclude all .svn and .git |
793
|
|
|
* PATTERN: ^(.*)\.(svn|git)(.*)$"; |
794
|
|
|
* |
795
|
|
|
* exclude root directory /test |
796
|
|
|
* PATTERN: "^\/test(.*)$"; |
797
|
|
|
* |
798
|
|
|
* exclude the wp-admin folder |
799
|
|
|
* PATTERN: ^(\/wp-admin)(.*)$"; |
800
|
|
|
* |
801
|
|
|
* exclude the wp-admin, wp-includes and wp-config.php |
802
|
|
|
* PATTERN: ^\/(wp-admin|wp-includes|wp-config.php)(.*)$"; |
803
|
|
|
* |
804
|
|
|
* exclude all .avi files |
805
|
|
|
* PATTERN: ^(.*)$(?<=(avi))"; |
806
|
|
|
* |
807
|
|
|
* exclude all .jpg and gif files |
808
|
|
|
* PATTERN: ^(.*)$(?<=(gif|jpg))"; |
809
|
|
|
* |
810
|
|
|
* exclude all cache folders from wp-content/ |
811
|
|
|
* PATTERN: ^\/wp-content(.*)\/cache($|\/)(.*)"; |
812
|
|
|
* |
813
|
|
|
* exclude the backup folders |
814
|
|
|
* PATTERN: (^|^\/)(wp-content\/backups|administrator\/backups)(.*)$"; |
815
|
|
|
*/ |
816
|
|
|
private function is_excluded_regex($file) |
817
|
|
|
{ |
818
|
|
|
//$this->logger->debug(sprintf(("Checking if %s is excluded"), $file['path'])); |
|
|
|
|
819
|
|
|
|
820
|
|
|
$regex_patterns = explode(PHP_EOL, $this->xcloner_settings->get_xcloner_option('xcloner_regex_exclude')); |
821
|
|
|
|
822
|
|
|
//print_r($regex_patterns);exit; |
|
|
|
|
823
|
|
|
|
824
|
|
|
if(is_array($regex_patterns)) |
825
|
|
|
{ |
826
|
|
|
//$this->excluded_files = array(); |
|
|
|
|
827
|
|
|
//$this->excluded_files[] ="(.*)\.(git)(.*)$"; |
|
|
|
|
828
|
|
|
//$this->excluded_files[] ="wp-content\/backups(.*)$"; |
|
|
|
|
829
|
|
|
|
830
|
|
|
foreach($regex_patterns as $excluded_file_pattern) |
831
|
|
|
{ |
832
|
|
|
|
833
|
|
|
if( substr($excluded_file_pattern, strlen($excluded_file_pattern)-1, strlen($excluded_file_pattern)) == "\r") |
834
|
|
|
$excluded_file_pattern = substr($excluded_file_pattern, 0, strlen($excluded_file_pattern)-1); |
835
|
|
|
|
836
|
|
|
if($file['path'] == "/") |
837
|
|
|
$needle = "/"; |
838
|
|
|
else |
839
|
|
|
$needle = "/".$file['path']; |
840
|
|
|
//echo $needle."---".$excluded_file_pattern."---\n"; |
|
|
|
|
841
|
|
|
|
842
|
|
|
if(@preg_match("/(^|^\/)".$excluded_file_pattern."/i", $needle)){ |
843
|
|
|
return $excluded_file_pattern; |
844
|
|
|
} |
845
|
|
|
} |
846
|
|
|
} |
847
|
|
|
|
848
|
|
|
return false; |
849
|
|
|
} |
850
|
|
|
|
851
|
|
|
public function store_file($file, $storage = 'start_filesystem') |
852
|
|
|
{ |
853
|
|
|
$this->logger->debug(sprintf("Storing %s in the backup list", $file['path'])); |
854
|
|
|
|
855
|
|
|
if(!isset($file['size'])) |
856
|
|
|
$file['size'] = 0; |
857
|
|
|
if(!isset($file['visibility'])) |
858
|
|
|
$file['visibility'] = "private"; |
859
|
|
|
|
860
|
|
|
$csv_filename = str_replace('"','""', $file['path']); |
861
|
|
|
|
862
|
|
|
$line = '"'.($csv_filename).'","'.$file['timestamp'].'","'.$file['size'].'","'.$file['visibility'].'","'.$storage.'"'.PHP_EOL; |
863
|
|
|
|
864
|
|
|
$this->last_logged_file = $file['path']; |
865
|
|
|
|
866
|
|
|
if($file['type'] == "dir"){ |
867
|
|
|
try{ |
868
|
|
|
$this->tmp_filesystem_append->write($this->get_temp_dir_handler(), $file['path']."\n"); |
869
|
|
|
}catch(Exception $e){ |
870
|
|
|
$this->logger->error($e->getMessage()); |
871
|
|
|
} |
872
|
|
|
} |
873
|
|
|
|
874
|
|
|
if($this->get_diff_timestamp_start()) |
875
|
|
|
{ |
876
|
|
|
if($file['type'] != "file" && $response = $this->check_file_diff_time($file)) |
877
|
|
|
{ |
878
|
|
|
$this->logger->info(sprintf("Directory %s archiving skipped on differential backup %s", $file['path'], $response), array("FILESYSTEM SCAN","DIR DIFF")); |
879
|
|
|
return false; |
880
|
|
|
} |
881
|
|
|
} |
882
|
|
|
|
883
|
|
|
try{ |
884
|
|
|
if(!$this->tmp_filesystem_append->has($this->get_included_files_handler())) |
|
|
|
|
885
|
|
|
{ |
886
|
|
|
//adding fix for UTF-8 CSV preview |
887
|
|
|
$start_line = "\xEF\xBB\xBF".'"Filename","Timestamp","Size","Visibility","Storage"'.PHP_EOL; |
888
|
|
|
$this->tmp_filesystem_append->write($this->get_included_files_handler(), $start_line); |
|
|
|
|
889
|
|
|
} |
890
|
|
|
|
891
|
|
|
$this->tmp_filesystem_append->write($this->get_included_files_handler(), $line); |
|
|
|
|
892
|
|
|
|
893
|
|
|
}catch(Exception $e){ |
894
|
|
|
|
895
|
|
|
$this->logger->error($e->getMessage()); |
896
|
|
|
} |
897
|
|
|
|
898
|
|
|
return true; |
899
|
|
|
} |
900
|
|
|
|
901
|
|
|
public function get_fileystem_handler() |
902
|
|
|
{ |
903
|
|
|
return $this; |
904
|
|
|
} |
905
|
|
|
|
906
|
|
|
public function get_filesystem($system = "") |
907
|
|
|
{ |
908
|
|
|
if($system == "storage_filesystem_append") |
909
|
|
|
return $this->storage_filesystem_append; |
910
|
|
|
elseif($system == "tmp_filesystem_append") |
911
|
|
|
return $this->tmp_filesystem_append; |
912
|
|
|
elseif($system == "tmp_filesystem") |
913
|
|
|
return $this->tmp_filesystem; |
914
|
|
|
elseif($system == "storage_filesystem") |
915
|
|
|
return $this->storage_filesystem; |
916
|
|
|
else |
917
|
|
|
return $this->start_filesystem; |
918
|
|
|
} |
919
|
|
|
|
920
|
|
|
public function get_adapter($system) |
921
|
|
|
{ |
922
|
|
|
if($system == "tmp_filesystem") |
923
|
|
|
return $this->tmp_adapter; |
924
|
|
|
elseif($system == "storage_filesystem") |
925
|
|
|
return $this->storage_adapter; |
926
|
|
|
else |
927
|
|
|
return $this->start_adapter; |
928
|
|
|
} |
929
|
|
|
|
930
|
|
|
private function scan_finished() |
931
|
|
|
{ |
932
|
|
|
if($this->tmp_filesystem_append->has($this->get_temp_dir_handler()) && $this->tmp_filesystem_append->getSize($this->get_temp_dir_handler())) |
|
|
|
|
933
|
|
|
return false; |
934
|
|
|
|
935
|
|
|
if($this->tmp_filesystem->has($this->get_temp_dir_handler())) |
936
|
|
|
$this->tmp_filesystem->delete($this->get_temp_dir_handler()); |
937
|
|
|
|
938
|
|
|
$this->logger->debug(sprintf(("File scan finished"))); |
939
|
|
|
|
940
|
|
|
return true; |
941
|
|
|
} |
942
|
|
|
|
943
|
|
|
private function calc_to_bytes($mb_size) |
944
|
|
|
{ |
945
|
|
|
return $mb_size*(1024*1024); |
946
|
|
|
} |
947
|
|
|
|
948
|
|
|
} |
949
|
|
|
|
You can fix this by adding a namespace to your class:
When choosing a vendor namespace, try to pick something that is not too generic to avoid conflicts with other libraries.