1 | import csv |
||
2 | import os |
||
3 | import sqlite3 |
||
4 | from typing import List, Dict, Optional |
||
5 | |||
6 | |||
7 | View Code Duplication | class DataLayer: |
|
0 ignored issues
–
show
Duplication
introduced
by
![]() |
|||
8 | """ |
||
9 | Provides an interface to the SQLite3 data_layer |
||
10 | """ |
||
11 | instance = None |
||
12 | """ |
||
13 | The singleton instance of this class. |
||
14 | |||
15 | :type instance: backuppc_clone.DataLayer.DataLayer |
||
16 | """ |
||
17 | |||
18 | # ------------------------------------------------------------------------------------------------------------------ |
||
19 | def __init__(self, database: str): |
||
20 | """ |
||
21 | Object constructor. |
||
22 | |||
23 | @param str database: Path to the SQLite database. |
||
24 | """ |
||
25 | if DataLayer.instance is not None: |
||
26 | raise Exception("This class is a singleton!") |
||
27 | else: |
||
28 | DataLayer.instance = self |
||
29 | |||
30 | self.__database: str = database |
||
31 | """ |
||
32 | The path to the SQLite database. |
||
33 | """ |
||
34 | |||
35 | self.__connection: sqlite3.Connection = sqlite3.connect(':memory:') |
||
36 | """ |
||
37 | The connection to the database. |
||
38 | """ |
||
39 | |||
40 | self.__last_rowid: int = -1 |
||
41 | """ |
||
42 | The last rowid as returns by the last used cursor. |
||
43 | """ |
||
44 | |||
45 | self.connect() |
||
46 | |||
47 | # ------------------------------------------------------------------------------------------------------------------ |
||
48 | def disconnect(self) -> None: |
||
49 | """ |
||
50 | Disconnects from the SQLite database. |
||
51 | """ |
||
52 | self.__connection.commit() |
||
53 | self.__connection.close() |
||
54 | |||
55 | # ------------------------------------------------------------------------------------------------------------------ |
||
56 | def connect(self) -> None: |
||
57 | """ |
||
58 | Connects to the SQLite database. |
||
59 | """ |
||
60 | self.__connection = sqlite3.connect(self.__database, isolation_level="EXCLUSIVE") |
||
61 | |||
62 | tmp_dir = os.path.join(os.path.dirname(self.__database), 'tmp') |
||
63 | self.execute_none('pragma temp_store = 1') |
||
64 | self.execute_none('pragma temp_store_directory = \'{}\''.format(tmp_dir)) |
||
65 | self.execute_none('pragma main.cache_size = -200000') |
||
66 | |||
67 | # ------------------------------------------------------------------------------------------------------------------ |
||
68 | def backup_delete(self, bck_id: int) -> None: |
||
69 | """ |
||
70 | Deletes cascading a host backup. |
||
71 | |||
72 | @param int bck_id: The ID of the host backup. |
||
73 | """ |
||
74 | self.backup_empty(bck_id) |
||
75 | self.execute_none('delete from BKC_BACKUP where bck_id=?', (bck_id,)) |
||
76 | |||
77 | # ------------------------------------------------------------------------------------------------------------------ |
||
78 | def backup_empty(self, bck_id: int) -> None: |
||
79 | """ |
||
80 | Removes the tree from a host backup. |
||
81 | |||
82 | @param int bck_id: The ID of the host backup. |
||
83 | """ |
||
84 | self.execute_none('delete from BKC_BACKUP_TREE where bck_id=?', (bck_id,)) |
||
85 | |||
86 | # ------------------------------------------------------------------------------------------------------------------ |
||
87 | def backup_get_all(self) -> List[Dict]: |
||
88 | """ |
||
89 | Selects all cloned backups. |
||
90 | """ |
||
91 | sql = """ |
||
92 | select hst.hst_name |
||
93 | , bck.bck_number |
||
94 | from BKC_HOST hst |
||
95 | inner join BKC_BACKUP bck on bck.hst_id = hst.hst_id""" |
||
96 | |||
97 | return self.execute_rows(sql) |
||
98 | |||
99 | # ------------------------------------------------------------------------------------------------------------------ |
||
100 | def backup_get_next(self, end_time: int) -> Dict: |
||
101 | """ |
||
102 | Selects the next backup to clone. |
||
103 | |||
104 | :rtype: dict|None |
||
105 | """ |
||
106 | sql = """ |
||
107 | select bob.bob_host |
||
108 | , bob.bob_number |
||
109 | , bob.bob_end_time |
||
110 | , bob.bob_level |
||
111 | , bob.bob_type |
||
112 | from BKC_ORIGINAL_BACKUP bob |
||
113 | left outer join BKC_HOST hst on hst.hst_name = bob.bob_host |
||
114 | left outer join BKC_BACKUP bck on bck.hst_id = hst.hst_id and |
||
115 | bck.bck_number = bob.bob_number |
||
116 | where bck.bck_id is null |
||
117 | and bob.bob_version like '3.%' |
||
118 | and bob.bob_type in ('full', 'incr') |
||
119 | and bob.bob_end_time is not null |
||
120 | and (bob.bob_end_time < ? or ? = -1) |
||
121 | order by bob.bob_type |
||
122 | , bob.bob_end_time desc |
||
123 | limit 0, 1""" |
||
124 | |||
125 | return self.execute_row0(sql, (end_time, end_time)) |
||
126 | |||
127 | # ------------------------------------------------------------------------------------------------------------------ |
||
128 | def backup_get_obsolete(self) -> List[Dict]: |
||
129 | """ |
||
130 | Selects obsolete host backups. |
||
131 | |||
132 | :rtype: list[dict] |
||
133 | """ |
||
134 | sql = """ |
||
135 | select hst.hst_name |
||
136 | , bck.bck_number |
||
137 | from BKC_HOST hst |
||
138 | inner join BKC_BACKUP bck on bck.hst_id = hst.hst_id |
||
139 | left outer join BKC_ORIGINAL_BACKUP bob on bob.bob_host = hst.hst_name and |
||
140 | bob.bob_number = bck.bck_number |
||
141 | where bob.rowid is null |
||
142 | order by hst.hst_name |
||
143 | , bck.bck_number""" |
||
144 | |||
145 | return self.execute_rows(sql) |
||
146 | |||
147 | # ------------------------------------------------------------------------------------------------------------------ |
||
148 | def backup_partially_cloned(self) -> List[Dict]: |
||
149 | """ |
||
150 | Selects partially cloned host backups. |
||
151 | |||
152 | :rtype: list[dict] |
||
153 | """ |
||
154 | sql = """ |
||
155 | select hst.hst_name |
||
156 | , bck.bck_number |
||
157 | from BKC_HOST hst |
||
158 | inner join BKC_BACKUP bck on bck.hst_id = hst.hst_id |
||
159 | where ifnull(bck.bck_in_progress, 1) = 1 |
||
160 | order by hst.hst_name |
||
161 | , bck.bck_number""" |
||
162 | |||
163 | return self.execute_rows(sql) |
||
164 | |||
165 | # ------------------------------------------------------------------------------------------------------------------ |
||
166 | def backup_set_in_progress(self, bck_id: int, bck_in_progress: int) -> None: |
||
167 | """ |
||
168 | Updates the in progress flag of a host backup. |
||
169 | |||
170 | @param int bck_id: The ID of the host backup. |
||
171 | @param int bck_in_progress: The in progress flag. |
||
172 | """ |
||
173 | if bck_in_progress != 0: |
||
174 | bck_in_progress = 1 |
||
175 | |||
176 | sql = """ |
||
177 | update BKC_BACKUP |
||
178 | set bck_in_progress = ? |
||
179 | where bck_id = ?""" |
||
180 | |||
181 | self.execute_none(sql, (bck_in_progress, bck_id)) |
||
182 | |||
183 | # ------------------------------------------------------------------------------------------------------------------ |
||
184 | def backup_get_stats(self, bck_id: int) -> Dict: |
||
185 | """ |
||
186 | Selects the statistics of a host backup. |
||
187 | |||
188 | @param int bck_id: The ID of the host backup. |
||
189 | """ |
||
190 | self.__connection.row_factory = DataLayer.dict_factory |
||
191 | |||
192 | sql = """ |
||
193 | select count(bbt_inode_original) as '#files' |
||
194 | , sum(case when bbt_inode_original is null then 1 else 0 end) as '#dirs' |
||
195 | from BKC_BACKUP_TREE |
||
196 | where bck_id = ?""" |
||
197 | |||
198 | return self.execute_row1(sql, (bck_id,)) |
||
199 | |||
200 | # ------------------------------------------------------------------------------------------------------------------ |
||
201 | def backup_prepare_required_clone_pool_files(self, bck_id: int) -> int: |
||
202 | """ |
||
203 | Prepares the files required for a host backup that are not yet copied from the original pool to the clone pool. |
||
204 | |||
205 | @param int bck_id: The ID of the host backup. |
||
206 | |||
207 | :rtype: int |
||
208 | """ |
||
209 | self.execute_none('delete from TMP_CLONE_POOL_REQUIRED') |
||
210 | |||
211 | sql = """ |
||
212 | insert into TMP_CLONE_POOL_REQUIRED( bpl_inode_original |
||
213 | , bpl_dir |
||
214 | , bpl_name ) |
||
215 | select distinct bpl_inode_original |
||
216 | , bpl_dir |
||
217 | , bpl_name |
||
218 | from BKC_BACKUP_TREE bbt |
||
219 | inner join BKC_POOL bpl on bpl.bpl_inode_original = bbt.bbt_inode_original |
||
220 | where bbt.bck_id = ? |
||
221 | and bpl.bpl_inode_clone is null""" |
||
222 | |||
223 | self.execute_none(sql, (bck_id,)) |
||
224 | |||
225 | sql = """ |
||
226 | select count(distinct bpl_inode_original) |
||
227 | from TMP_CLONE_POOL_REQUIRED""" |
||
228 | |||
229 | return self.execute_singleton1(sql) |
||
230 | |||
231 | # ------------------------------------------------------------------------------------------------------------------ |
||
232 | def backup_prepare_tree(self, bck_id: int) -> int: |
||
233 | """ |
||
234 | Selects the file entries of a host backup. |
||
235 | |||
236 | @param int bck_id: The ID of the host backup. |
||
237 | |||
238 | :rtype: int |
||
239 | """ |
||
240 | self.execute_none('delete from TMP_BACKUP_TREE') |
||
241 | |||
242 | sql = """ |
||
243 | insert into TMP_BACKUP_TREE( bpl_inode_original |
||
244 | , bpl_dir |
||
245 | , bpl_name |
||
246 | |||
247 | , bbt_seq |
||
248 | , bbt_inode_original |
||
249 | , bbt_dir |
||
250 | , bbt_name ) |
||
251 | select bpl.bpl_inode_original |
||
252 | , bpl.bpl_dir |
||
253 | , bpl.bpl_name |
||
254 | |||
255 | , bbt.bbt_seq |
||
256 | , bbt.bbt_inode_original |
||
257 | , bbt.bbt_dir |
||
258 | , bbt.bbt_name |
||
259 | from BKC_BACKUP_TREE bbt |
||
260 | left outer join BKC_POOL bpl on bpl.bpl_inode_original = bbt.bbt_inode_original |
||
261 | where bbt.bck_id = ?""" |
||
262 | |||
263 | self.execute_none(sql, (bck_id,)) |
||
264 | |||
265 | sql = """ |
||
266 | select count(*) |
||
267 | from TMP_BACKUP_TREE""" |
||
268 | |||
269 | return self.execute_singleton1(sql) |
||
270 | |||
271 | # ------------------------------------------------------------------------------------------------------------------ |
||
272 | def backup_yield_required_clone_pool_files(self): |
||
273 | """ |
||
274 | Selects the pool files required for a host backup that are not yet copied from the original pool to the clone |
||
275 | pool. |
||
276 | """ |
||
277 | self.__connection.row_factory = DataLayer.dict_factory |
||
278 | |||
279 | sql = """ |
||
280 | select bpl_inode_original |
||
281 | , bpl_dir |
||
282 | , bpl_name |
||
283 | from TMP_CLONE_POOL_REQUIRED |
||
284 | order by bpl_dir |
||
285 | , bpl_name""" |
||
286 | |||
287 | cursor = self.__connection.cursor() |
||
288 | cursor.execute(sql) |
||
289 | while True: |
||
290 | rows = cursor.fetchmany(10000) |
||
291 | if not rows: |
||
292 | cursor.close() |
||
293 | return |
||
294 | yield rows |
||
295 | |||
296 | # ------------------------------------------------------------------------------------------------------------------ |
||
297 | def backup_yield_tree(self): |
||
298 | """ |
||
299 | Selects the file entries of a host backup. |
||
300 | """ |
||
301 | self.__connection.row_factory = DataLayer.dict_factory |
||
302 | |||
303 | sql = """ |
||
304 | select bpl_inode_original |
||
305 | , bpl_dir |
||
306 | , bpl_name |
||
307 | |||
308 | , bbt_inode_original |
||
309 | , bbt_dir |
||
310 | , bbt_name |
||
311 | from TMP_BACKUP_TREE |
||
312 | order by bbt_seq |
||
313 | , bpl_dir |
||
314 | , bpl_name""" |
||
315 | |||
316 | cursor = self.__connection.cursor() |
||
317 | cursor.execute(sql) |
||
318 | while True: |
||
319 | rows = cursor.fetchmany(10000) |
||
320 | if not rows: |
||
321 | cursor.close() |
||
322 | return |
||
323 | yield rows |
||
324 | |||
325 | # ------------------------------------------------------------------------------------------------------------------ |
||
326 | def commit(self) -> None: |
||
327 | """ |
||
328 | Commits the current transaction. |
||
329 | """ |
||
330 | self.__connection.commit() |
||
331 | |||
332 | # ------------------------------------------------------------------------------------------------------------------ |
||
333 | @staticmethod |
||
334 | def dict_factory(cursor: sqlite3.Cursor, old_row: List) -> Dict: |
||
335 | """ |
||
336 | Dictionary factory for return results with dictionaries. |
||
337 | |||
338 | @param Cursor cursor: The cursor. |
||
339 | @param list old_row: A row from the result a query. |
||
340 | |||
341 | :rtype: dict |
||
342 | """ |
||
343 | new_row = {} |
||
344 | for index, col in enumerate(cursor.description): |
||
345 | new_row[col[0]] = old_row[index] |
||
346 | |||
347 | return new_row |
||
348 | |||
349 | # ------------------------------------------------------------------------------------------------------------------ |
||
350 | def execute_none(self, sql: str, *params) -> None: |
||
351 | """ |
||
352 | Executes a SQL statement that does not select any rows |
||
353 | |||
354 | @param str sql: The SQL calling the stored procedure. |
||
355 | @param iterable params: The arguments for the stored procedure. |
||
356 | """ |
||
357 | self.__connection.row_factory = None |
||
358 | |||
359 | cursor = self.__connection.cursor() |
||
360 | cursor.execute(sql, *params) |
||
361 | self.__last_rowid = cursor.lastrowid |
||
362 | cursor.close() |
||
363 | |||
364 | # ------------------------------------------------------------------------------------------------------------------ |
||
365 | def execute_row0(self, sql: str, *params) -> Optional[Dict]: |
||
366 | """ |
||
367 | Executes a SQL statement that selects 0 or 1 row. |
||
368 | |||
369 | @param str sql: The SQL calling the stored procedure. |
||
370 | @param iterable params: The arguments for the stored procedure. |
||
371 | |||
372 | :rtype: dict |
||
373 | """ |
||
374 | self.__connection.row_factory = DataLayer.dict_factory |
||
375 | |||
376 | cursor = self.__connection.cursor() |
||
377 | cursor.execute(sql, *params) |
||
378 | rows = cursor.fetchall() |
||
379 | self.__last_rowid = cursor.lastrowid |
||
380 | cursor.close() |
||
381 | |||
382 | if len(rows) == 0: |
||
383 | return None |
||
384 | |||
385 | return rows[0] |
||
386 | |||
387 | # ------------------------------------------------------------------------------------------------------------------ |
||
388 | def execute_row1(self, sql: str, *params) -> Dict: |
||
389 | """ |
||
390 | Executes a SQL statement that selects 1 row. |
||
391 | |||
392 | @param str sql: The SQL calling the stored procedure. |
||
393 | @param iterable params: The arguments for the stored procedure. |
||
394 | |||
395 | :rtype: dict |
||
396 | """ |
||
397 | self.__connection.row_factory = DataLayer.dict_factory |
||
398 | |||
399 | cursor = self.__connection.cursor() |
||
400 | cursor.execute(sql, *params) |
||
401 | rows = cursor.fetchall() |
||
402 | self.__last_rowid = cursor.lastrowid |
||
403 | cursor.close() |
||
404 | |||
405 | return rows[0] |
||
406 | |||
407 | # ------------------------------------------------------------------------------------------------------------------ |
||
408 | def execute_rows(self, sql: str, *params) -> List[Dict]: |
||
409 | """ |
||
410 | Executes a SQL statement that selects 0, 1, or more rows. |
||
411 | |||
412 | @param str sql: The SQL calling the stored procedure. |
||
413 | @param iterable params: The arguments for the stored procedure. |
||
414 | |||
415 | :rtype: list[dict] |
||
416 | """ |
||
417 | self.__connection.row_factory = DataLayer.dict_factory |
||
418 | |||
419 | cursor = self.__connection.cursor() |
||
420 | cursor.execute(sql, *params) |
||
421 | rows = cursor.fetchall() |
||
422 | self.__last_rowid = cursor.lastrowid |
||
423 | cursor.close() |
||
424 | |||
425 | return rows |
||
426 | |||
427 | # ------------------------------------------------------------------------------------------------------------------ |
||
428 | def execute_singleton0(self, sql: str, *params): |
||
429 | """ |
||
430 | Executes a SQL statement that selects 0 or 1 row with 1 column. Returns the value of selected column or None. |
||
431 | |||
432 | @param str sql: The SQL calling the stored procedure. |
||
433 | @param iterable params: The arguments for the stored procedure. |
||
434 | |||
435 | :rtype: * |
||
436 | """ |
||
437 | self.__connection.row_factory = None |
||
438 | |||
439 | cursor = self.__connection.cursor() |
||
440 | cursor.execute(sql, *params) |
||
441 | row = cursor.fetchone() |
||
442 | if row: |
||
443 | ret = row[0] |
||
444 | else: |
||
445 | ret = None |
||
446 | self.__last_rowid = cursor.lastrowid |
||
447 | cursor.close() |
||
448 | |||
449 | return ret |
||
450 | |||
451 | # ------------------------------------------------------------------------------------------------------------------ |
||
452 | def execute_singleton1(self, sql: str, *params): |
||
453 | """ |
||
454 | Executes a SQL statement that selects 1 row with 1 column. Returns the value of selected column. |
||
455 | |||
456 | @param str sql: The SQL calling the stored procedure. |
||
457 | @param iterable params: The arguments for the stored procedure. |
||
458 | |||
459 | :rtype: * |
||
460 | """ |
||
461 | self.__connection.row_factory = None |
||
462 | |||
463 | cursor = self.__connection.cursor() |
||
464 | cursor.execute(sql, *params) |
||
465 | ret = cursor.fetchone()[0] |
||
466 | self.__last_rowid = cursor.lastrowid |
||
467 | cursor.close() |
||
468 | |||
469 | return ret |
||
470 | |||
471 | # ------------------------------------------------------------------------------------------------------------------ |
||
472 | def get_host_id(self, hostname: str) -> int: |
||
473 | """ |
||
474 | Returns the ID of a host. If the host does not exists it will be inserted. |
||
475 | |||
476 | @param str hostname: The name of the host. |
||
477 | |||
478 | :rtype: int |
||
479 | """ |
||
480 | hst_id = self.execute_singleton0('select hst_id from BKC_HOST where hst_name = ?', (hostname,)) |
||
481 | if hst_id is None: |
||
482 | self.execute_none('insert into BKC_HOST(hst_name) values(?)', (hostname,)) |
||
483 | hst_id = self.__last_rowid |
||
484 | |||
485 | return hst_id |
||
486 | |||
487 | # ------------------------------------------------------------------------------------------------------------------ |
||
488 | def get_bck_id(self, hst_id: int, bck_number: int) -> int: |
||
489 | """ |
||
490 | Returns the ID of a host backup. If the backup does not exist it will be inserted. |
||
491 | |||
492 | @param int hst_id: The ID of the host. |
||
493 | @param int bck_number: The number of the backup. |
||
494 | |||
495 | :rtype: int |
||
496 | """ |
||
497 | sql = """ |
||
498 | select bck_id |
||
499 | from BKC_BACKUP |
||
500 | where hst_id = ? |
||
501 | and bck_number = ?""" |
||
502 | |||
503 | bck_id = self.execute_singleton0(sql, (hst_id, bck_number)) |
||
504 | if bck_id is None: |
||
505 | self.execute_none('insert into BKC_BACKUP(hst_id, bck_number) values(?, ?)', (hst_id, bck_number)) |
||
506 | bck_id = self.__last_rowid |
||
507 | |||
508 | return bck_id |
||
509 | |||
510 | # ------------------------------------------------------------------------------------------------------------------ |
||
511 | def host_delete(self, host: str) -> None: |
||
512 | """ |
||
513 | Deletes cascading a host. |
||
514 | |||
515 | @param str host: The name of the host. |
||
516 | """ |
||
517 | sql = """ |
||
518 | select bck_id |
||
519 | from BKC_HOST hst |
||
520 | inner join BKC_BACKUP bck on bck.hst_id = hst.hst_id |
||
521 | where hst.hst_name = ?""" |
||
522 | |||
523 | rows = self.execute_rows(sql, (host,)) |
||
524 | for row in rows: |
||
525 | self.backup_delete(row['bck_id']) |
||
526 | |||
527 | self.execute_none('delete from BKC_HOST where hst_name=?', (host,)) |
||
528 | |||
529 | # ------------------------------------------------------------------------------------------------------------------ |
||
530 | def host_get_obsolete(self) -> List[Dict]: |
||
531 | """ |
||
532 | Selects hosts that are cloned but no longer in the original pool. |
||
533 | |||
534 | :rtype: list[dict] |
||
535 | """ |
||
536 | sql = """ |
||
537 | select hst.hst_name |
||
538 | from BKC_HOST hst |
||
539 | left outer join BKC_ORIGINAL_BACKUP bob on bob.bob_host = hst.hst_name |
||
540 | where bob.rowid is null |
||
541 | order by hst.hst_name""" |
||
542 | |||
543 | return self.execute_rows(sql) |
||
544 | |||
545 | # ------------------------------------------------------------------------------------------------------------------ |
||
546 | def import_csv(self, |
||
547 | table_name: str, |
||
548 | column_names: List[str], |
||
549 | path: str, |
||
550 | truncate: bool = True, |
||
551 | defaults: Dict = None): |
||
552 | """ |
||
553 | Import a CSV file into a table. |
||
554 | |||
555 | @param str table_name: The name of the table. |
||
556 | @param list column_names: The columns names. |
||
557 | @param str path: The path to the CSV file. |
||
558 | @param bool truncate: If True the table will be truncated first. |
||
559 | @param dict[str,*]|None defaults: The default values for columns not in the CSV file. |
||
560 | """ |
||
561 | if truncate: |
||
562 | self.execute_none('delete from {}'.format(table_name)) |
||
563 | |||
564 | default_values = [] |
||
565 | if defaults: |
||
566 | for column_name in defaults: |
||
567 | column_names.append(column_name) |
||
568 | default_values.append(defaults[column_name]) |
||
569 | |||
570 | place_holders = [] |
||
571 | for _ in range(0, len(column_names)): |
||
572 | place_holders.append('?') |
||
573 | |||
574 | sql = 'insert into {}({}) values ({})'.format(table_name, ', '.join(column_names), ', '.join(place_holders)) |
||
575 | cursor = self.__connection.cursor() |
||
576 | rows = [] |
||
577 | with open(path, 'r') as csv_file: |
||
578 | csv_reader = csv.reader(csv_file) |
||
579 | for row in csv_reader: |
||
580 | # Replace empty string with None. |
||
581 | for index, field in enumerate(row): |
||
582 | if field == '': |
||
583 | row[index] = None |
||
584 | |||
585 | if defaults: |
||
586 | row.extend(default_values) |
||
587 | |||
588 | rows.append(row) |
||
589 | |||
590 | if len(rows) == 1000: |
||
591 | cursor.executemany(sql, rows) |
||
592 | rows = [] |
||
593 | |||
594 | if rows: |
||
595 | cursor.executemany(sql, rows) |
||
596 | |||
597 | cursor.close() |
||
598 | |||
599 | # ------------------------------------------------------------------------------------------------------------------ |
||
600 | def original_backup_insert(self, |
||
601 | bob_host: str, |
||
602 | bob_number: int, |
||
603 | bob_end_time: str, |
||
604 | bob_version: int, |
||
605 | bob_level: int, |
||
606 | bob_type: str) -> None: |
||
607 | """ |
||
608 | Inserts an original host backup. |
||
609 | """ |
||
610 | sql = """ |
||
611 | insert into BKC_ORIGINAL_BACKUP( bob_host |
||
612 | , bob_number |
||
613 | , bob_end_time |
||
614 | , bob_version |
||
615 | , bob_level |
||
616 | , bob_type ) |
||
617 | values( ? |
||
618 | , ? |
||
619 | , ? |
||
620 | , ? |
||
621 | , ? |
||
622 | , ? )""" |
||
623 | self.execute_none(sql, (bob_host, bob_number, bob_end_time, bob_version, bob_level, bob_type)) |
||
624 | |||
625 | # ------------------------------------------------------------------------------------------------------------------ |
||
626 | def original_backup_get_stats(self) -> Dict: |
||
627 | """ |
||
628 | Select statistics of the original backups. |
||
629 | |||
630 | :rtype: dict |
||
631 | """ |
||
632 | sql = """ |
||
633 | select count(distinct bob_host) as '#hosts' |
||
634 | , count(*) as '#backups' |
||
635 | from BKC_ORIGINAL_BACKUP""" |
||
636 | |||
637 | return self.execute_row1(sql) |
||
638 | |||
639 | # ------------------------------------------------------------------------------------------------------------------ |
||
640 | def original_backup_truncate(self) -> None: |
||
641 | """ |
||
642 | Truncates table BKC_ORIGINAL_BACKUP. |
||
643 | """ |
||
644 | self.execute_none('delete from BKC_ORIGINAL_BACKUP') |
||
645 | |||
646 | # ------------------------------------------------------------------------------------------------------------------ |
||
647 | def overview_get_stats(self) -> Dict: |
||
648 | """ |
||
649 | Select statistics of the original backups and cloned backups. |
||
650 | |||
651 | :rtype: dict |
||
652 | """ |
||
653 | sql = """ |
||
654 | select sum(case when cnt1=1 then 1 else 0 end) as n_backups |
||
655 | , sum(case when cnt1=1 and cnt2=1 then 1 else 0 end) as n_cloned_backups |
||
656 | , sum(case when cnt1=1 and cnt2=0 then 1 else 0 end) as n_not_cloned_backups |
||
657 | , sum(case when cnt1=0 and cnt2=1 then 1 else 0 end) as n_obsolete_cloned_backups |
||
658 | from |
||
659 | ( |
||
660 | select sum(case when src=1 then 1 else 0 end) as cnt1 |
||
661 | , sum(case when src=2 then 1 else 0 end) as cnt2 |
||
662 | from ( |
||
663 | select bob_host |
||
664 | , bob_number |
||
665 | , 1 as src |
||
666 | from BKC_ORIGINAL_BACKUP |
||
667 | |||
668 | union all |
||
669 | |||
670 | select hst.hst_name |
||
671 | , bck.bck_number |
||
672 | , 2 as src |
||
673 | from BKC_BACKUP bck |
||
674 | join BKC_HOST hst on hst.hst_id = bck.hst_id |
||
675 | ) t |
||
676 | group by bob_host |
||
677 | , bob_number |
||
678 | )""" |
||
679 | |||
680 | return self.execute_row1(sql) |
||
681 | |||
682 | # ------------------------------------------------------------------------------------------------------------------ |
||
683 | def parameter_get_value(self, prm_code: str) -> str: |
||
684 | """ |
||
685 | Select the value of a parameter. |
||
686 | |||
687 | @param str prm_code: The code of the parameter. |
||
688 | |||
689 | :rtype: * |
||
690 | """ |
||
691 | return self.execute_singleton1('select prm_value from BKC_PARAMETER where prm_code = ?', (prm_code,)) |
||
692 | |||
693 | # ------------------------------------------------------------------------------------------------------------------ |
||
694 | def parameter_update_value(self, prm_code: str, prm_value: str) -> None: |
||
695 | """ |
||
696 | Sets the value of a parameter. |
||
697 | |||
698 | @param str prm_code: The code of the parameter. |
||
699 | @param str prm_value: The value of the parameter. |
||
700 | """ |
||
701 | self.execute_none('update BKC_PARAMETER set prm_value = ? where prm_code = ?', (prm_value, prm_code)) |
||
702 | |||
703 | # ------------------------------------------------------------------------------------------------------------------ |
||
704 | def pool_delete_obsolete_original_rows(self) -> int: |
||
705 | """ |
||
706 | Deletes rows (i.e. files) from BKC_POOL that are no longer in the actual original pool. |
||
707 | """ |
||
708 | self.execute_none('delete from TMP_ID') |
||
709 | |||
710 | sql = """ |
||
711 | insert into TMP_ID(tmp_id) |
||
712 | select bpl.bpl_id |
||
713 | from BKC_POOL bpl |
||
714 | left outer join IMP_POOL imp on imp.imp_inode = bpl.bpl_inode_original and |
||
715 | imp.imp_dir = bpl.bpl_dir and |
||
716 | imp.imp_name = bpl.bpl_name |
||
717 | where bpl.bpl_inode_original is not null |
||
718 | and imp.rowid is null""" |
||
719 | self.execute_none(sql) |
||
720 | |||
721 | sql = """ |
||
722 | delete from BKC_POOL |
||
723 | where bpl_id in (select tmp_id from TMP_ID)""" |
||
724 | self.execute_none(sql) |
||
725 | |||
726 | return self.execute_singleton1('select count(*) from TMP_ID') |
||
727 | |||
728 | # ------------------------------------------------------------------------------------------------------------------ |
||
729 | def pool_delete_row(self, bpl_id: int) -> None: |
||
730 | """ |
||
731 | Deletes a row from the pool metadata. |
||
732 | |||
733 | @param int bpl_id: The rowid. |
||
734 | """ |
||
735 | self.execute_none('delete from BKC_POOL where bpl_id=?', (bpl_id,)) |
||
736 | |||
737 | # ------------------------------------------------------------------------------------------------------------------ |
||
738 | def pool_insert_new_original(self) -> None: |
||
739 | """ |
||
740 | Inserts new row into BKC_POOL based on an import from the original pool. |
||
741 | """ |
||
742 | self.execute_none('delete from TMP_POOL') |
||
743 | |||
744 | sql = """ |
||
745 | insert into TMP_POOL( tmp_inode |
||
746 | , tmp_dir |
||
747 | , tmp_name ) |
||
748 | select tmp_inode |
||
749 | , tmp_dir |
||
750 | , tmp_name |
||
751 | from |
||
752 | ( |
||
753 | select bpl_inode_original as tmp_inode |
||
754 | , bpl_dir as tmp_dir |
||
755 | , bpl_name as tmp_name |
||
756 | , 1 as src |
||
757 | from BKC_POOL |
||
758 | |||
759 | union all |
||
760 | |||
761 | select imp_inode |
||
762 | , imp_dir |
||
763 | , imp_name |
||
764 | , 2 as src |
||
765 | from IMP_POOL |
||
766 | ) t |
||
767 | group by tmp_inode |
||
768 | , tmp_dir |
||
769 | , tmp_name |
||
770 | having sum(case when src=1 then 1 else 0 end) < sum(case when src=2 then 1 else 0 end)""" |
||
771 | self.execute_none(sql) |
||
772 | |||
773 | sql = """ |
||
774 | insert into BKC_POOL( bpl_inode_original |
||
775 | , bpl_dir |
||
776 | , bpl_name ) |
||
777 | select tmp_inode |
||
778 | , tmp_dir |
||
779 | , tmp_name |
||
780 | from TMP_POOL""" |
||
781 | |||
782 | self.execute_none(sql) |
||
783 | |||
784 | # ------------------------------------------------------------------------------------------------------------------ |
||
785 | def pool_prepare_obsolete_clone_files(self) -> int: |
||
786 | """ |
||
787 | Prepares the clone pool files that are obsolete (i.e. no longer in the original pool). |
||
788 | |||
789 | :rtype: int |
||
790 | """ |
||
791 | self.execute_none('delete from TMP_CLONE_POOL_OBSOLETE') |
||
792 | |||
793 | sql = """ |
||
794 | insert into TMP_CLONE_POOL_OBSOLETE( bpl_id |
||
795 | , bpl_dir |
||
796 | , bpl_name ) |
||
797 | select bpl.bpl_id |
||
798 | , bpl.bpl_dir |
||
799 | , bpl.bpl_name |
||
800 | from BKC_POOL bpl |
||
801 | left outer join IMP_POOL imp on imp.imp_inode = bpl.bpl_inode_original and |
||
802 | imp.imp_dir = bpl.bpl_dir and |
||
803 | imp.imp_name = bpl.bpl_name |
||
804 | where bpl.bpl_inode_clone is not null |
||
805 | and imp.rowid is null""" |
||
806 | |||
807 | self.execute_none(sql) |
||
808 | |||
809 | sql = """ |
||
810 | select count(*) |
||
811 | from TMP_CLONE_POOL_OBSOLETE""" |
||
812 | |||
813 | return self.execute_singleton1(sql) |
||
814 | |||
815 | # ------------------------------------------------------------------------------------------------------------------ |
||
816 | def pool_update_by_inode_original(self, |
||
817 | bpl_inode_original: int, |
||
818 | bpl_inode_clone: int, |
||
819 | pbl_size: int, |
||
820 | pbl_mtime: int) -> None: |
||
821 | """ |
||
822 | Sets the inode number of the clone, mtime and size of a file in the pool given an inode number of a file the |
||
823 | original pool. |
||
824 | |||
825 | @param int bpl_inode_original: The inode number of a file in the original pool. |
||
826 | @param int bpl_inode_clone: The inode number of the pool file in the clone. |
||
827 | @param int pbl_size: The size of the pool file. |
||
828 | @param int pbl_mtime: The mtime of the pool file. |
||
829 | """ |
||
830 | sql = """ |
||
831 | update BKC_POOL |
||
832 | set bpl_inode_clone = ? |
||
833 | , bpl_size = ? |
||
834 | , bpl_mtime = ? |
||
835 | where bpl_inode_original = ?""" |
||
836 | |||
837 | self.execute_none(sql, (bpl_inode_clone, pbl_size, pbl_mtime, bpl_inode_original)) |
||
838 | |||
839 | # ------------------------------------------------------------------------------------------------------------------ |
||
840 | def pool_yield_obsolete_clone_files(self): |
||
841 | """ |
||
842 | Selects the clone pool files that are obsolete (i.e. no longer in the original pool). |
||
843 | """ |
||
844 | self.__connection.row_factory = DataLayer.dict_factory |
||
845 | |||
846 | sql = """ |
||
847 | select bpl_id |
||
848 | , bpl_dir |
||
849 | , bpl_name |
||
850 | from TMP_CLONE_POOL_OBSOLETE""" |
||
851 | |||
852 | cursor = self.__connection.cursor() |
||
853 | cursor.execute(sql) |
||
854 | while True: |
||
855 | rows = cursor.fetchmany(10000) |
||
856 | if not rows: |
||
857 | cursor.close() |
||
858 | return |
||
859 | yield rows |
||
860 | |||
861 | # ------------------------------------------------------------------------------------------------------------------ |
||
862 | def vacuum(self) -> None: |
||
863 | """ |
||
864 | Executes the vacuum command. |
||
865 | """ |
||
866 | self.execute_none('vacuum') |
||
867 | |||
868 | # ---------------------------------------------------------------------------------------------------------------------- |
||
869 |