Issues (23)

backuppc_clone/DataLayer.py (1 issue)

1
import csv
2
import os
3
import sqlite3
4
from typing import List, Dict, Optional
5
6
7 View Code Duplication
class DataLayer:
0 ignored issues
show
This code seems to be duplicated in your project.
Loading history...
8
    """
9
    Provides an interface to the SQLite3 data_layer
10
    """
11
    instance = None
12
    """
13
    The singleton instance of this class.
14
15
    :type instance: backuppc_clone.DataLayer.DataLayer
16
    """
17
18
    # ------------------------------------------------------------------------------------------------------------------
19
    def __init__(self, database: str):
20
        """
21
        Object constructor.
22
23
        @param str database: Path to the SQLite database.
24
        """
25
        if DataLayer.instance is not None:
26
            raise Exception("This class is a singleton!")
27
        else:
28
            DataLayer.instance = self
29
30
        self.__database: str = database
31
        """
32
        The path to the SQLite database.
33
        """
34
35
        self.__connection: sqlite3.Connection = sqlite3.connect(':memory:')
36
        """
37
        The connection to the database.
38
        """
39
40
        self.__last_rowid: int = -1
41
        """
42
        The last rowid as returns by the last used cursor.
43
        """
44
45
        self.connect()
46
47
    # ------------------------------------------------------------------------------------------------------------------
48
    def disconnect(self) -> None:
49
        """
50
        Disconnects from the SQLite database.
51
        """
52
        self.__connection.commit()
53
        self.__connection.close()
54
55
    # ------------------------------------------------------------------------------------------------------------------
56
    def connect(self) -> None:
57
        """
58
        Connects to the SQLite database.
59
        """
60
        self.__connection = sqlite3.connect(self.__database, isolation_level="EXCLUSIVE")
61
62
        tmp_dir = os.path.join(os.path.dirname(self.__database), 'tmp')
63
        self.execute_none('pragma temp_store = 1')
64
        self.execute_none('pragma temp_store_directory = \'{}\''.format(tmp_dir))
65
        self.execute_none('pragma main.cache_size = -200000')
66
67
    # ------------------------------------------------------------------------------------------------------------------
68
    def backup_delete(self, bck_id: int) -> None:
69
        """
70
        Deletes cascading a host backup.
71
72
        @param int bck_id: The ID of the host backup.
73
        """
74
        self.backup_empty(bck_id)
75
        self.execute_none('delete from BKC_BACKUP where bck_id=?', (bck_id,))
76
77
    # ------------------------------------------------------------------------------------------------------------------
78
    def backup_empty(self, bck_id: int) -> None:
79
        """
80
        Removes the tree from a host backup.
81
82
        @param int bck_id: The ID of the host backup.
83
        """
84
        self.execute_none('delete from BKC_BACKUP_TREE where bck_id=?', (bck_id,))
85
86
    # ------------------------------------------------------------------------------------------------------------------
87
    def backup_get_all(self) -> List[Dict]:
88
        """
89
        Selects all cloned backups.
90
        """
91
        sql = """
92
select hst.hst_name
93
,      bck.bck_number
94
from       BKC_HOST   hst
95
inner join BKC_BACKUP bck  on  bck.hst_id = hst.hst_id"""
96
97
        return self.execute_rows(sql)
98
99
    # ------------------------------------------------------------------------------------------------------------------
100
    def backup_get_next(self, end_time: int) -> Dict:
101
        """
102
        Selects the next backup to clone.
103
104
        :rtype: dict|None
105
        """
106
        sql = """
107
select bob.bob_host
108
,      bob.bob_number
109
,      bob.bob_end_time
110
,      bob.bob_level
111
,      bob.bob_type
112
from            BKC_ORIGINAL_BACKUP bob
113
left outer join BKC_HOST            hst  on  hst.hst_name = bob.bob_host
114
left outer join BKC_BACKUP          bck  on  bck.hst_id     = hst.hst_id and
115
                                             bck.bck_number = bob.bob_number
116
where bck.bck_id is null 
117
and   bob.bob_version like '3.%'
118
and   bob.bob_type in ('full', 'incr')
119
and   bob.bob_end_time is not null
120
and  (bob.bob_end_time < ? or ? = -1)
121
order by bob.bob_type 
122
,        bob.bob_end_time desc
123
limit 0, 1"""
124
125
        return self.execute_row0(sql, (end_time, end_time))
126
127
    # ------------------------------------------------------------------------------------------------------------------
128
    def backup_get_obsolete(self) -> List[Dict]:
129
        """
130
        Selects obsolete host backups.
131
132
        :rtype: list[dict]
133
        """
134
        sql = """
135
select hst.hst_name
136
,      bck.bck_number
137
from            BKC_HOST            hst
138
inner join      BKC_BACKUP          bck  on  bck.hst_id = hst.hst_id
139
left outer join BKC_ORIGINAL_BACKUP bob  on  bob.bob_host   = hst.hst_name and
140
                                             bob.bob_number = bck.bck_number
141
where bob.rowid is null
142
order by hst.hst_name
143
,        bck.bck_number"""
144
145
        return self.execute_rows(sql)
146
147
    # ------------------------------------------------------------------------------------------------------------------
148
    def backup_partially_cloned(self) -> List[Dict]:
149
        """
150
        Selects partially cloned host backups.
151
152
        :rtype: list[dict]
153
        """
154
        sql = """
155
select hst.hst_name
156
,      bck.bck_number
157
from            BKC_HOST   hst
158
inner join      BKC_BACKUP bck  on  bck.hst_id = hst.hst_id
159
where ifnull(bck.bck_in_progress, 1) = 1
160
order by hst.hst_name
161
,        bck.bck_number"""
162
163
        return self.execute_rows(sql)
164
165
    # ------------------------------------------------------------------------------------------------------------------
166
    def backup_set_in_progress(self, bck_id: int, bck_in_progress: int) -> None:
167
        """
168
        Updates the in progress flag of a host backup.
169
170
        @param int bck_id: The ID of the host backup.
171
        @param int bck_in_progress: The in progress flag.
172
        """
173
        if bck_in_progress != 0:
174
            bck_in_progress = 1
175
176
        sql = """
177
update BKC_BACKUP
178
set    bck_in_progress = ? 
179
where  bck_id = ?"""
180
181
        self.execute_none(sql, (bck_in_progress, bck_id))
182
183
    # ------------------------------------------------------------------------------------------------------------------
184
    def backup_get_stats(self, bck_id: int) -> Dict:
185
        """
186
        Selects the statistics of a host backup.
187
188
        @param int bck_id: The ID of the host backup.
189
        """
190
        self.__connection.row_factory = DataLayer.dict_factory
191
192
        sql = """
193
select count(bbt_inode_original)                                   as '#files'
194
,      sum(case when bbt_inode_original is null then 1 else 0 end) as '#dirs'
195
from BKC_BACKUP_TREE
196
where bck_id = ?"""
197
198
        return self.execute_row1(sql, (bck_id,))
199
200
    # ------------------------------------------------------------------------------------------------------------------
201
    def backup_prepare_required_clone_pool_files(self, bck_id: int) -> int:
202
        """
203
        Prepares the files required for a host backup that are not yet copied from the original pool to the clone pool.
204
205
        @param int bck_id: The ID of the host backup.
206
207
        :rtype: int
208
        """
209
        self.execute_none('delete from TMP_CLONE_POOL_REQUIRED')
210
211
        sql = """
212
insert into TMP_CLONE_POOL_REQUIRED( bpl_inode_original
213
,                                    bpl_dir
214
,                                    bpl_name )
215
select distinct bpl_inode_original
216
,               bpl_dir
217
,               bpl_name
218
from       BKC_BACKUP_TREE bbt
219
inner join BKC_POOL        bpl  on bpl.bpl_inode_original = bbt.bbt_inode_original
220
where bbt.bck_id = ?
221
and   bpl.bpl_inode_clone is null"""
222
223
        self.execute_none(sql, (bck_id,))
224
225
        sql = """
226
select count(distinct bpl_inode_original)
227
from   TMP_CLONE_POOL_REQUIRED"""
228
229
        return self.execute_singleton1(sql)
230
231
    # ------------------------------------------------------------------------------------------------------------------
232
    def backup_prepare_tree(self, bck_id: int) -> int:
233
        """
234
        Selects the file entries of a host backup.
235
236
        @param int bck_id: The ID of the host backup.
237
238
        :rtype: int
239
        """
240
        self.execute_none('delete from TMP_BACKUP_TREE')
241
242
        sql = """
243
insert into TMP_BACKUP_TREE( bpl_inode_original
244
,                            bpl_dir
245
,                            bpl_name   
246
247
,                            bbt_seq                             
248
,                            bbt_inode_original
249
,                            bbt_dir
250
,                            bbt_name )
251
select bpl.bpl_inode_original
252
,      bpl.bpl_dir
253
,      bpl.bpl_name
254
255
,      bbt.bbt_seq
256
,      bbt.bbt_inode_original
257
,      bbt.bbt_dir
258
,      bbt.bbt_name
259
from            BKC_BACKUP_TREE bbt
260
left outer join BKC_POOL        bpl  on bpl.bpl_inode_original = bbt.bbt_inode_original
261
where bbt.bck_id = ?"""
262
263
        self.execute_none(sql, (bck_id,))
264
265
        sql = """
266
select count(*)
267
from   TMP_BACKUP_TREE"""
268
269
        return self.execute_singleton1(sql)
270
271
    # ------------------------------------------------------------------------------------------------------------------
272
    def backup_yield_required_clone_pool_files(self):
273
        """
274
        Selects the pool files required for a host backup that are not yet copied from the original pool to the clone
275
        pool.
276
        """
277
        self.__connection.row_factory = DataLayer.dict_factory
278
279
        sql = """
280
select bpl_inode_original
281
,      bpl_dir
282
,      bpl_name
283
from   TMP_CLONE_POOL_REQUIRED
284
order by bpl_dir
285
,        bpl_name"""
286
287
        cursor = self.__connection.cursor()
288
        cursor.execute(sql)
289
        while True:
290
            rows = cursor.fetchmany(10000)
291
            if not rows:
292
                cursor.close()
293
                return
294
            yield rows
295
296
    # ------------------------------------------------------------------------------------------------------------------
297
    def backup_yield_tree(self):
298
        """
299
        Selects the file entries of a host backup.
300
        """
301
        self.__connection.row_factory = DataLayer.dict_factory
302
303
        sql = """
304
select bpl_inode_original
305
,      bpl_dir
306
,      bpl_name
307
308
,      bbt_inode_original
309
,      bbt_dir
310
,      bbt_name
311
from   TMP_BACKUP_TREE
312
order by bbt_seq
313
,        bpl_dir
314
,        bpl_name"""
315
316
        cursor = self.__connection.cursor()
317
        cursor.execute(sql)
318
        while True:
319
            rows = cursor.fetchmany(10000)
320
            if not rows:
321
                cursor.close()
322
                return
323
            yield rows
324
325
    # ------------------------------------------------------------------------------------------------------------------
326
    def commit(self) -> None:
327
        """
328
        Commits the current transaction.
329
        """
330
        self.__connection.commit()
331
332
    # ------------------------------------------------------------------------------------------------------------------
333
    @staticmethod
334
    def dict_factory(cursor: sqlite3.Cursor, old_row: List) -> Dict:
335
        """
336
        Dictionary factory for return results with dictionaries.
337
338
        @param Cursor cursor: The cursor.
339
        @param list old_row: A row from the result a query.
340
341
        :rtype: dict
342
        """
343
        new_row = {}
344
        for index, col in enumerate(cursor.description):
345
            new_row[col[0]] = old_row[index]
346
347
        return new_row
348
349
    # ------------------------------------------------------------------------------------------------------------------
350
    def execute_none(self, sql: str, *params) -> None:
351
        """
352
        Executes a SQL statement that does not select any rows
353
354
        @param str sql: The SQL calling the stored procedure.
355
        @param iterable params: The arguments for the stored procedure.
356
        """
357
        self.__connection.row_factory = None
358
359
        cursor = self.__connection.cursor()
360
        cursor.execute(sql, *params)
361
        self.__last_rowid = cursor.lastrowid
362
        cursor.close()
363
364
    # ------------------------------------------------------------------------------------------------------------------
365
    def execute_row0(self, sql: str, *params) -> Optional[Dict]:
366
        """
367
        Executes a SQL statement that selects 0 or 1 row.
368
369
        @param str sql: The SQL calling the stored procedure.
370
        @param iterable params: The arguments for the stored procedure.
371
372
        :rtype: dict
373
        """
374
        self.__connection.row_factory = DataLayer.dict_factory
375
376
        cursor = self.__connection.cursor()
377
        cursor.execute(sql, *params)
378
        rows = cursor.fetchall()
379
        self.__last_rowid = cursor.lastrowid
380
        cursor.close()
381
382
        if len(rows) == 0:
383
            return None
384
385
        return rows[0]
386
387
    # ------------------------------------------------------------------------------------------------------------------
388
    def execute_row1(self, sql: str, *params) -> Dict:
389
        """
390
        Executes a SQL statement that selects 1 row.
391
392
        @param str sql: The SQL calling the stored procedure.
393
        @param iterable params: The arguments for the stored procedure.
394
395
        :rtype: dict
396
        """
397
        self.__connection.row_factory = DataLayer.dict_factory
398
399
        cursor = self.__connection.cursor()
400
        cursor.execute(sql, *params)
401
        rows = cursor.fetchall()
402
        self.__last_rowid = cursor.lastrowid
403
        cursor.close()
404
405
        return rows[0]
406
407
    # ------------------------------------------------------------------------------------------------------------------
408
    def execute_rows(self, sql: str, *params) -> List[Dict]:
409
        """
410
        Executes a SQL statement that selects 0, 1, or more rows.
411
412
        @param str sql: The SQL calling the stored procedure.
413
        @param iterable params: The arguments for the stored procedure.
414
415
        :rtype: list[dict]
416
        """
417
        self.__connection.row_factory = DataLayer.dict_factory
418
419
        cursor = self.__connection.cursor()
420
        cursor.execute(sql, *params)
421
        rows = cursor.fetchall()
422
        self.__last_rowid = cursor.lastrowid
423
        cursor.close()
424
425
        return rows
426
427
    # ------------------------------------------------------------------------------------------------------------------
428
    def execute_singleton0(self, sql: str, *params):
429
        """
430
        Executes a SQL statement that selects 0 or 1 row with 1 column. Returns the value of selected column or None.
431
432
        @param str sql: The SQL calling the stored procedure.
433
        @param iterable params: The arguments for the stored procedure.
434
435
        :rtype: *
436
        """
437
        self.__connection.row_factory = None
438
439
        cursor = self.__connection.cursor()
440
        cursor.execute(sql, *params)
441
        row = cursor.fetchone()
442
        if row:
443
            ret = row[0]
444
        else:
445
            ret = None
446
        self.__last_rowid = cursor.lastrowid
447
        cursor.close()
448
449
        return ret
450
451
    # ------------------------------------------------------------------------------------------------------------------
452
    def execute_singleton1(self, sql: str, *params):
453
        """
454
        Executes a SQL statement that selects 1 row with 1 column. Returns the value of selected column.
455
456
        @param str sql: The SQL calling the stored procedure.
457
        @param iterable params: The arguments for the stored procedure.
458
459
        :rtype: *
460
        """
461
        self.__connection.row_factory = None
462
463
        cursor = self.__connection.cursor()
464
        cursor.execute(sql, *params)
465
        ret = cursor.fetchone()[0]
466
        self.__last_rowid = cursor.lastrowid
467
        cursor.close()
468
469
        return ret
470
471
    # ------------------------------------------------------------------------------------------------------------------
472
    def get_host_id(self, hostname: str) -> int:
473
        """
474
        Returns the ID of a host. If the host does not exists it will be inserted.
475
476
        @param str hostname: The name of the host.
477
478
        :rtype: int
479
        """
480
        hst_id = self.execute_singleton0('select hst_id from BKC_HOST where hst_name = ?', (hostname,))
481
        if hst_id is None:
482
            self.execute_none('insert into BKC_HOST(hst_name) values(?)', (hostname,))
483
            hst_id = self.__last_rowid
484
485
        return hst_id
486
487
    # ------------------------------------------------------------------------------------------------------------------
488
    def get_bck_id(self, hst_id: int, bck_number: int) -> int:
489
        """
490
        Returns the ID of a host backup. If the backup does not exist it will be inserted.
491
492
        @param int hst_id: The ID of the host.
493
        @param int bck_number: The number of the backup.
494
495
        :rtype: int
496
        """
497
        sql = """
498
select bck_id 
499
from   BKC_BACKUP 
500
where  hst_id     = ?
501
and    bck_number = ?"""
502
503
        bck_id = self.execute_singleton0(sql, (hst_id, bck_number))
504
        if bck_id is None:
505
            self.execute_none('insert into BKC_BACKUP(hst_id, bck_number) values(?, ?)', (hst_id, bck_number))
506
            bck_id = self.__last_rowid
507
508
        return bck_id
509
510
    # ------------------------------------------------------------------------------------------------------------------
511
    def host_delete(self, host: str) -> None:
512
        """
513
        Deletes cascading a host.
514
515
        @param str host: The name of the host.
516
        """
517
        sql = """
518
select bck_id
519
from       BKC_HOST   hst
520
inner join BKC_BACKUP bck  on  bck.hst_id = hst.hst_id
521
where hst.hst_name = ?"""
522
523
        rows = self.execute_rows(sql, (host,))
524
        for row in rows:
525
            self.backup_delete(row['bck_id'])
526
527
        self.execute_none('delete from BKC_HOST where hst_name=?', (host,))
528
529
    # ------------------------------------------------------------------------------------------------------------------
530
    def host_get_obsolete(self) -> List[Dict]:
531
        """
532
        Selects hosts that are cloned but no longer in the original pool.
533
534
        :rtype: list[dict]
535
        """
536
        sql = """
537
select hst.hst_name
538
from            BKC_HOST            hst
539
left outer join BKC_ORIGINAL_BACKUP bob  on  bob.bob_host = hst.hst_name
540
where bob.rowid is null 
541
order by hst.hst_name"""
542
543
        return self.execute_rows(sql)
544
545
    # ------------------------------------------------------------------------------------------------------------------
546
    def import_csv(self,
547
                   table_name: str,
548
                   column_names: List[str],
549
                   path: str,
550
                   truncate: bool = True,
551
                   defaults: Dict = None):
552
        """
553
        Import a CSV file into a table.
554
555
        @param str table_name: The name of the table.
556
        @param list column_names: The columns names.
557
        @param str path: The path to the CSV file.
558
        @param bool truncate: If True the table will be truncated first.
559
        @param dict[str,*]|None defaults: The default values for columns not in the CSV file.
560
        """
561
        if truncate:
562
            self.execute_none('delete from {}'.format(table_name))
563
564
        default_values = []
565
        if defaults:
566
            for column_name in defaults:
567
                column_names.append(column_name)
568
                default_values.append(defaults[column_name])
569
570
        place_holders = []
571
        for _ in range(0, len(column_names)):
572
            place_holders.append('?')
573
574
        sql = 'insert into {}({}) values ({})'.format(table_name, ', '.join(column_names), ', '.join(place_holders))
575
        cursor = self.__connection.cursor()
576
        rows = []
577
        with open(path, 'r') as csv_file:
578
            csv_reader = csv.reader(csv_file)
579
            for row in csv_reader:
580
                # Replace empty string with None.
581
                for index, field in enumerate(row):
582
                    if field == '':
583
                        row[index] = None
584
585
                if defaults:
586
                    row.extend(default_values)
587
588
                rows.append(row)
589
590
                if len(rows) == 1000:
591
                    cursor.executemany(sql, rows)
592
                    rows = []
593
594
        if rows:
595
            cursor.executemany(sql, rows)
596
597
        cursor.close()
598
599
    # ------------------------------------------------------------------------------------------------------------------
600
    def original_backup_insert(self,
601
                               bob_host: str,
602
                               bob_number: int,
603
                               bob_end_time: str,
604
                               bob_version: int,
605
                               bob_level: int,
606
                               bob_type: str) -> None:
607
        """
608
        Inserts an original host backup.
609
        """
610
        sql = """
611
insert into BKC_ORIGINAL_BACKUP( bob_host
612
,                                bob_number
613
,                                bob_end_time
614
,                                bob_version
615
,                                bob_level
616
,                                bob_type )
617
values( ?
618
,       ?
619
,       ?
620
,       ?
621
,       ?
622
,       ? )"""
623
        self.execute_none(sql, (bob_host, bob_number, bob_end_time, bob_version, bob_level, bob_type))
624
625
    # ------------------------------------------------------------------------------------------------------------------
626
    def original_backup_get_stats(self) -> Dict:
627
        """
628
        Select statistics of the original backups.
629
630
        :rtype: dict
631
        """
632
        sql = """
633
select count(distinct bob_host) as '#hosts'
634
,      count(*)                 as '#backups'
635
from   BKC_ORIGINAL_BACKUP"""
636
637
        return self.execute_row1(sql)
638
639
    # ------------------------------------------------------------------------------------------------------------------
640
    def original_backup_truncate(self) -> None:
641
        """
642
        Truncates table BKC_ORIGINAL_BACKUP.
643
        """
644
        self.execute_none('delete from BKC_ORIGINAL_BACKUP')
645
646
    # ------------------------------------------------------------------------------------------------------------------
647
    def overview_get_stats(self) -> Dict:
648
        """
649
        Select statistics of the original backups and cloned backups.
650
651
        :rtype: dict
652
        """
653
        sql = """
654
select sum(case when cnt1=1 then 1 else 0 end)            as n_backups
655
,      sum(case when cnt1=1 and cnt2=1 then 1 else 0 end) as n_cloned_backups
656
,      sum(case when cnt1=1 and cnt2=0 then 1 else 0 end) as n_not_cloned_backups
657
,      sum(case when cnt1=0 and cnt2=1 then 1 else 0 end) as n_obsolete_cloned_backups
658
from
659
(
660
  select sum(case when src=1 then 1 else 0 end) as cnt1
661
  ,      sum(case when src=2 then 1 else 0 end) as cnt2
662
  from (
663
         select bob_host
664
         ,      bob_number
665
         ,      1 as src
666
         from BKC_ORIGINAL_BACKUP
667
668
         union all
669
670
         select hst.hst_name
671
         ,      bck.bck_number
672
         ,      2  as src
673
         from BKC_BACKUP    bck
674
         join BKC_HOST hst on hst.hst_id = bck.hst_id
675
       ) t
676
  group by bob_host
677
  ,        bob_number
678
)"""
679
680
        return self.execute_row1(sql)
681
682
    # ------------------------------------------------------------------------------------------------------------------
683
    def parameter_get_value(self, prm_code: str) -> str:
684
        """
685
        Select the value of a parameter.
686
687
        @param str prm_code: The code of the parameter.
688
689
        :rtype: *
690
        """
691
        return self.execute_singleton1('select prm_value from BKC_PARAMETER where prm_code = ?', (prm_code,))
692
693
    # ------------------------------------------------------------------------------------------------------------------
694
    def parameter_update_value(self, prm_code: str, prm_value: str) -> None:
695
        """
696
        Sets the value of a parameter.
697
698
        @param str prm_code: The code of the parameter.
699
        @param str prm_value: The value of the parameter.
700
        """
701
        self.execute_none('update BKC_PARAMETER set prm_value = ? where prm_code = ?', (prm_value, prm_code))
702
703
    # ------------------------------------------------------------------------------------------------------------------
704
    def pool_delete_obsolete_original_rows(self) -> int:
705
        """
706
        Deletes rows (i.e. files) from BKC_POOL that are no longer in the actual original pool.
707
        """
708
        self.execute_none('delete from TMP_ID')
709
710
        sql = """
711
insert into TMP_ID(tmp_id)
712
select bpl.bpl_id
713
from            BKC_POOL bpl
714
left outer join IMP_POOL imp  on  imp.imp_inode = bpl.bpl_inode_original and
715
                                  imp.imp_dir   = bpl.bpl_dir            and
716
                                  imp.imp_name  = bpl.bpl_name
717
where bpl.bpl_inode_original is not null 
718
and   imp.rowid is null"""
719
        self.execute_none(sql)
720
721
        sql = """
722
delete from BKC_POOL
723
where bpl_id in (select tmp_id from TMP_ID)"""
724
        self.execute_none(sql)
725
726
        return self.execute_singleton1('select count(*) from TMP_ID')
727
728
    # ------------------------------------------------------------------------------------------------------------------
729
    def pool_delete_row(self, bpl_id: int) -> None:
730
        """
731
        Deletes a row from the pool metadata.
732
733
        @param int bpl_id: The rowid.
734
        """
735
        self.execute_none('delete from BKC_POOL where bpl_id=?', (bpl_id,))
736
737
    # ------------------------------------------------------------------------------------------------------------------
738
    def pool_insert_new_original(self) -> None:
739
        """
740
        Inserts new row into BKC_POOL based on an import from the original pool.
741
        """
742
        self.execute_none('delete from TMP_POOL')
743
744
        sql = """
745
insert into TMP_POOL( tmp_inode
746
,                     tmp_dir
747
,                     tmp_name )
748
select tmp_inode
749
,      tmp_dir
750
,      tmp_name
751
from
752
(
753
  select bpl_inode_original as tmp_inode
754
  ,      bpl_dir            as tmp_dir
755
  ,      bpl_name           as tmp_name
756
  ,      1                  as src
757
  from   BKC_POOL
758
759
  union all
760
761
  select imp_inode
762
  ,      imp_dir
763
  ,      imp_name
764
  ,      2                  as src
765
  from   IMP_POOL
766
) t
767
group by tmp_inode
768
,        tmp_dir
769
,        tmp_name
770
having sum(case when src=1 then 1 else 0 end) < sum(case when src=2 then 1 else 0 end)"""
771
        self.execute_none(sql)
772
773
        sql = """
774
insert into BKC_POOL( bpl_inode_original
775
,                     bpl_dir
776
,                     bpl_name )
777
select tmp_inode
778
,      tmp_dir
779
,      tmp_name
780
from   TMP_POOL"""
781
782
        self.execute_none(sql)
783
784
    # ------------------------------------------------------------------------------------------------------------------
785
    def pool_prepare_obsolete_clone_files(self) -> int:
786
        """
787
        Prepares the clone pool files that are obsolete (i.e. no longer in the original pool).
788
789
        :rtype: int
790
        """
791
        self.execute_none('delete from TMP_CLONE_POOL_OBSOLETE')
792
793
        sql = """
794
insert into TMP_CLONE_POOL_OBSOLETE( bpl_id
795
,                                    bpl_dir
796
,                                    bpl_name )
797
select bpl.bpl_id
798
,      bpl.bpl_dir
799
,      bpl.bpl_name
800
from            BKC_POOL bpl
801
left outer join IMP_POOL imp  on  imp.imp_inode = bpl.bpl_inode_original and
802
                                  imp.imp_dir   = bpl.bpl_dir            and
803
                                  imp.imp_name  = bpl.bpl_name
804
where  bpl.bpl_inode_clone is not null
805
and    imp.rowid is null"""
806
807
        self.execute_none(sql)
808
809
        sql = """
810
select count(*)
811
from   TMP_CLONE_POOL_OBSOLETE"""
812
813
        return self.execute_singleton1(sql)
814
815
    # ------------------------------------------------------------------------------------------------------------------
816
    def pool_update_by_inode_original(self,
817
                                      bpl_inode_original: int,
818
                                      bpl_inode_clone: int,
819
                                      pbl_size: int,
820
                                      pbl_mtime: int) -> None:
821
        """
822
        Sets the inode number of the clone, mtime and size of a file in the pool given an inode number of a file the
823
        original pool.
824
825
        @param int bpl_inode_original: The inode number of a file in the original pool.
826
        @param int bpl_inode_clone: The inode number of the pool file in the clone.
827
        @param int pbl_size: The size of the pool file.
828
        @param int pbl_mtime: The mtime of the pool file.
829
        """
830
        sql = """
831
update BKC_POOL
832
set  bpl_inode_clone = ?
833
,    bpl_size        = ?
834
,    bpl_mtime       = ?
835
where bpl_inode_original = ?"""
836
837
        self.execute_none(sql, (bpl_inode_clone, pbl_size, pbl_mtime, bpl_inode_original))
838
839
    # ------------------------------------------------------------------------------------------------------------------
840
    def pool_yield_obsolete_clone_files(self):
841
        """
842
        Selects the clone pool files that are obsolete (i.e. no longer in the original pool).
843
        """
844
        self.__connection.row_factory = DataLayer.dict_factory
845
846
        sql = """
847
select bpl_id
848
,      bpl_dir
849
,      bpl_name
850
from   TMP_CLONE_POOL_OBSOLETE"""
851
852
        cursor = self.__connection.cursor()
853
        cursor.execute(sql)
854
        while True:
855
            rows = cursor.fetchmany(10000)
856
            if not rows:
857
                cursor.close()
858
                return
859
            yield rows
860
861
    # ------------------------------------------------------------------------------------------------------------------
862
    def vacuum(self) -> None:
863
        """
864
        Executes the vacuum command.
865
        """
866
        self.execute_none('vacuum')
867
868
# ----------------------------------------------------------------------------------------------------------------------
869