|
1
|
|
|
import os |
|
2
|
|
|
import sqlite3 |
|
3
|
|
|
from itertools import repeat |
|
4
|
|
|
from itertools import takewhile |
|
5
|
|
|
from sys import version_info |
|
6
|
|
|
|
|
7
|
|
|
if version_info[0] == 2: |
|
8
|
|
|
from itertools import izip_longest as zip_longest |
|
9
|
|
|
elif version_info[0] == 3: |
|
10
|
|
|
from itertools import zip_longest |
|
11
|
|
|
|
|
12
|
|
|
|
|
13
|
|
|
class BackedUpDict(object): |
|
14
|
|
|
def __init__(self, path): |
|
15
|
|
|
self.typemapping = {int: "int", str: "str", float: "float"} |
|
16
|
|
|
self.conn = sqlite3.connect(path) |
|
17
|
|
|
c = self.conn.cursor() |
|
18
|
|
|
for name, type_ in [("str", "text"), ("int", "integer"), ("float", "real")]: |
|
19
|
|
|
c.execute("""CREATE TABLE IF NOT EXISTS data_{}_str (key {}, value text);""".format(name, type_)) |
|
20
|
|
|
c.execute("""CREATE TABLE IF NOT EXISTS data_{}_int (key {}, value integer);""".format(name, type_)) |
|
21
|
|
|
c.execute("""CREATE TABLE IF NOT EXISTS data_{}_float (key {}, value real);""".format(name, type_)) |
|
22
|
|
|
self.conn.commit() |
|
23
|
|
|
|
|
24
|
|
|
def __getitem__(self, item): |
|
25
|
|
|
c = self.conn.cursor() |
|
26
|
|
|
res = list() |
|
27
|
|
|
for i in ["str", "int", "float"]: |
|
28
|
|
|
c.execute("""SELECT value FROM data_{}_{} where key=?;""".format(self.typemapping[type(item)], i), (item,)) |
|
29
|
|
|
res += c.fetchall() |
|
30
|
|
|
|
|
31
|
|
|
if len(res) == 1: |
|
32
|
|
|
return res[0][0] |
|
33
|
|
|
else: |
|
34
|
|
|
raise KeyError |
|
35
|
|
|
|
|
36
|
|
|
def get(self, item, default=None): |
|
37
|
|
|
try: |
|
38
|
|
|
res = self.__getitem__(item) |
|
39
|
|
|
return res |
|
40
|
|
|
except KeyError: |
|
41
|
|
|
return default |
|
42
|
|
|
|
|
43
|
|
|
def __setitem__(self, key, value): |
|
44
|
|
|
res = self.get(key, None) |
|
45
|
|
|
c = self.conn.cursor() |
|
46
|
|
|
if res is not None: |
|
47
|
|
|
c.execute("""DELETE FROM data_{}_{} WHERE key =?;""".format(self.typemapping[type(key)], |
|
48
|
|
|
self.typemapping[type(res)]), (key,)) |
|
49
|
|
|
c.execute("""INSERT INTO data_{}_{} VALUES (?,?);""".format(self.typemapping[type(key)], |
|
50
|
|
|
self.typemapping[type(value)]), (key, value)) |
|
51
|
|
|
self.conn.commit() |
|
52
|
|
|
|
|
53
|
|
|
def keys(self): |
|
54
|
|
|
c = self.conn.cursor() |
|
55
|
|
|
res = list() |
|
56
|
|
|
for i in ["str", "int", "float"]: |
|
57
|
|
|
for j in ["str", "int", "float"]: |
|
58
|
|
|
c.execute("""SELECT key FROM data_{}_{};""".format(i, j)) |
|
59
|
|
|
res += c.fetchall() |
|
60
|
|
|
return [i[0] for i in res] |
|
61
|
|
|
|
|
62
|
|
|
def __len__(self): |
|
63
|
|
|
return len(self.keys()) |
|
64
|
|
|
|
|
65
|
|
|
def __contains__(self, item): |
|
66
|
|
|
return item in self.keys() |
|
67
|
|
|
|
|
68
|
|
|
def __iter__(self): |
|
69
|
|
|
return iter(self.keys()) |
|
70
|
|
|
|
|
71
|
|
|
|
|
72
|
|
|
def obtain(filename): |
|
73
|
|
|
with open(filename, 'r') as file_: |
|
74
|
|
|
categories = [] |
|
75
|
|
|
res = [] |
|
76
|
|
|
for i, line in enumerate(file_): |
|
77
|
|
|
if i == 0: |
|
78
|
|
|
categories = list(map(str.strip, line.strip().split("\t"))) |
|
79
|
|
|
else: |
|
80
|
|
|
res.append(dict(zip_longest(categories, map(str.strip, line.split("\t")), fillvalue=""))) |
|
81
|
|
|
return res |
|
82
|
|
|
|
|
83
|
|
|
|
|
84
|
|
|
def persist(filename, dict_, mode="a", split="\t"): |
|
85
|
|
|
order = None |
|
86
|
|
|
if filename in os.listdir(os.getcwd()): |
|
87
|
|
|
with open(filename) as file_: |
|
88
|
|
|
order = file_.readline().strip().split(split) |
|
89
|
|
|
with open(filename, mode) as file_: |
|
90
|
|
|
if order is None: |
|
91
|
|
|
order = list(dict_.keys()) |
|
92
|
|
|
file_.write(split.join(order)) |
|
93
|
|
|
file_.write("\n") |
|
94
|
|
|
file_.write(split.join([str(dict_[i]) for i in order])) |
|
95
|
|
|
file_.write("\n") |
|
96
|
|
|
|
|
97
|
|
|
|
|
98
|
|
|
def count(filename): |
|
99
|
|
|
"""Credits to Michael Bacon/Quentin Pradet from Stackoverflow |
|
100
|
|
|
|
|
101
|
|
|
filename -- Name of the file, of which the amount of lines shall be counted |
|
102
|
|
|
""" |
|
103
|
|
|
f = open(filename, 'rb') |
|
104
|
|
|
bufgen = takewhile(lambda x: x, (f.raw.read(1024 * 1024) for _ in repeat(None))) |
|
105
|
|
|
return sum(buf.count(b'\n') for buf in bufgen) |
|
106
|
|
|
|
j340m3 /
python-msquaredc
