1
|
|
|
import fnmatch |
2
|
|
|
import logging |
3
|
|
|
|
4
|
|
|
from os.path import exists, isfile, join |
5
|
|
|
|
6
|
|
|
logger = logging.getLogger('binstar.projects.upload') |
7
|
|
|
|
8
|
|
|
|
9
|
|
|
class NoIgnoreFileException(IOError): |
10
|
|
|
def __init__(self, msg, *args, **kwargs): |
11
|
|
|
self.msg = msg |
12
|
|
|
super(NoIgnoreFileException, self).__init__(msg, *args, **kwargs) |
13
|
|
|
|
14
|
|
|
|
15
|
|
|
class FilterBase(object): |
16
|
|
|
def __init__(self, *args, **kwargs): |
17
|
|
|
raise NotImplemented |
18
|
|
|
|
19
|
|
|
def can_filter(cls): |
20
|
|
|
return True |
21
|
|
|
|
22
|
|
|
def run(self, pfile): |
23
|
|
|
raise NotImplemented |
24
|
|
|
|
25
|
|
|
def update_metadata(self, metadata={}): |
26
|
|
|
raise NotImplemented |
27
|
|
|
|
28
|
|
|
|
29
|
|
|
class VCSFilter(FilterBase): |
30
|
|
|
''' |
31
|
|
|
Version Control System Filtering |
32
|
|
|
''' |
33
|
|
|
def __init__(self, pfiles, *args, **kwargs): |
34
|
|
|
self.pfiles = pfiles |
35
|
|
|
|
36
|
|
|
def run(self, pfile): |
37
|
|
|
if pfile.relativepath.startswith('.git/'): |
38
|
|
|
return False |
39
|
|
|
if pfile.relativepath.startswith('.svn/'): |
40
|
|
|
return False |
41
|
|
|
if pfile.relativepath.startswith('.hg/'): |
42
|
|
|
return False |
43
|
|
|
if pfile.relativepath.startswith('.anaconda/'): |
44
|
|
|
return False |
45
|
|
|
return True |
46
|
|
|
|
47
|
|
|
|
48
|
|
|
class FilesFilter(FilterBase): |
49
|
|
|
''' |
50
|
|
|
Ignore specific files |
51
|
|
|
''' |
52
|
|
|
ignored = ['.anaconda/project-local.yml', '.anaconda/project-local.yaml'] |
53
|
|
|
|
54
|
|
|
def __init__(self, pfiles, *args, **kwargs): |
55
|
|
|
self.pfiles = pfiles |
56
|
|
|
|
57
|
|
|
def run(self, pfile): |
58
|
|
|
return pfile.relativepath not in self.ignored |
59
|
|
|
|
60
|
|
|
|
61
|
|
|
class LargeFilesFilter(FilterBase): |
62
|
|
|
max_file_size = 2097152 |
63
|
|
|
|
64
|
|
|
def __init__(self, pfiles, *args, **kwargs): |
65
|
|
|
self.pfiles = pfiles |
66
|
|
|
|
67
|
|
|
def run(self, pfile): |
68
|
|
|
if pfile.size > self.max_file_size: |
69
|
|
|
return False |
70
|
|
|
return True |
71
|
|
|
|
72
|
|
|
|
73
|
|
|
class ProjectIgnoreFilter(FilterBase): |
74
|
|
|
def __init__(self, pfiles, *args, **kwargs): |
75
|
|
|
self._patterns = None |
76
|
|
|
self.pfiles = pfiles |
77
|
|
|
self.basepath = kwargs.get('basepath', '.') |
78
|
|
|
|
79
|
|
|
@property |
80
|
|
|
def patterns(self): |
81
|
|
|
if self._patterns is None: |
82
|
|
|
try: |
83
|
|
|
self._patterns = ignore_patterns(self.basepath) |
84
|
|
|
except NoIgnoreFileException: |
85
|
|
|
logger.debug("No ignore file") |
86
|
|
|
return self._patterns |
87
|
|
|
|
88
|
|
|
def can_filter(self): |
89
|
|
|
return self.patterns is not None and len(self.patterns) > 0 |
90
|
|
|
|
91
|
|
|
def run(self, pfile): |
92
|
|
|
for pattern in self.patterns: |
93
|
|
|
if fnmatch.fnmatch(pfile.relativepath, pattern): |
94
|
|
|
return False |
95
|
|
|
elif fnmatch.fnmatch(pfile.relativepath.split('/')[0], pattern): |
96
|
|
|
return False |
97
|
|
|
return True |
98
|
|
|
|
99
|
|
|
|
100
|
|
|
def get_ignore_file(basepath): |
101
|
|
|
ignore_files = ['.projectignore', '.gitignore'] |
102
|
|
|
for ignore_file in ignore_files: |
103
|
|
|
ignore_file_path = join(basepath, ignore_file) |
104
|
|
|
if exists(ignore_file_path) and isfile(ignore_file_path): |
105
|
|
|
logger.debug("Ignore patterns file: {}".format(ignore_file_path)) |
106
|
|
|
return ignore_file_path |
107
|
|
|
raise NoIgnoreFileException("There is no .projectignore or .gitignore") |
108
|
|
|
|
109
|
|
|
|
110
|
|
|
def ignore_patterns(basepath): |
111
|
|
|
patterns = [] |
112
|
|
|
with open(get_ignore_file(basepath)) as ifile: |
113
|
|
|
for row in ifile: |
114
|
|
|
pattern = remove_comments(clean(row)) |
115
|
|
|
patterns.append(pattern) |
116
|
|
|
return patterns |
117
|
|
|
|
118
|
|
|
|
119
|
|
|
def clean(cad): |
120
|
|
|
return cad.strip() |
121
|
|
|
|
122
|
|
|
|
123
|
|
|
def remove_comments(cad): |
124
|
|
|
return cad.split("#", 1)[0].strip() |
125
|
|
|
|
126
|
|
|
filters = [VCSFilter, ProjectIgnoreFilter, LargeFilesFilter] |
127
|
|
|
|