Completed
Pull Request — master (#92)
by
unknown
24s
created

join_label_groups()   F

Complexity

Conditions 10

Size

Total Lines 35

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 10
dl 0
loc 35
rs 3.1304
c 0
b 0
f 0

How to fix   Complexity   

Complexity

Complex classes like join_label_groups() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

1
# -*- coding: utf-8 -*-
2
# -----------------------------------------------------------------------------
3
# Copyright (c) The Spyder Development Team
4
#
5
# Licensed under the terms of the MIT License
6
# (See LICENSE.txt for details)
7
# -----------------------------------------------------------------------------
8
"""Loghub filter and formatter."""
9
10
# yapf: disable
11
12
# Standard library imports
13
from collections import OrderedDict
14
import codecs
15
import re
16
import time
17
18
# Third party imports
19
from jinja2 import Template
20
21
# Local imports
22
from loghub.core.repo import GitHubRepo
23
from loghub.templates import (CHANGELOG_GROUPS_TEMPLATE_PATH,
24
                              CHANGELOG_ISSUE_GROUPS_TEMPLATE_PATH,
25
                              CHANGELOG_PR_GROUPS_TEMPLATE_PATH,
26
                              CHANGELOG_TEMPLATE_PATH,
27
                              RELEASE_GROUPS_TEMPLATE_PATH,
28
                              RELEASE_ISSUE_GROUPS_TEMPLATE_PATH,
29
                              RELEASE_PR_GROUPS_TEMPLATE_PATH,
30
                              RELEASE_TEMPLATE_PATH)
31
32
# yapf: enable
33
34
35
def filter_issues_fixed_by_prs(issues, prs):
36
    """
37
    Find related issues to prs and prs to issues that are fixed.
38
39
    This adds extra information to the issues and prs listings.
40
    """
41
    words = [
42
        'close', 'closes', 'fix', 'fixes', 'fixed', 'resolve', 'resolves',
43
        'resolved'
44
    ]
45
    pattern = re.compile(
46
        r'(?P<word>' + r'|'.join(words) + r') '
47
        r'((?P<repo>.*?)#(?P<number>\d*)|(?P<full_repo>.*)/(?P<number_2>\d*))',
48
        re.IGNORECASE, )
49
    issue_pr_map = {}
50
    pr_issue_map = {}
51
    for pr in prs:
52
        is_pr = bool(pr.get('pull_request'))
53
        if is_pr:
54
            pr_url = pr.html_url
55
            pr_number = pr.number
56
            repo_url = pr_url.split('/pull/')[0] + '/issues/'
57
            pr_issue_map[pr_url] = []
58
            body = pr.body or ''
59
            for matches in pattern.finditer(body):
60
                dic = matches.groupdict()
61
                issue_number = dic['number'] or dic['number_2'] or ''
62
                repo = dic['full_repo'] or dic['repo'] or repo_url
63
64
                # Repo name can't have spaces.
65
                if ' ' not in repo:
66
                    # In case spyder-ide/loghub#45 was for example used
67
                    if 'http' not in repo:
68
                        repo = 'https://github.com/' + repo
69
70
                    if '/issues' not in repo:
71
                        issue_url = repo + '/issues/' + issue_number
72
                    elif repo.endswith('/') and issue_number:
73
                        issue_url = repo + issue_number
74
                    elif issue_number:
75
                        issue_url = repo + '/' + issue_number
76
                    else:
77
                        issue_url = None
78
                else:
79
                    issue_url = None
80
81
                # Set the issue data
82
                issue_data = {'url': pr_url, 'text': pr_number}
83
                if issue_url is not None:
84
                    if issue_number in issue_pr_map:
85
                        issue_pr_map[issue_url].append(issue_data)
86
                    else:
87
                        issue_pr_map[issue_url] = [issue_data]
88
89
                    pr_data = {'url': issue_url, 'text': issue_number}
90
                    pr_issue_map[pr_url].append(pr_data)
91
92
            pr['loghub_related_issues'] = pr_issue_map[pr_url]
93
94
    for issue in issues:
95
        issue_url = issue.html_url
96
        if issue_url in issue_pr_map:
97
            issue['loghub_related_pulls'] = issue_pr_map[issue_url]
98
99
    # Now sort the numbers in descending order
100
    for issue in issues:
101
        related_pulls = issue.get('loghub_related_pulls', [])
102
        related_pulls = sorted(
103
            related_pulls, key=lambda p: p['url'], reverse=True)
104
        issue['loghub_related_pulls'] = related_pulls
105
106
    for pr in prs:
107
        related_issues = pr.get('loghub_related_issues', [])
108
        related_issues = sorted(
109
            related_issues, key=lambda i: i['url'], reverse=True)
110
        pr['loghub_related_issues'] = related_issues
111
112
    return issues, prs
113
114
115
def filter_prs_by_regex(issues, pr_label_regex):
116
    """Filter prs by issue regex."""
117
    filtered_prs = []
118
    pr_pattern = re.compile(pr_label_regex)
119
120
    for issue in issues:
121
        is_pr = bool(issue.get('pull_request'))
122
        labels = ' '.join(issue.get('loghub_label_names'))
123
124
        if is_pr:
125
            if pr_label_regex:
126
                pr_valid = bool(pr_pattern.search(labels))
127
                if pr_valid:
128
                    filtered_prs.append(issue)
129
            else:
130
                filtered_prs.append(issue)
131
132
    return filtered_prs
133
134
135
def filter_issues_by_regex(issues, issue_label_regex):
136
    """Filter issues by issue regex."""
137
    filtered_issues = []
138
    issue_pattern = re.compile(issue_label_regex)
139
140
    for issue in issues:
141
        is_pr = bool(issue.get('pull_request'))
142
        is_issue = not is_pr
143
        labels = ' '.join(issue.get('loghub_label_names'))
144
145
        if is_issue and issue_label_regex:
146
            issue_valid = bool(issue_pattern.search(labels))
147
            if issue_valid:
148
                filtered_issues.append(issue)
149
        elif is_issue and not issue_label_regex:
150
            filtered_issues.append(issue)
151
152
    return filtered_issues
153
154
155
def filter_issue_label_groups(issues, issue_label_groups):
156
    """Filter issues by the label groups."""
157
    grouped_filtered_issues = OrderedDict()
158
    if issue_label_groups:
159
        new_filtered_issues = []
160
        for label_group_dic in issue_label_groups:
161
            grouped_filtered_issues[label_group_dic['name']] = []
162
163
        for issue in issues:
164
            labels = issue.get('loghub_label_names')
165
            for label_group_dic in issue_label_groups:
166
                label = label_group_dic['label']
167
                name = label_group_dic['name']
168
                if label in labels:
169
                    grouped_filtered_issues[name].append(issue)
170
                    new_filtered_issues.append(issue)
171
172
        # Remove any empty groups
173
        for group, grouped_issues in grouped_filtered_issues.copy().items():
174
            if not grouped_issues:
175
                grouped_filtered_issues.pop(group)
176
    else:
177
        new_filtered_issues = issues
178
179
    return new_filtered_issues, grouped_filtered_issues
180
181
182
def join_label_groups(grouped_issues, grouped_prs, issue_label_groups,
183
                      pr_label_groups):
184
    """Combine issue and PR groups in to one dictionary.
185
186
    PR-only groups are added after all issue groups. Any groups that are
187
    shared between issues and PRs are added according to the order in the
188
    issues list of groups. This results in "label-groups" remaining in the
189
    same order originally specified even if a group does not have issues
190
    in it. Otherwise, a shared group may end up at the end of the combined
191
    dictionary and not in the order originally specified by the user.
192
193
    """
194
    issue_group_names = [x['name'] for x in issue_label_groups]
195
    pr_group_names = [x['name'] for x in pr_label_groups]
196
    shared_groups = []
197
    for idx, group_name in enumerate(issue_group_names):
198
        if len(pr_group_names) > idx and group_name == pr_group_names[idx]:
199
            shared_groups.append(group_name)
200
        else:
201
            break
202
203
    label_groups = OrderedDict()
204
    # add shared groups first
205
    for group_name in shared_groups:
206
        # make sure to copy the issue group in case it is added to
207
        label_groups[group_name] = grouped_issues.get(group_name, [])[:]
208
    # add any remaining issue groups
209
    for group_name, group in grouped_issues.items():
210
        if group_name in shared_groups:
211
            continue
212
        label_groups[group_name] = group[:]
213
    # add any remaining PR groups (extending any existing groups)
214
    for group_name, group in grouped_prs.items():
215
        label_groups.setdefault(group_name, []).extend(group)
216
    return label_groups
217
218
219
def create_changelog(repo=None,
220
                     username=None,
221
                     password=None,
222
                     token=None,
223
                     milestone=None,
224
                     since_tag=None,
225
                     until_tag=None,
226
                     branch=None,
227
                     output_format='changelog',
228
                     issue_label_regex='',
229
                     pr_label_regex='',
230
                     template_file=None,
231
                     issue_label_groups=None,
232
                     pr_label_groups=None,
233
                     batch=None,
234
                     show_prs=True):
235
    """Create changelog data for single and batched mode."""
236
    gh = GitHubRepo(
237
        username=username,
238
        password=password,
239
        token=token,
240
        repo=repo, )
241
242
    all_changelogs = []
243
    version_tag_prefix = 'v'
244
245
    if batch:
246
        # This will get all the issues, might eat up the api rate limit!
247
        base_issues = issues = gh.issues(state='closed', branch=branch)
248
        if batch == 'milestones':
249
            milestones = [i.get('title') for i in gh.milestones()]
250
            empty_items = [None] * len(milestones)
251
            items = list(zip(milestones, empty_items, empty_items))
252
        elif batch == 'tags':
253
            tags = [
254
                i.get('ref', '').replace('refs/tags/', '') for i in gh.tags()
255
            ]
256
            since_tags = [None] + tags
257
            until_tags = tags + [None]
258
            empty_items = [None] * len(since_tags)
259
            items = list(zip(empty_items, since_tags, until_tags))
260
    else:
261
        base_issues = None
262
        if milestone:
263
            items = [(milestone, None, None)]
264
        else:
265
            items = [(None, since_tag, until_tag)]
266
267
    for (milestone, since_tag, until_tag) in reversed(items):
268
        version = until_tag or None
269
        closed_at = None
270
        since = None
271
        until = None
272
273
        # Set milestone or from tag
274
        if milestone and not since_tag:
275
            milestone_data = gh.milestone(milestone)
276
            closed_at = milestone_data['closed_at']
277
            version = milestone
278
279
            if version.startswith(version_tag_prefix):
280
                version = version[len(version_tag_prefix):]
281
282
        elif not milestone and since_tag:
283
            since = gh.tag(since_tag)['tagger']['date']
284
            if until_tag:
285
                until = gh.tag(until_tag)['tagger']['date']
286
                closed_at = until
287
288
        # This returns issues and pull requests
289
        issues = gh.issues(
290
            milestone=milestone,
291
            state='closed',
292
            since=since,
293
            until=until,
294
            branch=branch,
295
            base_issues=base_issues, )
296
297
        # Filter by regex if available
298
        filtered_prs = filter_prs_by_regex(issues, pr_label_regex)
299
        filtered_issues = filter_issues_by_regex(issues, issue_label_regex)
300
301
        # If issue label grouping, filter issues
302
        new_filtered_issues, grouped_issues = filter_issue_label_groups(
303
            filtered_issues, issue_label_groups)
304
        new_filtered_prs, grouped_prs = filter_issue_label_groups(
305
            filtered_prs, pr_label_groups)
306
        label_groups = join_label_groups(grouped_issues, grouped_prs,
307
                                         issue_label_groups, pr_label_groups)
308
309
        filter_issues_fixed_by_prs(filtered_issues, filtered_prs)
310
311
        ch = render_changelog(
312
            repo,
313
            new_filtered_issues,
314
            new_filtered_prs,
315
            version,
316
            closed_at=closed_at,
317
            output_format=output_format,
318
            template_file=template_file,
319
            label_groups=label_groups,
320
            issue_label_groups=grouped_issues,
321
            pr_label_groups=grouped_prs,
322
            show_prs=show_prs)
323
324
        all_changelogs.append(ch)
325
326
    changelog = '\n'.join(all_changelogs)
327
    write_changelog(changelog=changelog)
328
329
    return changelog
330
331
332
def render_changelog(repo,
333
                     issues,
334
                     prs,
335
                     version=None,
336
                     closed_at=None,
337
                     output_format='changelog',
338
                     template_file=None,
339
                     issue_label_groups=None,
340
                     pr_label_groups=None,
341
                     label_groups=None,
342
                     show_prs=True):
343
    """Render changelog data on a jinja template."""
344
    # Header
345
    if not version:
346
        version = '<RELEASE_VERSION>'
347
348
    if closed_at:
349
        close_date = closed_at.split('T')[0]
350
    else:
351
        close_date = time.strftime("%Y/%m/%d")
352
353
    # Load template
354
    if template_file:
355
        filepath = template_file
356
    else:
357
        if issue_label_groups and pr_label_groups:
358
            if output_format == 'changelog':
359
                filepath = CHANGELOG_GROUPS_TEMPLATE_PATH
360
            else:
361
                filepath = RELEASE_GROUPS_TEMPLATE_PATH
362
        elif issue_label_groups:
363
            if output_format == 'changelog':
364
                filepath = CHANGELOG_ISSUE_GROUPS_TEMPLATE_PATH
365
            else:
366
                filepath = RELEASE_ISSUE_GROUPS_TEMPLATE_PATH
367
        elif pr_label_groups:
368
            if output_format == 'changelog':
369
                filepath = CHANGELOG_PR_GROUPS_TEMPLATE_PATH
370
            else:
371
                filepath = RELEASE_PR_GROUPS_TEMPLATE_PATH
372
        else:
373
            if output_format == 'changelog':
374
                filepath = CHANGELOG_TEMPLATE_PATH
375
            else:
376
                filepath = RELEASE_TEMPLATE_PATH
377
378
    with open(filepath) as f:
379
        data = f.read()
380
381
    repo_owner, repo_name = repo.split('/')
382
    template = Template(data)
383
    rendered = template.render(
384
        issues=issues,
385
        pull_requests=prs,
386
        version=version,
387
        close_date=close_date,
388
        repo_full_name=repo,
389
        repo_owner=repo_owner,
390
        repo_name=repo_name,
391
        label_groups=label_groups,
392
        issue_label_groups=issue_label_groups,
393
        pr_label_groups=pr_label_groups,
394
        show_prs=show_prs)
395
396
    return rendered
397
398
399
def write_changelog(changelog, output_file='CHANGELOG.temp'):
400
    """Output rendered result to prompt and file."""
401
    print('#' * 79)
402
    print(changelog)
403
    print('#' * 79)
404
405
    with codecs.open(output_file, "w", "utf-8") as f:
406
        f.write(changelog)
407