Completed
Pull Request — master (#92)
by
unknown
01:15
created

join_label_groups()   F

Complexity

Conditions 10

Size

Total Lines 35

Duplication

Lines 0
Ratio 0 %

Importance

Changes 0
Metric Value
cc 10
dl 0
loc 35
rs 3.1304
c 0
b 0
f 0

How to fix   Complexity   

Complexity

Complex classes like join_label_groups() often do a lot of different things. To break such a class down, we need to identify a cohesive component within that class. A common approach to find such a component is to look for fields/methods that share the same prefixes, or suffixes.

Once you have determined the fields that belong together, you can apply the Extract Class refactoring. If the component makes sense as a sub-class, Extract Subclass is also a candidate, and is often faster.

1
# -*- coding: utf-8 -*-
2
# -----------------------------------------------------------------------------
3
# Copyright (c) The Spyder Development Team
4
#
5
# Licensed under the terms of the MIT License
6
# (See LICENSE.txt for details)
7
# -----------------------------------------------------------------------------
8
"""Loghub filter and formatter."""
9
10
# yapf: disable
11
12
# Standard library imports
13
from collections import OrderedDict
14
import codecs
15
import re
16
import time
17
18
# Third party imports
19
from jinja2 import Template
20
21
# Local imports
22
from loghub.core.repo import GitHubRepo
23
from loghub.templates import (CHANGELOG_GROUPS_TEMPLATE_PATH,
24
                              CHANGELOG_ISSUE_GROUPS_TEMPLATE_PATH,
25
                              CHANGELOG_PR_GROUPS_TEMPLATE_PATH,
26
                              CHANGELOG_TEMPLATE_PATH,
27
                              RELEASE_GROUPS_TEMPLATE_PATH,
28
                              RELEASE_ISSUE_GROUPS_TEMPLATE_PATH,
29
                              RELEASE_PR_GROUPS_TEMPLATE_PATH,
30
                              RELEASE_TEMPLATE_PATH)
31
32
# yapf: enable
33
34
35
def filter_issues_fixed_by_prs(issues, prs):
36
    """
37
    Find related issues to prs and prs to issues that are fixed.
38
39
    This adds extra information to the issues and prs listings.
40
    """
41
    words = [
42
        'close', 'closes', 'fix', 'fixes', 'fixed', 'resolve', 'resolves',
43
        'resolved'
44
    ]
45
    pattern = re.compile(
46
        r'(?P<word>' + r'|'.join(words) + r') '
47
        r'((?P<repo>.*?)#(?P<number>\d*)|(?P<full_repo>.*)/(?P<number_2>\d*))',
48
        re.IGNORECASE, )
49
    issue_pr_map = {}
50
    pr_issue_map = {}
51
    for pr in prs:
52
        is_pr = bool(pr.get('pull_request'))
53
        if is_pr:
54
            pr_url = pr.html_url
55
            pr_number = pr.number
56
            repo_url = pr_url.split('/pull/')[0] + '/issues/'
57
            pr_issue_map[pr_url] = []
58
            body = pr.body or ''
59
            for matches in pattern.finditer(body):
60
                dic = matches.groupdict()
61
                issue_number = dic['number'] or dic['number_2'] or ''
62
                repo = dic['full_repo'] or dic['repo'] or repo_url
63
64
                # Repo name can't have spaces.
65
                if ' ' not in repo:
66
                    # In case spyder-ide/loghub#45 was for example used
67
                    if 'http' not in repo:
68
                        repo = 'https://github.com/' + repo
69
70
                    if '/issues' not in repo:
71
                        issue_url = repo + '/issues/' + issue_number
72
                    elif repo.endswith('/') and issue_number:
73
                        issue_url = repo + issue_number
74
                    elif issue_number:
75
                        issue_url = repo + '/' + issue_number
76
                    else:
77
                        issue_url = None
78
                else:
79
                    issue_url = None
80
81
                # Set the issue data
82
                issue_data = {'url': pr_url, 'text': pr_number}
83
                if issue_url is not None:
84
                    if issue_number in issue_pr_map:
85
                        issue_pr_map[issue_url].append(issue_data)
86
                    else:
87
                        issue_pr_map[issue_url] = [issue_data]
88
89
                    pr_data = {'url': issue_url, 'text': issue_number}
90
                    pr_issue_map[pr_url].append(pr_data)
91
92
            pr['loghub_related_issues'] = pr_issue_map[pr_url]
93
94
    for issue in issues:
95
        issue_url = issue.html_url
96
        if issue_url in issue_pr_map:
97
            issue['loghub_related_pulls'] = issue_pr_map[issue_url]
98
99
    # Now sort the numbers in descending order
100
    for issue in issues:
101
        related_pulls = issue.get('loghub_related_pulls', [])
102
        related_pulls = sorted(
103
            related_pulls, key=lambda p: p['url'], reverse=True)
104
        issue['loghub_related_pulls'] = related_pulls
105
106
    for pr in prs:
107
        related_issues = pr.get('loghub_related_issues', [])
108
        related_issues = sorted(
109
            related_issues, key=lambda i: i['url'], reverse=True)
110
        pr['loghub_related_issues'] = related_issues
111
112
    return issues, prs
113
114
115
def filter_prs_by_regex(issues, pr_label_regex):
116
    """Filter prs by issue regex."""
117
    filtered_prs = []
118
    pr_pattern = re.compile(pr_label_regex)
119
120
    for issue in issues:
121
        is_pr = bool(issue.get('pull_request'))
122
        labels = ' '.join(issue.get('loghub_label_names'))
123
124
        if is_pr:
125
            if pr_label_regex:
126
                pr_valid = bool(pr_pattern.search(labels))
127
                if pr_valid:
128
                    filtered_prs.append(issue)
129
            else:
130
                filtered_prs.append(issue)
131
132
    return filtered_prs
133
134
135
def filter_issues_by_regex(issues, issue_label_regex):
136
    """Filter issues by issue regex."""
137
    filtered_issues = []
138
    issue_pattern = re.compile(issue_label_regex)
139
140
    for issue in issues:
141
        is_pr = bool(issue.get('pull_request'))
142
        is_issue = not is_pr
143
        labels = ' '.join(issue.get('loghub_label_names'))
144
145
        if is_issue and issue_label_regex:
146
            issue_valid = bool(issue_pattern.search(labels))
147
            if issue_valid:
148
                filtered_issues.append(issue)
149
        elif is_issue and not issue_label_regex:
150
            filtered_issues.append(issue)
151
152
    return filtered_issues
153
154
155
def filter_issue_label_groups(issues, issue_label_groups):
156
    """Filter issues by the label groups."""
157
    grouped_filtered_issues = OrderedDict()
158
    if issue_label_groups:
159
        new_filtered_issues = []
160
        for label_group_dic in issue_label_groups:
161
            grouped_filtered_issues[label_group_dic['name']] = []
162
163
        for issue in issues:
164
            labels = issue.get('loghub_label_names')
165
            for label_group_dic in issue_label_groups:
166
                label = label_group_dic['label']
167
                name = label_group_dic['name']
168
                if label in labels:
169
                    grouped_filtered_issues[name].append(issue)
170
                    new_filtered_issues.append(issue)
171
172
        # Remove any empty groups
173
        for group, grouped_issues in grouped_filtered_issues.copy().items():
174
            if not grouped_issues:
175
                grouped_filtered_issues.pop(group)
176
    else:
177
        new_filtered_issues = issues
178
179
    return new_filtered_issues, grouped_filtered_issues
180
181
182
def join_label_groups(grouped_issues, grouped_prs, issue_label_groups,
183
                      pr_label_groups):
184
    """Combine issue and PR groups in to one dictionary.
185
186
    PR-only groups are added after all issue groups. Any groups that are
187
    shared between issues and PRs are added according to the order in the
188
    issues list of groups. This results in "label-groups" remaining in the
189
    same order originally specified even if a group does not have issues
190
    in it. Otherwise, a shared group may end up at the end of the combined
191
    dictionary and not in the order originally specified by the user.
192
193
    """
194
    issue_group_names = [x['name'] for x in issue_label_groups]
195
    pr_group_names = [x['name'] for x in pr_label_groups]
196
    shared_groups = []
197
    for idx, group_name in enumerate(issue_group_names):
198
        if len(pr_group_names) > idx and group_name == pr_group_names[idx]:
199
            shared_groups.append(group_name)
200
        else:
201
            break
202
203
    label_groups = OrderedDict()
204
    # add shared groups first
205
    for group_name in shared_groups:
206
        # make sure to copy the issue group in case it is added to
207
        label_groups[group_name] = grouped_issues.get(group_name, [])[:]
208
    # add any remaining issue groups
209
    for group_name, group in grouped_issues.items():
210
        if group_name in shared_groups:
211
            continue
212
        label_groups[group_name] = group[:]
213
    # add any remaining PR groups (extending any existing groups)
214
    for group_name, group in grouped_prs.items():
215
        label_groups.setdefault(group_name, []).extend(group)
216
    return label_groups
217
218
219
def create_changelog(repo=None,
220
                     username=None,
221
                     password=None,
222
                     token=None,
223
                     milestone=None,
224
                     since_tag=None,
225
                     until_tag=None,
226
                     branch=None,
227
                     output_format='changelog',
228
                     issue_label_regex='',
229
                     pr_label_regex='',
230
                     template_file=None,
231
                     issue_label_groups=None,
232
                     pr_label_groups=None,
233
                     batch=None,
234
                     show_prs=True):
235
    """Create changelog data for single and batched mode."""
236
    if issue_label_groups is None:
237
        issue_label_groups = []
238
    if pr_label_groups is None:
239
        pr_label_groups = []
240
241
    gh = GitHubRepo(
242
        username=username,
243
        password=password,
244
        token=token,
245
        repo=repo, )
246
247
    all_changelogs = []
248
    version_tag_prefix = 'v'
249
250
    if batch:
251
        # This will get all the issues, might eat up the api rate limit!
252
        base_issues = issues = gh.issues(state='closed', branch=branch)
253
        if batch == 'milestones':
254
            milestones = [i.get('title') for i in gh.milestones()]
255
            empty_items = [None] * len(milestones)
256
            items = list(zip(milestones, empty_items, empty_items))
257
        elif batch == 'tags':
258
            tags = [
259
                i.get('ref', '').replace('refs/tags/', '') for i in gh.tags()
260
            ]
261
            since_tags = [None] + tags
262
            until_tags = tags + [None]
263
            empty_items = [None] * len(since_tags)
264
            items = list(zip(empty_items, since_tags, until_tags))
265
    else:
266
        base_issues = None
267
        if milestone:
268
            items = [(milestone, None, None)]
269
        else:
270
            items = [(None, since_tag, until_tag)]
271
272
    for (milestone, since_tag, until_tag) in reversed(items):
273
        version = until_tag or None
274
        closed_at = None
275
        since = None
276
        until = None
277
278
        # Set milestone or from tag
279
        if milestone and not since_tag:
280
            milestone_data = gh.milestone(milestone)
281
            closed_at = milestone_data['closed_at']
282
            version = milestone
283
284
            if version.startswith(version_tag_prefix):
285
                version = version[len(version_tag_prefix):]
286
287
        elif not milestone and since_tag:
288
            since = gh.tag(since_tag)['tagger']['date']
289
            if until_tag:
290
                until = gh.tag(until_tag)['tagger']['date']
291
                closed_at = until
292
293
        # This returns issues and pull requests
294
        issues = gh.issues(
295
            milestone=milestone,
296
            state='closed',
297
            since=since,
298
            until=until,
299
            branch=branch,
300
            base_issues=base_issues, )
301
302
        # Filter by regex if available
303
        filtered_prs = filter_prs_by_regex(issues, pr_label_regex)
304
        filtered_issues = filter_issues_by_regex(issues, issue_label_regex)
305
306
        # If issue label grouping, filter issues
307
        new_filtered_issues, grouped_issues = filter_issue_label_groups(
308
            filtered_issues, issue_label_groups)
309
        new_filtered_prs, grouped_prs = filter_issue_label_groups(
310
            filtered_prs, pr_label_groups)
311
        label_groups = join_label_groups(grouped_issues, grouped_prs,
312
                                         issue_label_groups, pr_label_groups)
313
314
        filter_issues_fixed_by_prs(filtered_issues, filtered_prs)
315
316
        ch = render_changelog(
317
            repo,
318
            new_filtered_issues,
319
            new_filtered_prs,
320
            version,
321
            closed_at=closed_at,
322
            output_format=output_format,
323
            template_file=template_file,
324
            label_groups=label_groups,
325
            issue_label_groups=grouped_issues,
326
            pr_label_groups=grouped_prs,
327
            show_prs=show_prs)
328
329
        all_changelogs.append(ch)
330
331
    changelog = '\n'.join(all_changelogs)
332
    write_changelog(changelog=changelog)
333
334
    return changelog
335
336
337
def render_changelog(repo,
338
                     issues,
339
                     prs,
340
                     version=None,
341
                     closed_at=None,
342
                     output_format='changelog',
343
                     template_file=None,
344
                     issue_label_groups=None,
345
                     pr_label_groups=None,
346
                     label_groups=None,
347
                     show_prs=True):
348
    """Render changelog data on a jinja template."""
349
    # Header
350
    if not version:
351
        version = '<RELEASE_VERSION>'
352
353
    if closed_at:
354
        close_date = closed_at.split('T')[0]
355
    else:
356
        close_date = time.strftime("%Y/%m/%d")
357
358
    # Load template
359
    if template_file:
360
        filepath = template_file
361
    else:
362
        if issue_label_groups and pr_label_groups:
363
            if output_format == 'changelog':
364
                filepath = CHANGELOG_GROUPS_TEMPLATE_PATH
365
            else:
366
                filepath = RELEASE_GROUPS_TEMPLATE_PATH
367
        elif issue_label_groups:
368
            if output_format == 'changelog':
369
                filepath = CHANGELOG_ISSUE_GROUPS_TEMPLATE_PATH
370
            else:
371
                filepath = RELEASE_ISSUE_GROUPS_TEMPLATE_PATH
372
        elif pr_label_groups:
373
            if output_format == 'changelog':
374
                filepath = CHANGELOG_PR_GROUPS_TEMPLATE_PATH
375
            else:
376
                filepath = RELEASE_PR_GROUPS_TEMPLATE_PATH
377
        else:
378
            if output_format == 'changelog':
379
                filepath = CHANGELOG_TEMPLATE_PATH
380
            else:
381
                filepath = RELEASE_TEMPLATE_PATH
382
383
    with open(filepath) as f:
384
        data = f.read()
385
386
    repo_owner, repo_name = repo.split('/')
387
    template = Template(data)
388
    rendered = template.render(
389
        issues=issues,
390
        pull_requests=prs,
391
        version=version,
392
        close_date=close_date,
393
        repo_full_name=repo,
394
        repo_owner=repo_owner,
395
        repo_name=repo_name,
396
        label_groups=label_groups,
397
        issue_label_groups=issue_label_groups,
398
        pr_label_groups=pr_label_groups,
399
        show_prs=show_prs)
400
401
    return rendered
402
403
404
def write_changelog(changelog, output_file='CHANGELOG.temp'):
405
    """Output rendered result to prompt and file."""
406
    print('#' * 79)
407
    print(changelog)
408
    print('#' * 79)
409
410
    with codecs.open(output_file, "w", "utf-8") as f:
411
        f.write(changelog)
412