Code Duplication    Length = 76-76 lines in 2 locations

pythonx/markdown_parser.py 1 location

@@ 82-157 (lines=76) @@
79
        if lower_url.startswith(scheme):
80
            return ''
81
    return escape(url, quote=True, smart_amp=False)
82
83
84
def preprocessing(text, tab=4):
85
    text = _newline_pattern.sub('\n', text)
86
    text = text.expandtabs(tab)
87
    text = text.replace('\u00a0', ' ')
88
    text = text.replace('\u2424', '\n')
89
    pattern = re.compile(r'^ +$', re.M)
90
    return pattern.sub('', text)
91
92
93
class BlockGrammar(object):
94
    """Grammars for block level tokens."""
95
96
    def_links = re.compile(
97
        r'^ *\[([^^\]]+)\]: *'  # [key]:
98
        r'<?([^\s>]+)>?'  # <link> or link
99
        r'(?: +["(]([^\n]+)[")])? *(?:\n+|$)'
100
    )
101
    def_footnotes = re.compile(
102
        r'^\[\^([^\]]+)\]: *('
103
        r'[^\n]*(?:\n+|$)'  # [^key]:
104
        r'(?: {1,}[^\n]*(?:\n+|$))*'
105
        r')'
106
    )
107
108
    newline = re.compile(r'^\n+')
109
    block_code = re.compile(r'^( {4}[^\n]+\n*)+')
110
    fences = re.compile(
111
        r'^ *(`{3,}|~{3,}) *(\S+)? *\n'  # ```lang
112
        r'([\s\S]+?)\s*'
113
        r'\1 *(?:\n+|$)'  # ```
114
    )
115
    hrule = re.compile(r'^ {0,3}[-*_](?: *[-*_]){2,} *(?:\n+|$)')
116
    heading = re.compile(r'^ *(#{1,6}) *([^\n]+?) *#* *(?:\n+|$)')
117
    lheading = re.compile(r'^([^\n]+)\n *(=|-)+ *(?:\n+|$)')
118
    block_quote = re.compile(r'^( *>[^\n]+(\n[^\n]+)*\n*)+')
119
    list_block = re.compile(
120
        r'^( *)([*+-]|\d+\.) [\s\S]+?'
121
        r'(?:'
122
        r'\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))'  # hrule
123
        r'|\n+(?=%s)'  # def links
124
        r'|\n+(?=%s)'  # def footnotes
125
        r'|\n{2,}'
126
        r'(?! )'
127
        r'(?!\1(?:[*+-]|\d+\.) )\n*'
128
        r'|'
129
        r'\s*$)' % (
130
            _pure_pattern(def_links),
131
            _pure_pattern(def_footnotes),
132
        )
133
    )
134
    list_item = re.compile(
135
        r'^(( *)(?:[*+-]|\d+\.) [^\n]*'
136
        r'(?:\n(?!\2(?:[*+-]|\d+\.) )[^\n]*)*)',
137
        flags=re.M
138
    )
139
    list_bullet = re.compile(r'^ *(?:[*+-]|\d+\.) +')
140
    paragraph = re.compile(
141
        r'^((?:[^\n]+\n?(?!'
142
        r'%s|%s|%s|%s|%s|%s|%s|%s|%s'
143
        r'))+)\n*' % (
144
            _pure_pattern(fences).replace(r'\1', r'\2'),
145
            _pure_pattern(list_block).replace(r'\1', r'\3'),
146
            _pure_pattern(hrule),
147
            _pure_pattern(heading),
148
            _pure_pattern(lheading),
149
            _pure_pattern(block_quote),
150
            _pure_pattern(def_links),
151
            _pure_pattern(def_footnotes),
152
            '<' + _block_tag,
153
        )
154
    )
155
    block_html = re.compile(
156
        r'^ *(?:%s|%s|%s) *(?:\n{2,}|\s*$)' % (
157
            r'<!--[\s\S]*?-->',
158
            r'<(%s)((?:%s)*?)>([\s\S]*?)<\/\1>' % (_block_tag, _valid_attr),
159
            r'<%s(?:%s)*?\s*\/?>' % (_block_tag, _valid_attr),
160
        )

pythonx/tests/markdown_parser.py 1 location

@@ 82-157 (lines=76) @@
79
    return pattern.sub('', text)
80
81
82
class BlockGrammar(object):
83
    """Grammars for block level tokens."""
84
85
    def_links = re.compile(
86
        r'^ *\[([^^\]]+)\]: *'  # [key]:
87
        r'<?([^\s>]+)>?'  # <link> or link
88
        r'(?: +["(]([^\n]+)[")])? *(?:\n+|$)'
89
    )
90
    def_footnotes = re.compile(
91
        r'^\[\^([^\]]+)\]: *('
92
        r'[^\n]*(?:\n+|$)'  # [^key]:
93
        r'(?: {1,}[^\n]*(?:\n+|$))*'
94
        r')'
95
    )
96
97
    newline = re.compile(r'^\n+')
98
    block_code = re.compile(r'^( {4}[^\n]+\n*)+')
99
    fences = re.compile(
100
        r'^ *(`{3,}|~{3,}) *(\S+)? *\n'  # ```lang
101
        r'([\s\S]+?)\s*'
102
        r'\1 *(?:\n+|$)'  # ```
103
    )
104
    hrule = re.compile(r'^ {0,3}[-*_](?: *[-*_]){2,} *(?:\n+|$)')
105
    heading = re.compile(r'^ *(#{1,6}) *([^\n]+?) *#* *(?:\n+|$)')
106
    lheading = re.compile(r'^([^\n]+)\n *(=|-)+ *(?:\n+|$)')
107
    block_quote = re.compile(r'^( *>[^\n]+(\n[^\n]+)*\n*)+')
108
    list_block = re.compile(
109
        r'^( *)([*+-]|\d+\.) [\s\S]+?'
110
        r'(?:'
111
        r'\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))'  # hrule
112
        r'|\n+(?=%s)'  # def links
113
        r'|\n+(?=%s)'  # def footnotes
114
        r'|\n{2,}'
115
        r'(?! )'
116
        r'(?!\1(?:[*+-]|\d+\.) )\n*'
117
        r'|'
118
        r'\s*$)' % (
119
            _pure_pattern(def_links),
120
            _pure_pattern(def_footnotes),
121
        )
122
    )
123
    list_item = re.compile(
124
        r'^(( *)(?:[*+-]|\d+\.) [^\n]*'
125
        r'(?:\n(?!\2(?:[*+-]|\d+\.) )[^\n]*)*)',
126
        flags=re.M
127
    )
128
    list_bullet = re.compile(r'^ *(?:[*+-]|\d+\.) +')
129
    paragraph = re.compile(
130
        r'^((?:[^\n]+\n?(?!'
131
        r'%s|%s|%s|%s|%s|%s|%s|%s|%s'
132
        r'))+)\n*' % (
133
            _pure_pattern(fences).replace(r'\1', r'\2'),
134
            _pure_pattern(list_block).replace(r'\1', r'\3'),
135
            _pure_pattern(hrule),
136
            _pure_pattern(heading),
137
            _pure_pattern(lheading),
138
            _pure_pattern(block_quote),
139
            _pure_pattern(def_links),
140
            _pure_pattern(def_footnotes),
141
            '<' + _block_tag,
142
        )
143
    )
144
    block_html = re.compile(
145
        r'^ *(?:%s|%s|%s) *(?:\n{2,}|\s*$)' % (
146
            r'<!--[\s\S]*?-->',
147
            r'<(%s)((?:%s)*?)>([\s\S]+?)<\/\1>' % (_block_tag, _valid_attr),
148
            r'<%s(?:%s)*?>' % (_block_tag, _valid_attr),
149
        )
150
    )
151
    table = re.compile(
152
        r'^ *\|(.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*'
153
    )
154
    nptable = re.compile(
155
        r'^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*'
156
    )
157
    text = re.compile(r'^[^\n]+')
158
159
160
class BlockLexer(object):