diff --git a/mistletoe/span_tokenizer.py b/mistletoe/span_tokenizer.py index 106bcf35..260e6938 100644 --- a/mistletoe/span_tokenizer.py +++ b/mistletoe/span_tokenizer.py @@ -33,6 +33,9 @@ def tokenize(content, token_types, fallback_token): # there's text between our current position and the next token yield fallback_token(content[index:min_index]) if min_match_obj: # min_match_obj is not None - yield min_token_type(min_match_obj.group(1)) + yield min_token_type(_first_not_none_group(min_match_obj)) index += min_match_obj.end() # update pointer else: return # min_match_obj is None; no more tokens + +def _first_not_none_group(match_obj): + return next(group for group in match_obj.groups() if group is not None) diff --git a/test/test_span_token.py b/test/test_span_token.py index 9c423823..c0cc5b1d 100644 --- a/test/test_span_token.py +++ b/test/test_span_token.py @@ -85,6 +85,11 @@ def test_raw(self): c = span_token.RawText('some text') helpers.check_equal(self, list(t.children)[0], c) + def test_parse(self): + t = span_token.Strong('_some text_') + c = span_token.Emphasis('some text') + helpers.check_equal(self, list(t.children)[0], c) + class TestInlineCode(unittest.TestCase): def test_raw(self): t = span_token.InlineCode('some code')