bartek/smnp-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create ast package

Browse Source
This commit is contained in:
Bartłomiej Pluta
2019-07-03 11:27:51 +02:00
parent 2823fd1896
commit c8ff5ce38f
38 changed files with 622 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
smnp/token/tokenizer.py
View File

@@ -65,6 +65,7 @@ def tokenize(lines):
return TokenList(filterTokens(filters, tokens))
def combinedTokenizer(line, current, lineNumber):
for tokenizer in tokenizers:
consumedChars, token = tokenizer(line, current, lineNumber)
@@ -72,10 +73,12 @@ def combinedTokenizer(line, current, lineNumber):
return (consumedChars, token)
return (0, None)
def filterTokens(filters, tokens):
if not filters:
return tokens
return filterTokens(filters[1:], (token for token in tokens if filters[0](token)))
return list(filterTokens(filters[1:], (token for token in tokens if filters[0](token))))
__all__ = ["tokenize"]