bartek/smnp-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add TypeParser (handling types list - specifiers etc.)

Browse Source
This commit is contained in:
Bartłomiej Pluta
2019-07-11 23:36:52 +02:00
parent 261530eb10
commit af3cb7027a
7 changed files with 87 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
smnp/token/tools.py
View File

@@ -16,10 +16,10 @@ def regexPatternTokenizer(type, pattern):
return tokenizer
def keywordsTokenizer(type, *keywords):
def keywordsTokenizer(type, *keywords, mapKeyword=lambda x: x):
def tokenizer(input, current, line):
for keyword in keywords:
result = keywordTokenizer(type, keyword)(input, current, line)
result = keywordTokenizer(type, keyword, mapKeyword)(input, current, line)
if result[0] > 0:
return result
return (0, None)
@@ -27,10 +27,10 @@ def keywordsTokenizer(type, *keywords):
return tokenizer
def keywordTokenizer(type, keyword):
def keywordTokenizer(type, keyword, mapKeyword=lambda x: x):
def tokenizer(input, current, line):
if len(input) >= current+len(keyword) and input[current:current+len(keyword)] == keyword:
return (len(keyword), Token(type, keyword, (line, current)))
return (len(keyword), Token(type, mapKeyword(keyword), (line, current)))
return (0, None)
return tokenizer