pali112
/
zwiftout
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extract tokenizeText()

This commit is contained in:
Rene Saarsoo 2020-09-22 22:03:54 +03:00
parent 0a6bf2fba4
commit eb4d0a3735
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/parser/tokenizer.ts
View File

@ -150,11 +150,12 @@ const tokenizeLabeledLine = (line: string, row: number): Token[] | undefined =>
throw new ParseError(`Unknown label "${label}:"`, { row, col: 0 });
};
const tokenizeText = (line: string, row: number): TextToken[] => {
return [{ type: "text", value: line.trim(), loc: { row, col: 0 } }];
};
const tokenizeRule = (line: string, row: number): Token[] => {
return (
tokenizeLabeledLine(line, row) ||
tokenizeComment(line, row) || [{ type: "text", value: line.trim(), loc: { row, col: 0 } }]
);
return tokenizeLabeledLine(line, row) || tokenizeComment(line, row) || tokenizeText(line, row);
};
export const tokenize = (file: string): Token[] => {