1 | // License: GPL. For details, see LICENSE file.
|
---|
2 | package org.openstreetmap.josm.tools.template_engine;
|
---|
3 |
|
---|
4 | import java.util.Arrays;
|
---|
5 | import java.util.HashSet;
|
---|
6 | import java.util.Set;
|
---|
7 |
|
---|
8 | /**
|
---|
9 | * This class converts a template string (stream of characters) into a stream of tokens.
|
---|
10 | *
|
---|
11 | * The result of the tokenization (also called lexical analysis) serves as input for the
|
---|
12 | * parser {@link TemplateParser}.
|
---|
13 | */
|
---|
14 | public class Tokenizer {
|
---|
15 |
|
---|
16 | public static class Token {
|
---|
17 | private final TokenType type;
|
---|
18 | private final int position;
|
---|
19 | private final String text;
|
---|
20 |
|
---|
21 | public Token(TokenType type, int position) {
|
---|
22 | this(type, position, null);
|
---|
23 | }
|
---|
24 |
|
---|
25 | public Token(TokenType type, int position, String text) {
|
---|
26 | this.type = type;
|
---|
27 | this.position = position;
|
---|
28 | this.text = text;
|
---|
29 | }
|
---|
30 |
|
---|
31 | public TokenType getType() {
|
---|
32 | return type;
|
---|
33 | }
|
---|
34 |
|
---|
35 | public int getPosition() {
|
---|
36 | return position;
|
---|
37 | }
|
---|
38 |
|
---|
39 | public String getText() {
|
---|
40 | return text;
|
---|
41 | }
|
---|
42 |
|
---|
43 | @Override
|
---|
44 | public String toString() {
|
---|
45 | return type + (text != null ? ' ' + text : "");
|
---|
46 | }
|
---|
47 | }
|
---|
48 |
|
---|
49 | public enum TokenType { CONDITION_START, VARIABLE_START, CONTEXT_SWITCH_START, END, PIPE, APOSTROPHE, TEXT, EOF }
|
---|
50 |
|
---|
51 | private final Set<Character> specialCharaters = new HashSet<>(Arrays.asList('$', '?', '{', '}', '|', '\'', '!'));
|
---|
52 |
|
---|
53 | private final String template;
|
---|
54 |
|
---|
55 | private int c;
|
---|
56 | private int index;
|
---|
57 | private Token currentToken;
|
---|
58 | private final StringBuilder text = new StringBuilder();
|
---|
59 |
|
---|
60 | /**
|
---|
61 | * Creates a new {@link Tokenizer}
|
---|
62 | * @param template the template as a user input string
|
---|
63 | */
|
---|
64 | public Tokenizer(String template) {
|
---|
65 | this.template = template;
|
---|
66 | getChar();
|
---|
67 | }
|
---|
68 |
|
---|
69 | private void getChar() {
|
---|
70 | if (index >= template.length()) {
|
---|
71 | c = -1;
|
---|
72 | } else {
|
---|
73 | c = template.charAt(index++);
|
---|
74 | }
|
---|
75 | }
|
---|
76 |
|
---|
77 | public Token nextToken() throws ParseError {
|
---|
78 | if (currentToken != null) {
|
---|
79 | Token result = currentToken;
|
---|
80 | currentToken = null;
|
---|
81 | return result;
|
---|
82 | }
|
---|
83 | int position = index;
|
---|
84 |
|
---|
85 | text.setLength(0);
|
---|
86 | switch (c) {
|
---|
87 | case -1:
|
---|
88 | return new Token(TokenType.EOF, position);
|
---|
89 | case '{':
|
---|
90 | getChar();
|
---|
91 | return new Token(TokenType.VARIABLE_START, position);
|
---|
92 | case '?':
|
---|
93 | getChar();
|
---|
94 | if (c == '{') {
|
---|
95 | getChar();
|
---|
96 | return new Token(TokenType.CONDITION_START, position);
|
---|
97 | } else
|
---|
98 | throw ParseError.unexpectedChar('{', (char) c, position);
|
---|
99 | case '!':
|
---|
100 | getChar();
|
---|
101 | if (c == '{') {
|
---|
102 | getChar();
|
---|
103 | return new Token(TokenType.CONTEXT_SWITCH_START, position);
|
---|
104 | } else
|
---|
105 | throw ParseError.unexpectedChar('{', (char) c, position);
|
---|
106 | case '}':
|
---|
107 | getChar();
|
---|
108 | return new Token(TokenType.END, position);
|
---|
109 | case '|':
|
---|
110 | getChar();
|
---|
111 | return new Token(TokenType.PIPE, position);
|
---|
112 | case '\'':
|
---|
113 | getChar();
|
---|
114 | return new Token(TokenType.APOSTROPHE, position);
|
---|
115 | default:
|
---|
116 | while (c != -1 && !specialCharaters.contains((char) c)) {
|
---|
117 | if (c == '\\') {
|
---|
118 | getChar();
|
---|
119 | if (c == 'n') {
|
---|
120 | c = '\n';
|
---|
121 | }
|
---|
122 | }
|
---|
123 | text.append((char) c);
|
---|
124 | getChar();
|
---|
125 | }
|
---|
126 | return new Token(TokenType.TEXT, position, text.toString());
|
---|
127 | }
|
---|
128 | }
|
---|
129 |
|
---|
130 | public Token lookAhead() throws ParseError {
|
---|
131 | if (currentToken == null) {
|
---|
132 | currentToken = nextToken();
|
---|
133 | }
|
---|
134 | return currentToken;
|
---|
135 | }
|
---|
136 |
|
---|
137 | public Token skip(char lastChar) {
|
---|
138 | currentToken = null;
|
---|
139 | int position = index;
|
---|
140 | StringBuilder result = new StringBuilder();
|
---|
141 | while (c != lastChar && c != -1) {
|
---|
142 | if (c == '\\') {
|
---|
143 | getChar();
|
---|
144 | }
|
---|
145 | result.append((char) c);
|
---|
146 | getChar();
|
---|
147 | }
|
---|
148 | return new Token(TokenType.TEXT, position, result.toString());
|
---|
149 | }
|
---|
150 | }
|
---|