1 module parser;
2
3
4 import std.algorithm;
5 import std.array;
6 import std.conv;
7 import std.stdio;
8 import std.string;
9
10 import compressor;
11 import context;
12 import definer;
13 import lexer;
14 import tags;
15 import token;
16 import utils;
17
18
19 struct Parser {
20 struct Options {
21 CompressOptions compression = CompressOptions.defaults;
22 bool lines;
23 string[] search;
24 string[]* deps;
25 }
26
27 this(Options options) {
28 this.options = options;
29 this.settings = [ "defaultFilters": [ Setting(0, "defaults", "html") ], ];
30 }
31
32 static string compile(string fileName, Options options) {
33 auto source = Definer.process(fileName, Definer.Options(options.lines, options.search, options.deps));
34 //writeln(source);
35 auto parser = Parser(options);
36
37 source = parser.compileString(fileName, source)
38 .replace("`);write(`", "") // collapse consecutive writes
39 .replace("\r\n", "\n"); // convert windows to unix line endings to save some bytes
40
41 return source;
42 }
43
44 private:
45 string compileString(string sourceName, string source) {
46 try {
47 return parseSource(sourceName, source);
48 } catch(ParserException parserError) {
49 std.stdio.stderr.writeln(parserError.context.sourceName, '(', parserError.context.line, "): ", parserError.msg);
50 return null;
51 }
52 }
53
54
55 string parseSource(string sourceName, string source) {
56 auto context = new Context(sourceName, source);
57
58 Appender!string result;
59 result.reserve(64 * 1024);
60
61 const size_t end = source.length - min(Tag.OpenTag.length, Tag.CloseTag.length);
62 while (context.cursor < end) {
63 auto remaining = context.remaining();
64 auto indexOpen = remaining.indexOf(Tag.OpenTag);
65 if (indexOpen == -1)
66 break;
67
68 if (!context.defining)
69 result.put(escaper(remaining[0..indexOpen], context));
70 context.advance(indexOpen);
71
72 const size_t contentStart = indexOpen + Tag.OpenTag.length;
73 auto indexClose = remaining.indexOf(Tag.CloseTag, contentStart);
74 while (indexClose != -1) {
75 if (balancedQuotes(remaining[contentStart..indexClose]))
76 break;
77
78 indexClose = remaining.indexOf(Tag.CloseTag, indexClose + Tag.CloseTag.length);
79 }
80
81 if (indexClose == -1)
82 throw new ParserException(concat("missing '", cast(string)Tag.CloseTag, "' to close tag '", cast(string)Tag.OpenTag, "'"), context);
83
84 context.advance(Tag.OpenTag.length);
85 indexClose -= contentStart;
86
87 auto replaced = replacer(source[context.cursor..context.cursor + indexClose], context);
88 if (!context.defining)
89 result.put(replaced);
90 context.advance(indexClose + Tag.CloseTag.length);
91 }
92 context.expectTagClosed();
93
94 result.put((context.cursor > 0) ? escaper(context.remaining(), context) : escaper(source, context));
95
96 context.advance(context.remaining.length);
97
98 foreach(name, values; settings) {
99 if (values.length > 1)
100 throw new ParserException(("missing pop for '", name, "' - stack is not empty"), context);
101 }
102
103 return result.data();
104 }
105
106
107 auto iterate(string content, Context context) {
108 context.tagOpen(content[0..1], content[1..$]);
109 content = content[1..$].strip();
110 return concat("{foreach(", content, "){");
111 }
112
113
114 auto close(string content, Context context) {
115 auto tag = context.tagClose();
116 return Tag.CloseTag;
117 }
118
119
120 auto conditional(string content, Context context) {
121 context.tagOpen(content[0..1], content[1..$]);
122 content = content[1..$].strip();
123 return concat("{if(", content, "){");
124 }
125
126
127 auto orelse(string content, Context context) {
128 context.expectTagOpen("?", ":");
129 content = content[1..$];
130 if (content.length)
131 return concat("}else if(", content, "){");
132 return "}else{";
133 }
134
135
136 auto eval(string content, Context context) {
137 return content[1..$];
138 }
139
140 auto filter(string content, string[] filters, Context context) {
141 foreach(i, filter; filters) {
142 filter = filter.strip;
143 // note: if you add a filter here, remember to update include() in definer.d if applicable
144 switch (filter) {
145 case "capitalize":
146 content ~= ".capitalize";
147 break;
148 case "lower":
149 content ~= ".toLower";
150 break;
151 case "upper":
152 content ~= ".toUpper";
153 break;
154 case "none":
155 break;
156 case "html":
157 content ~= ".escapeHTML";
158 break;
159 case "format_html":
160 content ~= ".formatHTML!(FormatHTMLOptions.Escape)";
161 break;
162 case "format_html_links":
163 content ~= ".formatHTML!(FormatHTMLOptions.Escape | FormatHTMLOptions.CreateLinks)";
164 break;
165 case "js":
166 content ~= ".escapeJS";
167 break;
168 case "url":
169 content ~= ".encodeURI";
170 break;
171 case "token_url":
172 content ~= format(`.appendURIParam("v", "%s")`, token.get);
173 break;
174 default:
175 throw new ParserException(concat("invalid filter '", filter, "'"), context);
176 }
177 }
178 return content;
179 }
180
181 auto translate(string content, Context context) {
182 content = content[1..$].strip;
183
184 auto filters = getSetting("defaultFilters").split(',');
185 auto pipe = content.lastIndexOf("|");
186 if (pipe != -1) {
187 filters = content[pipe + 1..$].split(',');
188 } else {
189 pipe = content.length;
190 }
191
192 auto indexOpen = pipe;
193 auto indexClose = content.lastIndexOf(")", pipe);
194 if (indexClose != -1) {
195 indexOpen = content.lastIndexOf("(", indexClose);
196 while (indexOpen != -1) {
197 if (balanced(content[indexOpen..indexClose + 1], '(', ')'))
198 break;
199 indexOpen = content.lastIndexOf("(", indexOpen);
200 }
201
202 if (indexOpen == -1)
203 throw new ParserException("unexpected ')'", context);
204 } else {
205 indexClose = indexOpen;
206 }
207
208 auto tag = content[0..indexOpen];
209 auto args = (indexOpen == indexClose) ? null : content[indexOpen + 1..indexClose].strip;
210 if (!args.empty)
211 args = "," ~ args;
212
213 return concat("write(", filter(concat("translate(", tag, args, ")"), filters, context), ");");
214 }
215
216
217 auto interpolate(string content, Context context) {
218 content = content.strip();
219 if (content.length > 0) {
220 auto filters = getSetting("defaultFilters").split(',');
221 auto pipe = content.lastIndexOf("|");
222 auto end = (pipe != -1) ? pipe : content.length;
223 if (pipe != -1)
224 filters = content[pipe + 1..$].split(',');
225
226 content = concat("writable(", content[0..end].strip(), ")");
227 return concat("write(", filter(content, filters, context), ");");
228 }
229 return null;
230 }
231
232 auto define(string content, Context context) {
233 auto lex = Lexer(cast(ubyte[])content[1..$]);
234 try {
235 auto tok = lex.expect(Tok.Identifier);
236 if (tok == Tok.Identifier) {
237 if (lex.value == "set") {
238 setSetting(lex, content, context);
239 } else if (lex.value == "push") {
240 pushSetting(lex, content, context);
241 } else if (lex.value == "pop") {
242 popSetting(lex, content, context);
243 } else {
244 assert(0);
245 }
246 }
247 } catch (LexerException e) {
248 throw new ParserException(e.msg, context);
249 }
250
251 return null;
252 }
253
254
255 void pushSetting(ref Lexer lex, string content, Context context) {
256 lex.popFront;
257 lex.expect(Tok.Identifier);
258
259 auto name = lex.popValue;
260 auto pstack = name in settings;
261 if (!pstack)
262 throw new ParserException(concat("unknown setting '", name, "'"), context);
263
264 lex.expect(Tok.EndOfInput);
265
266 *pstack ~= Setting(context.line, context.sourceName, (*pstack)[$ - 1].value);
267 }
268
269 void popSetting(ref Lexer lex, string content, Context context) {
270 lex.popFront;
271 lex.expect(Tok.Identifier);
272
273 auto name = lex.popValue;
274 auto pstack = name in settings;
275 if (!pstack)
276 throw new ParserException(concat("unknown setting '", name, "'"), context);
277
278 lex.expect(Tok.EndOfInput);
279
280 if (pstack.length < 1)
281 throw new ParserException(concat("mismatching pop '", name, "' - stack is empty"), context);
282 --pstack.length;
283 }
284
285 void setSetting(ref Lexer lex, string content, Context context) {
286 lex.popFront;
287 lex.expect(Tok.Identifier);
288
289 auto name = lex.popValue;
290 auto pstack = name in settings;
291 if (!pstack)
292 throw new ParserException(concat("unknown setting '", name, "'"), context);
293
294 lex.expect(":");
295 lex.popFront;
296 lex.expect(Tok.Literal);
297
298 auto literal = lex.literal;
299 auto value = lex.popValue;
300
301 lex.expect(Tok.EndOfInput);
302
303 if (literal == Literal.String)
304 value = value[1..$-1];
305
306 (*pstack)[$ - 1] = Setting(context.line, context.sourceName, value);
307 }
308
309
310 string getSetting(string name) {
311 if (auto psetting = name in settings)
312 return (*psetting)[$ - 1].value;
313 throw new Exception(concat("internal: trying to get unknown setting '", name, "'"));
314 }
315
316
317 auto line(string content, Context context) {
318 auto info = content[1..$].strip;
319 assert(info[0] == '#');
320 auto infos = info[6..$].split(' ');
321 context.sourceName = infos[1].unquoted;
322 context.line = infos[0].to!uint;
323
324 return concat("\n", content[1..$].strip, "\n");
325 }
326
327
328 auto replacer(string content, Context context) {
329 if (content.length > 0) {
330 auto tag = content[0..1];
331 switch(tag) {
332 case Tag.Iterate:
333 return iterate(content, context);
334 case Tag.Close:
335 return close(content, context);
336 case Tag.If:
337 return conditional(content, context);
338 case Tag.OrElse:
339 return orelse(content, context);
340 case Tag.Evaluate:
341 return eval(content, context);
342 case Tag.Translate:
343 return translate(content, context);
344 case Tag.LineInfo:
345 return line(content, context);
346 case Tag.Comment:
347 return null;
348 case Tag.Define:
349 return define(content, context);
350 case Tag.Include:
351 assert(0);
352 default:
353 return interpolate(content, context);
354 }
355 }
356 return null;
357 }
358
359
360 auto escaper(string content, Context context) {
361 if (content.length) {
362 if (options.compression)
363 content = compress(content, options.compression);
364 return concat("write(`", content.replace("`", "\\`").replace("\r", ""), "`);");
365 }
366 return null;
367 }
368
369 private:
370 Options options;
371
372 struct Setting {
373 size_t line;
374 string source;
375 string value;
376 }
377
378 Setting[][string] settings;
379 }