1 module parser;
2 
3 
4 import std.algorithm;
5 import std.array;
6 import std.conv;
7 import std.stdio;
8 import std.string;
9 
10 import compressor;
11 import context;
12 import definer;
13 import lexer;
14 import tags;
15 import token;
16 import utils;
17 
18 
19 struct Parser {
20 	struct Options {
21 		CompressOptions compression = CompressOptions.defaults;
22 		bool lines;
23 		string[] search;
24 		string[]* deps;
25 	}
26 
27 	this(Options options) {
28 		this.options = options;
29 		this.settings = [ "defaultFilters": [ Setting(0, "defaults", "html") ], ];
30 	}
31 
32 	static string compile(string fileName, Options options) {
33 		auto source = Definer.process(fileName, Definer.Options(options.lines, options.search, options.deps));
34 		//writeln(source);
35 		auto parser = Parser(options);
36 
37 		source = parser.compileString(fileName, source);
38 			.replace("`);write(`", "")  // collapse consecutive writes
39 			.replace("\r\n", "\n");     // convert windows to unix line endings to save some bytes
40 
41 		return source;
42 	}
43 
44 private:
45 	string compileString(string sourceName, string source) {
46 		try {
47 			return parseSource(sourceName, source);
48 		} catch(ParserException parserError) {
49 			std.stdio.stderr.writeln(parserError.context.sourceName, '(', parserError.context.line, "): ", parserError.msg);
50 			return null;
51 		}
52 	}
53 
54 
55 	string parseSource(string sourceName, string source) {
56 		auto context = new Context(sourceName, source);
57 
58 		Appender!string result;
59 		result.reserve(64 * 1024);
60 
61 		const size_t end = source.length - min(Tag.OpenTag.length, Tag.CloseTag.length);
62 		while (context.cursor < end) {
63 			auto remaining = context.remaining();
64 			auto indexOpen = remaining.indexOf(Tag.OpenTag);
65 			if (indexOpen == -1)
66 				break;
67 
68 			if (!context.defining)
69 				result.put(escaper(remaining[0..indexOpen], context));
70 			context.advance(indexOpen);
71 
72 			const size_t contentStart = indexOpen + Tag.OpenTag.length;
73 			auto indexClose = remaining.indexOf(Tag.CloseTag, contentStart);
74 			while (indexClose != -1) {
75 				if (balancedQuotes(remaining[contentStart..indexClose]))
76 					break;
77 
78 				indexClose = remaining.indexOf(Tag.CloseTag, indexClose + Tag.CloseTag.length);
79 			}
80 
81 			if (indexClose == -1)
82 				throw new ParserException(concat("missing '", cast(string)Tag.CloseTag, "' to close tag '", cast(string)Tag.OpenTag, "'"), context);
83 
84 			context.advance(Tag.OpenTag.length);
85 			indexClose -= contentStart;
86 
87 			auto replaced = replacer(source[context.cursor..context.cursor + indexClose], context);
88 			if (!context.defining)
89 				result.put(replaced);
90 			context.advance(indexClose + Tag.CloseTag.length);
91 		}
92 		context.expectTagClosed();
93 
94 		result.put((context.cursor > 0) ? escaper(context.remaining(), context) : escaper(source, context));
95 
96 		context.advance(context.remaining.length);
97 
98 		foreach(name, values; settings) {
99 			if (values.length > 1)
100 				throw new ParserException(("missing pop for '", name, "' - stack is not empty"), context);
101 		}
102 
103 		return result.data();
104 	}
105 
106 
107 	auto iterate(string content, Context context) {
108 		context.tagOpen(content[0..1], content[1..$]);
109 		content = content[1..$].strip();
110 		return concat("{foreach(", content, "){");
111 	}
112 
113 
114 	auto close(string content, Context context) {
115 		auto tag = context.tagClose();
116 		return Tag.CloseTag;
117 	}
118 
119 
120 	auto conditional(string content, Context context) {
121 		context.tagOpen(content[0..1], content[1..$]);
122 		content = content[1..$].strip();
123 		return concat("{if(", content, "){");
124 	}
125 
126 
127 	auto orelse(string content, Context context) {
128 		context.expectTagOpen("?", ":");
129 		content = content[1..$];
130 		if (content.length)
131 			return concat("}else if(", content, "){");
132 		return "}else{";
133 	}
134 
135 
136 	auto eval(string content, Context context) {
137 		return content[1..$];
138 	}
139 
140 	auto filter(string content, string[] filters, Context context) {
141 		foreach(i, filter; filters) {
142 			filter = filter.strip;
143 			// note: if you add a filter here, remember to update include() in definer.d if applicable
144 			switch (filter) {
145 				case "capitalize":
146 					content ~= ".capitalize";
147 					break;
148 				case "lower":
149 					content ~= ".toLower";
150 					break;
151 				case "upper":
152 					content ~= ".toUpper";
153 					break;
154 				case "none":
155 					break;
156 				case "html":
157 					content ~= ".escapeHTML";
158 					break;
159 				case "format_html":
160 					content ~= ".formatHTML!(FormatHTMLOptions.Escape)";
161 					break;
162 				case "format_html_links":
163 					content ~= ".formatHTML!(FormatHTMLOptions.Escape | FormatHTMLOptions.CreateLinks)";
164 					break;
165 				case "js":
166 					content ~= ".escapeJS";
167 					break;
168 				case "url":
169 					content ~= ".encodeURI";
170 					break;
171 				case "token_url":
172 					content ~= format(`.appendURIParam("v", "%s")`, token.get);
173 					break;
174 				default:
175 					throw new ParserException(concat("invalid filter '", filter, "'"), context);
176 			}
177 		}
178 		return content;
179 	}
180 
181 	auto translate(string content, Context context) {
182 		content	= content[1..$].strip;
183 
184 		auto filters = getSetting("defaultFilters").split(',');
185 		auto pipe = content.lastIndexOf("|");
186 		if (pipe != -1) {
187 			filters = content[pipe + 1..$].split(',');
188 		} else {
189 			pipe = content.length;
190 		}
191 
192 		auto indexOpen = pipe;
193 		auto indexClose = content.lastIndexOf(")", pipe);
194 		if (indexClose != -1) {
195 			indexOpen = content.lastIndexOf("(", indexClose);
196 			while (indexOpen != -1) {
197 				if (balanced(content[indexOpen..indexClose + 1], '(', ')'))
198 					break;
199 				indexOpen = content.lastIndexOf("(", indexOpen);
200 			}
201 
202 			if (indexOpen == -1)
203 				throw new ParserException("unexpected ')'", context);
204 		} else {
205 			indexClose = indexOpen;
206 		}
207 
208 		auto tag = content[0..indexOpen];
209 		auto args = (indexOpen == indexClose) ? null : content[indexOpen + 1..indexClose].strip;
210 		if (!args.empty)
211 			args = "," ~ args;
212 
213 		return concat("write(", filter(concat("translate(", tag, args, ")"), filters, context), ");");
214 	}
215 
216 
217 	auto interpolate(string content, Context context) {
218 		content = content.strip();
219 		if (content.length > 0) {
220 			auto filters = getSetting("defaultFilters").split(',');
221 			auto pipe = content.lastIndexOf("|");
222 			auto end = (pipe != -1) ? pipe : content.length;
223 			if (pipe != -1)
224 				filters = content[pipe + 1..$].split(',');
225 
226 			content = concat("writable(", content[0..end].strip(), ")");
227 			return concat("write(", filter(content, filters, context), ");");
228 		}
229 		return null;
230 	}
231 
232 	auto define(string content, Context context) {
233 		auto lex = Lexer(cast(ubyte[])content[1..$]);
234 		try {
235 			auto tok = lex.expect(Tok.Identifier);
236 			if (tok == Tok.Identifier) {
237 				if (lex.value == "set") {
238 					setSetting(lex, content, context);
239 				} else if (lex.value == "push") {
240 					pushSetting(lex, content, context);
241 				} else if (lex.value == "pop") {
242 					popSetting(lex, content, context);
243 				} else {
244 					assert(0);
245 				}
246 			}
247 		} catch (LexerException e) {
248 			throw new ParserException(e.msg, context);
249 		}
250 
251 		return null;
252 	}
253 
254 
255 	void pushSetting(ref Lexer lex, string content, Context context) {
256 		lex.popFront;
257 		lex.expect(Tok.Identifier);
258 
259 		auto name = lex.popValue;
260 		auto pstack = name in settings;
261 		if (!pstack)
262 			throw new ParserException(concat("unknown setting '", name, "'"), context);
263 
264 		lex.expect(Tok.EndOfInput);
265 
266 		*pstack ~= Setting(context.line, context.sourceName, (*pstack)[$ - 1].value);
267 	}
268 
269 	void popSetting(ref Lexer lex, string content, Context context) {
270 		lex.popFront;
271 		lex.expect(Tok.Identifier);
272 
273 		auto name = lex.popValue;
274 		auto pstack = name in settings;
275 		if (!pstack)
276 			throw new ParserException(concat("unknown setting '", name, "'"), context);
277 
278 		lex.expect(Tok.EndOfInput);
279 
280 		if (pstack.length < 1)
281 			throw new ParserException(concat("mismatching pop '", name, "' - stack is empty"), context);
282 		--pstack.length;
283 	}
284 
285 	void setSetting(ref Lexer lex, string content, Context context) {
286 		lex.popFront;
287 		lex.expect(Tok.Identifier);
288 
289 		auto name = lex.popValue;
290 		auto pstack = name in settings;
291 		if (!pstack)
292 			throw new ParserException(concat("unknown setting '", name, "'"), context);
293 
294 		lex.expect(":");
295 		lex.popFront;
296 		lex.expect(Tok.Literal);
297 
298 		auto literal = lex.literal;
299 		auto value = lex.popValue;
300 
301 		lex.expect(Tok.EndOfInput);
302 
303 		if (literal == Literal.String)
304 			value = value[1..$-1];
305 
306 		(*pstack)[$ - 1] = Setting(context.line, context.sourceName, value);
307 	}
308 
309 
310 	string getSetting(string name) {
311 		if (auto psetting = name in settings)
312 			return (*psetting)[$ - 1].value;
313 		throw new Exception(concat("internal: trying to get unknown setting '", name, "'"));
314 	}
315 
316 
317 	auto line(string content, Context context) {
318 		auto info = content[1..$].strip;
319 		assert(info[0] == '#');
320 		auto infos = info[6..$].split(' ');
321 		context.sourceName = infos[1].unquoted;
322 		context.line = infos[0].to!uint;
323 
324 		return concat("\n", content[1..$].strip, "\n");
325 	}
326 
327 
328 	auto replacer(string content, Context context) {
329 		if (content.length > 0) {
330 			auto tag = content[0..1];
331 			switch(tag) {
332 				case Tag.Iterate:
333 					return iterate(content, context);
334 				case Tag.Close:
335 					return close(content, context);
336 				case Tag.If:
337 					return conditional(content, context);
338 				case Tag.OrElse:
339 					return orelse(content, context);
340 				case Tag.Evaluate:
341 					return eval(content, context);
342 				case Tag.Translate:
343 					return translate(content, context);
344 				case Tag.LineInfo:
345 					return line(content, context);
346 				case Tag.Comment:
347 					return null;
348 				case Tag.Define:
349 					return define(content, context);
350 				case Tag.Include:
351 					assert(0);
352 				default:
353 					return interpolate(content, context);
354 			}
355 		}
356 		return null;
357 	}
358 
359 
360 	auto escaper(string content, Context context) {
361 		if (content.length) {
362 			if (options.compression)
363 			   content = compress(content, options.compression);
364 			return concat("write(`", content.replace("`", "\\`").replace("\r", ""), "`);");
365 		}
366 		return null;
367 	}
368 
369 private:
370 	Options options;
371 
372 	struct Setting {
373 		size_t line;
374 		string source;
375 		string value;
376 	}
377 
378 	Setting[][string] settings;
379 }