1 module parser; 2 3 4 import std.algorithm; 5 import std.array; 6 import std.conv; 7 import std.stdio; 8 import std.string; 9 10 import compressor; 11 import context; 12 import definer; 13 import lexer; 14 import tags; 15 import token; 16 import utils; 17 18 19 struct Parser { 20 struct Options { 21 CompressOptions compression = CompressOptions.defaults; 22 bool lines; 23 string[] search; 24 string[]* deps; 25 } 26 27 this(Options options) { 28 this.options = options; 29 this.settings = [ "defaultFilters": [ Setting(0, "defaults", "html") ], ]; 30 } 31 32 static string compile(string fileName, Options options) { 33 auto source = Definer.process(fileName, Definer.Options(options.lines, options.search, options.deps)); 34 //writeln(source); 35 auto parser = Parser(options); 36 37 source = parser.compileString(fileName, source) 38 .replace("`);write(`", "") // collapse consecutive writes 39 .replace("\r\n", "\n"); // convert windows to unix line endings to save some bytes 40 41 return source; 42 } 43 44 private: 45 string compileString(string sourceName, string source) { 46 try { 47 return parseSource(sourceName, source); 48 } catch(ParserException parserError) { 49 std.stdio.stderr.writeln(parserError.context.sourceName, '(', parserError.context.line, "): ", parserError.msg); 50 return null; 51 } 52 } 53 54 55 string parseSource(string sourceName, string source) { 56 auto context = new Context(sourceName, source); 57 58 Appender!string result; 59 result.reserve(64 * 1024); 60 61 const size_t end = source.length - min(Tag.OpenTag.length, Tag.CloseTag.length); 62 while (context.cursor < end) { 63 auto remaining = context.remaining(); 64 auto indexOpen = remaining.indexOf(Tag.OpenTag); 65 if (indexOpen == -1) 66 break; 67 68 if (!context.defining) 69 result.put(escaper(remaining[0..indexOpen], context)); 70 context.advance(indexOpen); 71 72 const size_t contentStart = indexOpen + Tag.OpenTag.length; 73 auto indexClose = remaining.indexOf(Tag.CloseTag, contentStart); 74 while (indexClose != -1) { 75 if (balancedQuotes(remaining[contentStart..indexClose])) 76 break; 77 78 indexClose = remaining.indexOf(Tag.CloseTag, indexClose + Tag.CloseTag.length); 79 } 80 81 if (indexClose == -1) 82 throw new ParserException(concat("missing '", cast(string)Tag.CloseTag, "' to close tag '", cast(string)Tag.OpenTag, "'"), context); 83 84 context.advance(Tag.OpenTag.length); 85 indexClose -= contentStart; 86 87 auto replaced = replacer(source[context.cursor..context.cursor + indexClose], context); 88 if (!context.defining) 89 result.put(replaced); 90 context.advance(indexClose + Tag.CloseTag.length); 91 } 92 context.expectTagClosed(); 93 94 result.put((context.cursor > 0) ? escaper(context.remaining(), context) : escaper(source, context)); 95 96 context.advance(context.remaining.length); 97 98 foreach(name, values; settings) { 99 if (values.length > 1) 100 throw new ParserException(("missing pop for '", name, "' - stack is not empty"), context); 101 } 102 103 return result.data(); 104 } 105 106 107 auto iterate(string content, Context context) { 108 context.tagOpen(content[0..1], content[1..$]); 109 content = content[1..$].strip(); 110 return concat("{foreach(", content, "){"); 111 } 112 113 114 auto close(string content, Context context) { 115 auto tag = context.tagClose(); 116 return Tag.CloseTag; 117 } 118 119 120 auto conditional(string content, Context context) { 121 context.tagOpen(content[0..1], content[1..$]); 122 content = content[1..$].strip(); 123 return concat("{if(", content, "){"); 124 } 125 126 127 auto orelse(string content, Context context) { 128 context.expectTagOpen("?", ":"); 129 content = content[1..$]; 130 if (content.length) 131 return concat("}else if(", content, "){"); 132 return "}else{"; 133 } 134 135 136 auto eval(string content, Context context) { 137 return content[1..$]; 138 } 139 140 auto filter(string content, string[] filters, Context context) { 141 foreach(i, filter; filters) { 142 filter = filter.strip; 143 // note: if you add a filter here, remember to update include() in definer.d if applicable 144 switch (filter) { 145 case "capitalize": 146 content ~= ".capitalize"; 147 break; 148 case "lower": 149 content ~= ".toLower"; 150 break; 151 case "upper": 152 content ~= ".toUpper"; 153 break; 154 case "none": 155 break; 156 case "html": 157 content ~= ".escapeHTML"; 158 break; 159 case "format_html": 160 content ~= ".formatHTML!(FormatHTMLOptions.Escape)"; 161 break; 162 case "format_html_links": 163 content ~= ".formatHTML!(FormatHTMLOptions.Escape | FormatHTMLOptions.CreateLinks)"; 164 break; 165 case "js": 166 content ~= ".escapeJS"; 167 break; 168 case "url": 169 content ~= ".encodeURI"; 170 break; 171 case "token_url": 172 content ~= format(`.appendURIParam("v", "%s")`, token.get); 173 break; 174 default: 175 throw new ParserException(concat("invalid filter '", filter, "'"), context); 176 } 177 } 178 return content; 179 } 180 181 auto translate(string content, Context context) { 182 content = content[1..$].strip; 183 184 auto filters = getSetting("defaultFilters").split(','); 185 auto pipe = content.lastIndexOf("|"); 186 if (pipe != -1) { 187 filters = content[pipe + 1..$].split(','); 188 } else { 189 pipe = content.length; 190 } 191 192 auto indexOpen = pipe; 193 auto indexClose = content.lastIndexOf(")", pipe); 194 if (indexClose != -1) { 195 indexOpen = content.lastIndexOf("(", indexClose); 196 while (indexOpen != -1) { 197 if (balanced(content[indexOpen..indexClose + 1], '(', ')')) 198 break; 199 indexOpen = content.lastIndexOf("(", indexOpen); 200 } 201 202 if (indexOpen == -1) 203 throw new ParserException("unexpected ')'", context); 204 } else { 205 indexClose = indexOpen; 206 } 207 208 auto tag = content[0..indexOpen]; 209 auto args = (indexOpen == indexClose) ? null : content[indexOpen + 1..indexClose].strip; 210 if (!args.empty) 211 args = "," ~ args; 212 213 return concat("write(", filter(concat("translate(", tag, args, ")"), filters, context), ");"); 214 } 215 216 217 auto interpolate(string content, Context context) { 218 content = content.strip(); 219 if (content.length > 0) { 220 auto filters = getSetting("defaultFilters").split(','); 221 auto pipe = content.lastIndexOf("|"); 222 auto end = (pipe != -1) ? pipe : content.length; 223 if (pipe != -1) 224 filters = content[pipe + 1..$].split(','); 225 226 content = concat("writable(", content[0..end].strip(), ")"); 227 return concat("write(", filter(content, filters, context), ");"); 228 } 229 return null; 230 } 231 232 auto define(string content, Context context) { 233 auto lex = Lexer(cast(ubyte[])content[1..$]); 234 try { 235 auto tok = lex.expect(Tok.Identifier); 236 if (tok == Tok.Identifier) { 237 if (lex.value == "set") { 238 setSetting(lex, content, context); 239 } else if (lex.value == "push") { 240 pushSetting(lex, content, context); 241 } else if (lex.value == "pop") { 242 popSetting(lex, content, context); 243 } else { 244 assert(0); 245 } 246 } 247 } catch (LexerException e) { 248 throw new ParserException(e.msg, context); 249 } 250 251 return null; 252 } 253 254 255 void pushSetting(ref Lexer lex, string content, Context context) { 256 lex.popFront; 257 lex.expect(Tok.Identifier); 258 259 auto name = lex.popValue; 260 auto pstack = name in settings; 261 if (!pstack) 262 throw new ParserException(concat("unknown setting '", name, "'"), context); 263 264 lex.expect(Tok.EndOfInput); 265 266 *pstack ~= Setting(context.line, context.sourceName, (*pstack)[$ - 1].value); 267 } 268 269 void popSetting(ref Lexer lex, string content, Context context) { 270 lex.popFront; 271 lex.expect(Tok.Identifier); 272 273 auto name = lex.popValue; 274 auto pstack = name in settings; 275 if (!pstack) 276 throw new ParserException(concat("unknown setting '", name, "'"), context); 277 278 lex.expect(Tok.EndOfInput); 279 280 if (pstack.length < 1) 281 throw new ParserException(concat("mismatching pop '", name, "' - stack is empty"), context); 282 --pstack.length; 283 } 284 285 void setSetting(ref Lexer lex, string content, Context context) { 286 lex.popFront; 287 lex.expect(Tok.Identifier); 288 289 auto name = lex.popValue; 290 auto pstack = name in settings; 291 if (!pstack) 292 throw new ParserException(concat("unknown setting '", name, "'"), context); 293 294 lex.expect(":"); 295 lex.popFront; 296 lex.expect(Tok.Literal); 297 298 auto literal = lex.literal; 299 auto value = lex.popValue; 300 301 lex.expect(Tok.EndOfInput); 302 303 if (literal == Literal.String) 304 value = value[1..$-1]; 305 306 (*pstack)[$ - 1] = Setting(context.line, context.sourceName, value); 307 } 308 309 310 string getSetting(string name) { 311 if (auto psetting = name in settings) 312 return (*psetting)[$ - 1].value; 313 throw new Exception(concat("internal: trying to get unknown setting '", name, "'")); 314 } 315 316 317 auto line(string content, Context context) { 318 auto info = content[1..$].strip; 319 assert(info[0] == '#'); 320 auto infos = info[6..$].split(' '); 321 context.sourceName = infos[1].unquoted; 322 context.line = infos[0].to!uint; 323 324 return concat("\n", content[1..$].strip, "\n"); 325 } 326 327 328 auto replacer(string content, Context context) { 329 if (content.length > 0) { 330 auto tag = content[0..1]; 331 switch(tag) { 332 case Tag.Iterate: 333 return iterate(content, context); 334 case Tag.Close: 335 return close(content, context); 336 case Tag.If: 337 return conditional(content, context); 338 case Tag.OrElse: 339 return orelse(content, context); 340 case Tag.Evaluate: 341 return eval(content, context); 342 case Tag.Translate: 343 return translate(content, context); 344 case Tag.LineInfo: 345 return line(content, context); 346 case Tag.Comment: 347 return null; 348 case Tag.Define: 349 return define(content, context); 350 case Tag.Include: 351 assert(0); 352 default: 353 return interpolate(content, context); 354 } 355 } 356 return null; 357 } 358 359 360 auto escaper(string content, Context context) { 361 if (content.length) { 362 if (options.compression) 363 content = compress(content, options.compression); 364 return concat("write(`", content.replace("`", "\\`").replace("\r", ""), "`);"); 365 } 366 return null; 367 } 368 369 private: 370 Options options; 371 372 struct Setting { 373 size_t line; 374 string source; 375 string value; 376 } 377 378 Setting[][string] settings; 379 }