Inja 3.3.0
A Template Engine for Modern C++
Loading...
Searching...
No Matches
parser.hpp
1#ifndef INCLUDE_INJA_PARSER_HPP_
2#define INCLUDE_INJA_PARSER_HPP_
3
4#include <limits>
5#include <stack>
6#include <string>
7#include <utility>
8#include <queue>
9#include <vector>
10
11#include "config.hpp"
12#include "exceptions.hpp"
13#include "function_storage.hpp"
14#include "lexer.hpp"
15#include "node.hpp"
16#include "template.hpp"
17#include "token.hpp"
18#include "utils.hpp"
19
20#include <nlohmann/json.hpp>
21
22namespace inja {
23
27class Parser {
28 const ParserConfig &config;
29
30 Lexer lexer;
31 TemplateStorage &template_storage;
32 const FunctionStorage &function_storage;
33
34 Token tok, peek_tok;
35 bool have_peek_tok {false};
36
37 size_t current_paren_level {0};
38 size_t current_bracket_level {0};
39 size_t current_brace_level {0};
40
41 nonstd::string_view json_literal_start;
42
43 BlockNode *current_block {nullptr};
44 ExpressionListNode *current_expression_list {nullptr};
45 std::stack<std::pair<FunctionNode*, size_t>> function_stack;
46 std::vector<std::shared_ptr<ExpressionNode>> arguments;
47
48 std::stack<std::shared_ptr<FunctionNode>> operator_stack;
49 std::stack<IfStatementNode*> if_statement_stack;
50 std::stack<ForStatementNode*> for_statement_stack;
51 std::stack<BlockStatementNode*> block_statement_stack;
52
53 inline void throw_parser_error(const std::string &message) {
54 INJA_THROW(ParserError(message, lexer.current_position()));
55 }
56
57 inline void get_next_token() {
58 if (have_peek_tok) {
59 tok = peek_tok;
60 have_peek_tok = false;
61 } else {
62 tok = lexer.scan();
63 }
64 }
65
66 inline void get_peek_token() {
67 if (!have_peek_tok) {
68 peek_tok = lexer.scan();
69 have_peek_tok = true;
70 }
71 }
72
73 inline void add_json_literal(const char* content_ptr) {
74 nonstd::string_view json_text(json_literal_start.data(), tok.text.data() - json_literal_start.data() + tok.text.size());
75 arguments.emplace_back(std::make_shared<LiteralNode>(json::parse(json_text), json_text.data() - content_ptr));
76 }
77
78 inline void add_operator() {
79 auto function = operator_stack.top();
80 operator_stack.pop();
81
82 for (int i = 0; i < function->number_args; ++i) {
83 function->arguments.insert(function->arguments.begin(), arguments.back());
84 arguments.pop_back();
85 }
86 arguments.emplace_back(function);
87 }
88
89 void add_to_template_storage(nonstd::string_view path, std::string& template_name) {
90 if (config.search_included_templates_in_files && template_storage.find(template_name) == template_storage.end()) {
91 // Build the relative path
92 template_name = static_cast<std::string>(path) + template_name;
93 if (template_name.compare(0, 2, "./") == 0) {
94 template_name.erase(0, 2);
95 }
96
97 if (template_storage.find(template_name) == template_storage.end()) {
98 auto include_template = Template(load_file(template_name));
99 template_storage.emplace(template_name, include_template);
100 parse_into_template(template_storage[template_name], template_name);
101 }
102 }
103 }
104
105 bool parse_expression(Template &tmpl, Token::Kind closing) {
106 while (tok.kind != closing && tok.kind != Token::Kind::Eof) {
107 // Literals
108 switch (tok.kind) {
109 case Token::Kind::String: {
110 if (current_brace_level == 0 && current_bracket_level == 0) {
111 json_literal_start = tok.text;
112 add_json_literal(tmpl.content.c_str());
113 }
114
115 } break;
116 case Token::Kind::Number: {
117 if (current_brace_level == 0 && current_bracket_level == 0) {
118 json_literal_start = tok.text;
119 add_json_literal(tmpl.content.c_str());
120 }
121
122 } break;
123 case Token::Kind::LeftBracket: {
124 if (current_brace_level == 0 && current_bracket_level == 0) {
125 json_literal_start = tok.text;
126 }
127 current_bracket_level += 1;
128
129 } break;
130 case Token::Kind::LeftBrace: {
131 if (current_brace_level == 0 && current_bracket_level == 0) {
132 json_literal_start = tok.text;
133 }
134 current_brace_level += 1;
135
136 } break;
137 case Token::Kind::RightBracket: {
138 if (current_bracket_level == 0) {
139 throw_parser_error("unexpected ']'");
140 }
141
142 current_bracket_level -= 1;
143 if (current_brace_level == 0 && current_bracket_level == 0) {
144 add_json_literal(tmpl.content.c_str());
145 }
146
147 } break;
148 case Token::Kind::RightBrace: {
149 if (current_brace_level == 0) {
150 throw_parser_error("unexpected '}'");
151 }
152
153 current_brace_level -= 1;
154 if (current_brace_level == 0 && current_bracket_level == 0) {
155 add_json_literal(tmpl.content.c_str());
156 }
157
158 } break;
159 case Token::Kind::Id: {
160 get_peek_token();
161
162 // Json Literal
163 if (tok.text == static_cast<decltype(tok.text)>("true") || tok.text == static_cast<decltype(tok.text)>("false") || tok.text == static_cast<decltype(tok.text)>("null")) {
164 if (current_brace_level == 0 && current_bracket_level == 0) {
165 json_literal_start = tok.text;
166 add_json_literal(tmpl.content.c_str());
167 }
168
169 // Operator
170 } else if (tok.text == "and" || tok.text == "or" || tok.text == "in" || tok.text == "not") {
171 goto parse_operator;
172
173 // Functions
174 } else if (peek_tok.kind == Token::Kind::LeftParen) {
175 operator_stack.emplace(std::make_shared<FunctionNode>(static_cast<std::string>(tok.text), tok.text.data() - tmpl.content.c_str()));
176 function_stack.emplace(operator_stack.top().get(), current_paren_level);
177
178 // Variables
179 } else {
180 arguments.emplace_back(std::make_shared<JsonNode>(static_cast<std::string>(tok.text), tok.text.data() - tmpl.content.c_str()));
181 }
182
183 // Operators
184 } break;
185 case Token::Kind::Equal:
186 case Token::Kind::NotEqual:
187 case Token::Kind::GreaterThan:
188 case Token::Kind::GreaterEqual:
189 case Token::Kind::LessThan:
190 case Token::Kind::LessEqual:
191 case Token::Kind::Plus:
192 case Token::Kind::Minus:
193 case Token::Kind::Times:
194 case Token::Kind::Slash:
195 case Token::Kind::Power:
196 case Token::Kind::Percent:
197 case Token::Kind::Dot: {
198
199 parse_operator:
200 FunctionStorage::Operation operation;
201 switch (tok.kind) {
202 case Token::Kind::Id: {
203 if (tok.text == "and") {
204 operation = FunctionStorage::Operation::And;
205 } else if (tok.text == "or") {
206 operation = FunctionStorage::Operation::Or;
207 } else if (tok.text == "in") {
208 operation = FunctionStorage::Operation::In;
209 } else if (tok.text == "not") {
210 operation = FunctionStorage::Operation::Not;
211 } else {
212 throw_parser_error("unknown operator in parser.");
213 }
214 } break;
215 case Token::Kind::Equal: {
216 operation = FunctionStorage::Operation::Equal;
217 } break;
218 case Token::Kind::NotEqual: {
219 operation = FunctionStorage::Operation::NotEqual;
220 } break;
221 case Token::Kind::GreaterThan: {
222 operation = FunctionStorage::Operation::Greater;
223 } break;
224 case Token::Kind::GreaterEqual: {
225 operation = FunctionStorage::Operation::GreaterEqual;
226 } break;
227 case Token::Kind::LessThan: {
228 operation = FunctionStorage::Operation::Less;
229 } break;
230 case Token::Kind::LessEqual: {
231 operation = FunctionStorage::Operation::LessEqual;
232 } break;
233 case Token::Kind::Plus: {
234 operation = FunctionStorage::Operation::Add;
235 } break;
236 case Token::Kind::Minus: {
237 operation = FunctionStorage::Operation::Subtract;
238 } break;
239 case Token::Kind::Times: {
240 operation = FunctionStorage::Operation::Multiplication;
241 } break;
242 case Token::Kind::Slash: {
243 operation = FunctionStorage::Operation::Division;
244 } break;
245 case Token::Kind::Power: {
246 operation = FunctionStorage::Operation::Power;
247 } break;
248 case Token::Kind::Percent: {
249 operation = FunctionStorage::Operation::Modulo;
250 } break;
251 case Token::Kind::Dot: {
252 operation = FunctionStorage::Operation::AtId;
253 } break;
254 default: {
255 throw_parser_error("unknown operator in parser.");
256 }
257 }
258 auto function_node = std::make_shared<FunctionNode>(operation, tok.text.data() - tmpl.content.c_str());
259
260 while (!operator_stack.empty() && ((operator_stack.top()->precedence > function_node->precedence) || (operator_stack.top()->precedence == function_node->precedence && function_node->associativity == FunctionNode::Associativity::Left)) && (operator_stack.top()->operation != FunctionStorage::Operation::ParenLeft)) {
261 add_operator();
262 }
263
264 operator_stack.emplace(function_node);
265
266 } break;
267 case Token::Kind::Comma: {
268 if (current_brace_level == 0 && current_bracket_level == 0) {
269 if (function_stack.empty()) {
270 throw_parser_error("unexpected ','");
271 }
272
273 function_stack.top().first->number_args += 1;
274 }
275
276 } break;
277 case Token::Kind::Colon: {
278 if (current_brace_level == 0 && current_bracket_level == 0) {
279 throw_parser_error("unexpected ':'");
280 }
281
282 } break;
283 case Token::Kind::LeftParen: {
284 current_paren_level += 1;
285 operator_stack.emplace(std::make_shared<FunctionNode>(FunctionStorage::Operation::ParenLeft, tok.text.data() - tmpl.content.c_str()));
286
287 get_peek_token();
288 if (peek_tok.kind == Token::Kind::RightParen) {
289 if (!function_stack.empty() && function_stack.top().second == current_paren_level - 1) {
290 function_stack.top().first->number_args = 0;
291 }
292 }
293
294 } break;
295 case Token::Kind::RightParen: {
296 current_paren_level -= 1;
297 while (!operator_stack.empty() && operator_stack.top()->operation != FunctionStorage::Operation::ParenLeft) {
298 add_operator();
299 }
300
301 if (!operator_stack.empty() && operator_stack.top()->operation == FunctionStorage::Operation::ParenLeft) {
302 operator_stack.pop();
303 }
304
305 if (!function_stack.empty() && function_stack.top().second == current_paren_level) {
306 auto func = function_stack.top().first;
307 auto function_data = function_storage.find_function(func->name, func->number_args);
308 if (function_data.operation == FunctionStorage::Operation::None) {
309 throw_parser_error("unknown function " + func->name);
310 }
311 func->operation = function_data.operation;
312 if (function_data.operation == FunctionStorage::Operation::Callback) {
313 func->callback = function_data.callback;
314 }
315
316 if (operator_stack.empty()) {
317 throw_parser_error("internal error at function " + func->name);
318 }
319
320 add_operator();
321 function_stack.pop();
322 }
323 }
324 default:
325 break;
326 }
327
328 get_next_token();
329 }
330
331 while (!operator_stack.empty()) {
332 add_operator();
333 }
334
335 if (arguments.size() == 1) {
336 current_expression_list->root = arguments[0];
337 arguments = {};
338
339 } else if (arguments.size() > 1) {
340 throw_parser_error("malformed expression");
341 }
342
343 return true;
344 }
345
346 bool parse_statement(Template &tmpl, Token::Kind closing, nonstd::string_view path) {
347 if (tok.kind != Token::Kind::Id) {
348 return false;
349 }
350
351 if (tok.text == static_cast<decltype(tok.text)>("if")) {
352 get_next_token();
353
354 auto if_statement_node = std::make_shared<IfStatementNode>(current_block, tok.text.data() - tmpl.content.c_str());
355 current_block->nodes.emplace_back(if_statement_node);
356 if_statement_stack.emplace(if_statement_node.get());
357 current_block = &if_statement_node->true_statement;
358 current_expression_list = &if_statement_node->condition;
359
360 if (!parse_expression(tmpl, closing)) {
361 return false;
362 }
363
364 } else if (tok.text == static_cast<decltype(tok.text)>("else")) {
365 if (if_statement_stack.empty()) {
366 throw_parser_error("else without matching if");
367 }
368 auto &if_statement_data = if_statement_stack.top();
369 get_next_token();
370
371 if_statement_data->has_false_statement = true;
372 current_block = &if_statement_data->false_statement;
373
374 // Chained else if
375 if (tok.kind == Token::Kind::Id && tok.text == static_cast<decltype(tok.text)>("if")) {
376 get_next_token();
377
378 auto if_statement_node = std::make_shared<IfStatementNode>(true, current_block, tok.text.data() - tmpl.content.c_str());
379 current_block->nodes.emplace_back(if_statement_node);
380 if_statement_stack.emplace(if_statement_node.get());
381 current_block = &if_statement_node->true_statement;
382 current_expression_list = &if_statement_node->condition;
383
384 if (!parse_expression(tmpl, closing)) {
385 return false;
386 }
387 }
388
389 } else if (tok.text == static_cast<decltype(tok.text)>("endif")) {
390 if (if_statement_stack.empty()) {
391 throw_parser_error("endif without matching if");
392 }
393
394 // Nested if statements
395 while (if_statement_stack.top()->is_nested) {
396 if_statement_stack.pop();
397 }
398
399 auto &if_statement_data = if_statement_stack.top();
400 get_next_token();
401
402 current_block = if_statement_data->parent;
403 if_statement_stack.pop();
404
405 } else if (tok.text == static_cast<decltype(tok.text)>("block")) {
406 get_next_token();
407
408 if (tok.kind != Token::Kind::Id) {
409 throw_parser_error("expected block name, got '" + tok.describe() + "'");
410 }
411
412 const std::string block_name = static_cast<std::string>(tok.text);
413
414 auto block_statement_node = std::make_shared<BlockStatementNode>(current_block, block_name, tok.text.data() - tmpl.content.c_str());
415 current_block->nodes.emplace_back(block_statement_node);
416 block_statement_stack.emplace(block_statement_node.get());
417 current_block = &block_statement_node->block;
418 auto success = tmpl.block_storage.emplace(block_name, block_statement_node);
419 if (!success.second) {
420 throw_parser_error("block with the name '" + block_name + "' does already exist");
421 }
422
423 get_next_token();
424
425 } else if (tok.text == static_cast<decltype(tok.text)>("endblock")) {
426 if (block_statement_stack.empty()) {
427 throw_parser_error("endblock without matching block");
428 }
429
430 auto &block_statement_data = block_statement_stack.top();
431 get_next_token();
432
433 current_block = block_statement_data->parent;
434 block_statement_stack.pop();
435
436 } else if (tok.text == static_cast<decltype(tok.text)>("for")) {
437 get_next_token();
438
439 // options: for a in arr; for a, b in obj
440 if (tok.kind != Token::Kind::Id) {
441 throw_parser_error("expected id, got '" + tok.describe() + "'");
442 }
443
444 Token value_token = tok;
445 get_next_token();
446
447 // Object type
448 std::shared_ptr<ForStatementNode> for_statement_node;
449 if (tok.kind == Token::Kind::Comma) {
450 get_next_token();
451 if (tok.kind != Token::Kind::Id) {
452 throw_parser_error("expected id, got '" + tok.describe() + "'");
453 }
454
455 Token key_token = std::move(value_token);
456 value_token = tok;
457 get_next_token();
458
459 for_statement_node = std::make_shared<ForObjectStatementNode>(static_cast<std::string>(key_token.text), static_cast<std::string>(value_token.text), current_block, tok.text.data() - tmpl.content.c_str());
460
461 // Array type
462 } else {
463 for_statement_node = std::make_shared<ForArrayStatementNode>(static_cast<std::string>(value_token.text), current_block, tok.text.data() - tmpl.content.c_str());
464 }
465
466 current_block->nodes.emplace_back(for_statement_node);
467 for_statement_stack.emplace(for_statement_node.get());
468 current_block = &for_statement_node->body;
469 current_expression_list = &for_statement_node->condition;
470
471 if (tok.kind != Token::Kind::Id || tok.text != static_cast<decltype(tok.text)>("in")) {
472 throw_parser_error("expected 'in', got '" + tok.describe() + "'");
473 }
474 get_next_token();
475
476 if (!parse_expression(tmpl, closing)) {
477 return false;
478 }
479
480 } else if (tok.text == static_cast<decltype(tok.text)>("endfor")) {
481 if (for_statement_stack.empty()) {
482 throw_parser_error("endfor without matching for");
483 }
484
485 auto &for_statement_data = for_statement_stack.top();
486 get_next_token();
487
488 current_block = for_statement_data->parent;
489 for_statement_stack.pop();
490
491 } else if (tok.text == static_cast<decltype(tok.text)>("include")) {
492 get_next_token();
493
494 if (tok.kind != Token::Kind::String) {
495 throw_parser_error("expected string, got '" + tok.describe() + "'");
496 }
497
498 std::string template_name = json::parse(tok.text).get_ref<const std::string &>();
499 add_to_template_storage(path, template_name);
500
501 current_block->nodes.emplace_back(std::make_shared<IncludeStatementNode>(template_name, tok.text.data() - tmpl.content.c_str()));
502
503 get_next_token();
504
505 } else if (tok.text == static_cast<decltype(tok.text)>("extends")) {
506 get_next_token();
507
508 if (tok.kind != Token::Kind::String) {
509 throw_parser_error("expected string, got '" + tok.describe() + "'");
510 }
511
512 std::string template_name = json::parse(tok.text).get_ref<const std::string &>();
513 add_to_template_storage(path, template_name);
514
515 current_block->nodes.emplace_back(std::make_shared<ExtendsStatementNode>(template_name, tok.text.data() - tmpl.content.c_str()));
516
517 get_next_token();
518
519 } else if (tok.text == static_cast<decltype(tok.text)>("set")) {
520 get_next_token();
521
522 if (tok.kind != Token::Kind::Id) {
523 throw_parser_error("expected variable name, got '" + tok.describe() + "'");
524 }
525
526 std::string key = static_cast<std::string>(tok.text);
527 get_next_token();
528
529 auto set_statement_node = std::make_shared<SetStatementNode>(key, tok.text.data() - tmpl.content.c_str());
530 current_block->nodes.emplace_back(set_statement_node);
531 current_expression_list = &set_statement_node->expression;
532
533 if (tok.text != static_cast<decltype(tok.text)>("=")) {
534 throw_parser_error("expected '=', got '" + tok.describe() + "'");
535 }
536 get_next_token();
537
538 if (!parse_expression(tmpl, closing)) {
539 return false;
540 }
541
542 } else {
543 return false;
544 }
545 return true;
546 }
547
548 void parse_into(Template &tmpl, nonstd::string_view path) {
549 lexer.start(tmpl.content);
550 current_block = &tmpl.root;
551
552 for (;;) {
553 get_next_token();
554 switch (tok.kind) {
555 case Token::Kind::Eof: {
556 if (!if_statement_stack.empty()) {
557 throw_parser_error("unmatched if");
558 }
559 if (!for_statement_stack.empty()) {
560 throw_parser_error("unmatched for");
561 }
562 } return;
563 case Token::Kind::Text: {
564 current_block->nodes.emplace_back(std::make_shared<TextNode>(tok.text.data() - tmpl.content.c_str(), tok.text.size()));
565 } break;
566 case Token::Kind::StatementOpen: {
567 get_next_token();
568 if (!parse_statement(tmpl, Token::Kind::StatementClose, path)) {
569 throw_parser_error("expected statement, got '" + tok.describe() + "'");
570 }
571 if (tok.kind != Token::Kind::StatementClose) {
572 throw_parser_error("expected statement close, got '" + tok.describe() + "'");
573 }
574 } break;
575 case Token::Kind::LineStatementOpen: {
576 get_next_token();
577 if (!parse_statement(tmpl, Token::Kind::LineStatementClose, path)) {
578 throw_parser_error("expected statement, got '" + tok.describe() + "'");
579 }
580 if (tok.kind != Token::Kind::LineStatementClose && tok.kind != Token::Kind::Eof) {
581 throw_parser_error("expected line statement close, got '" + tok.describe() + "'");
582 }
583 } break;
584 case Token::Kind::ExpressionOpen: {
585 get_next_token();
586
587 auto expression_list_node = std::make_shared<ExpressionListNode>(tok.text.data() - tmpl.content.c_str());
588 current_block->nodes.emplace_back(expression_list_node);
589 current_expression_list = expression_list_node.get();
590
591 if (!parse_expression(tmpl, Token::Kind::ExpressionClose)) {
592 throw_parser_error("expected expression, got '" + tok.describe() + "'");
593 }
594
595 if (tok.kind != Token::Kind::ExpressionClose) {
596 throw_parser_error("expected expression close, got '" + tok.describe() + "'");
597 }
598 } break;
599 case Token::Kind::CommentOpen: {
600 get_next_token();
601 if (tok.kind != Token::Kind::CommentClose) {
602 throw_parser_error("expected comment close, got '" + tok.describe() + "'");
603 }
604 } break;
605 default: {
606 throw_parser_error("unexpected token '" + tok.describe() + "'");
607 } break;
608 }
609 }
610 }
611
612
613public:
614 explicit Parser(const ParserConfig &parser_config, const LexerConfig &lexer_config,
615 TemplateStorage &template_storage, const FunctionStorage &function_storage)
616 : config(parser_config), lexer(lexer_config), template_storage(template_storage), function_storage(function_storage) { }
617
618 Template parse(nonstd::string_view input, nonstd::string_view path) {
619 auto result = Template(static_cast<std::string>(input));
620 parse_into(result, path);
621 return result;
622 }
623
624 Template parse(nonstd::string_view input) {
625 return parse(input, "./");
626 }
627
628 void parse_into_template(Template& tmpl, nonstd::string_view filename) {
629 nonstd::string_view path = filename.substr(0, filename.find_last_of("/\\") + 1);
630
631 // StringRef path = sys::path::parent_path(filename);
632 auto sub_parser = Parser(config, lexer.get_config(), template_storage, function_storage);
633 sub_parser.parse_into(tmpl, path);
634 }
635
636 std::string load_file(nonstd::string_view filename) {
637 std::ifstream file;
638 open_file_or_throw(static_cast<std::string>(filename), file);
639 std::string text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
640 return text;
641 }
642};
643
644} // namespace inja
645
646#endif // INCLUDE_INJA_PARSER_HPP_
Definition: node.hpp:70
Definition: node.hpp:254
Class for builtin functions and user-defined callbacks.
Definition: function_storage.hpp:19
Class for lexing an inja Template.
Definition: lexer.hpp:16
Class for parsing an inja Template.
Definition: parser.hpp:27
Class for lexer configuration.
Definition: config.hpp:14
Class for parser configuration.
Definition: config.hpp:66
Definition: exceptions.hpp:29
The main inja Template.
Definition: template.hpp:18
Helper-class for the inja Lexer.
Definition: token.hpp:13