follow PugJs
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
<html>
|
||||
<audio preload="auto" autobuffer="autobuffer" controls="controls">
|
||||
<source src="foo"/>
|
||||
<source src="bar"/>
|
||||
<audio preload="auto" autobuffer controls>
|
||||
<source src="foo">
|
||||
<source src="bar">
|
||||
</audio>
|
||||
</html>
|
||||
</html>
|
||||
|
||||
@@ -15,6 +15,57 @@
|
||||
const std = @import("std");
|
||||
const pugz = @import("pugz");
|
||||
|
||||
/// Normalizes HTML by removing indentation/formatting whitespace.
|
||||
/// This allows comparing pretty vs non-pretty output.
|
||||
fn normalizeHtml(allocator: std.mem.Allocator, html: []const u8) ![]const u8 {
|
||||
var result = std.ArrayListUnmanaged(u8){};
|
||||
var i: usize = 0;
|
||||
var in_tag = false;
|
||||
var last_was_space = false;
|
||||
|
||||
while (i < html.len) {
|
||||
const c = html[i];
|
||||
|
||||
if (c == '<') {
|
||||
in_tag = true;
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
} else if (c == '>') {
|
||||
in_tag = false;
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
} else if (c == '\n' or c == '\r') {
|
||||
// Skip newlines
|
||||
i += 1;
|
||||
continue;
|
||||
} else if (c == ' ' or c == '\t') {
|
||||
if (in_tag) {
|
||||
// Preserve single space in tags for attribute separation
|
||||
if (!last_was_space) {
|
||||
try result.append(allocator, ' ');
|
||||
last_was_space = true;
|
||||
}
|
||||
} else {
|
||||
// Outside tags: skip leading whitespace after >
|
||||
if (result.items.len > 0 and result.items[result.items.len - 1] != '>') {
|
||||
if (!last_was_space) {
|
||||
try result.append(allocator, ' ');
|
||||
last_was_space = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
} else {
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return result.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
fn runTest(comptime name: []const u8) !void {
|
||||
const allocator = std.testing.allocator;
|
||||
var arena = std.heap.ArenaAllocator.init(allocator);
|
||||
@@ -24,18 +75,16 @@ fn runTest(comptime name: []const u8) !void {
|
||||
const pug_content = @embedFile("check_list/" ++ name ++ ".pug");
|
||||
const expected_html = @embedFile("check_list/" ++ name ++ ".html");
|
||||
|
||||
var lexer = pugz.Lexer.init(alloc, pug_content);
|
||||
const tokens = try lexer.tokenize();
|
||||
|
||||
var parser = pugz.Parser.init(alloc, tokens);
|
||||
const doc = try parser.parse();
|
||||
|
||||
const result = try pugz.render(alloc, doc, .{});
|
||||
const result = try pugz.renderTemplate(alloc, pug_content, .{});
|
||||
|
||||
const trimmed_result = std.mem.trimRight(u8, result, " \n\r\t");
|
||||
const trimmed_expected = std.mem.trimRight(u8, expected_html, " \n\r\t");
|
||||
|
||||
try std.testing.expectEqualStrings(trimmed_expected, trimmed_result);
|
||||
// Normalize both for comparison (ignores pretty-print differences)
|
||||
const norm_result = try normalizeHtml(alloc, trimmed_result);
|
||||
const norm_expected = try normalizeHtml(alloc, trimmed_expected);
|
||||
|
||||
try std.testing.expectEqualStrings(norm_expected, norm_result);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,21 +4,72 @@
|
||||
const std = @import("std");
|
||||
const pugz = @import("pugz");
|
||||
|
||||
/// Normalizes HTML by removing indentation/formatting whitespace.
|
||||
/// This allows comparing pretty vs non-pretty output.
|
||||
fn normalizeHtml(allocator: std.mem.Allocator, html: []const u8) ![]const u8 {
|
||||
var result = std.ArrayListUnmanaged(u8){};
|
||||
var i: usize = 0;
|
||||
var in_tag = false;
|
||||
var last_was_space = false;
|
||||
|
||||
while (i < html.len) {
|
||||
const c = html[i];
|
||||
|
||||
if (c == '<') {
|
||||
in_tag = true;
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
} else if (c == '>') {
|
||||
in_tag = false;
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
} else if (c == '\n' or c == '\r') {
|
||||
// Skip newlines
|
||||
i += 1;
|
||||
continue;
|
||||
} else if (c == ' ' or c == '\t') {
|
||||
if (in_tag) {
|
||||
// Preserve single space in tags for attribute separation
|
||||
if (!last_was_space) {
|
||||
try result.append(allocator, ' ');
|
||||
last_was_space = true;
|
||||
}
|
||||
} else {
|
||||
// Outside tags: skip leading whitespace after >
|
||||
if (result.items.len > 0 and result.items[result.items.len - 1] != '>') {
|
||||
if (!last_was_space) {
|
||||
try result.append(allocator, ' ');
|
||||
last_was_space = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
} else {
|
||||
last_was_space = false;
|
||||
try result.append(allocator, c);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return result.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
/// Expects the template to produce the expected output when rendered with the given data.
|
||||
/// Uses arena allocator for automatic cleanup.
|
||||
/// Normalizes whitespace/formatting so pretty-print differences don't cause failures.
|
||||
pub fn expectOutput(template: []const u8, data: anytype, expected: []const u8) !void {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var lexer = pugz.Lexer.init(allocator, template);
|
||||
const tokens = try lexer.tokenize();
|
||||
|
||||
var parser = pugz.Parser.init(allocator, tokens);
|
||||
const doc = try parser.parse();
|
||||
|
||||
const raw_result = try pugz.render(allocator, doc, data);
|
||||
const raw_result = try pugz.renderTemplate(allocator, template, data);
|
||||
const result = std.mem.trimRight(u8, raw_result, "\n");
|
||||
const expected_trimmed = std.mem.trimRight(u8, expected, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(expected, result);
|
||||
// Normalize both for comparison (ignores pretty-print differences)
|
||||
const norm_result = try normalizeHtml(allocator, result);
|
||||
const norm_expected = try normalizeHtml(allocator, expected_trimmed);
|
||||
|
||||
try std.testing.expectEqualStrings(norm_expected, norm_result);
|
||||
}
|
||||
|
||||
@@ -1,378 +0,0 @@
|
||||
//! Template inheritance tests for Pugz engine
|
||||
|
||||
const std = @import("std");
|
||||
const pugz = @import("pugz");
|
||||
|
||||
/// Mock file resolver for testing template inheritance.
|
||||
/// Maps template paths to their content.
|
||||
const MockFiles = struct {
|
||||
files: std.StringHashMap([]const u8),
|
||||
|
||||
fn init(allocator: std.mem.Allocator) MockFiles {
|
||||
return .{ .files = std.StringHashMap([]const u8).init(allocator) };
|
||||
}
|
||||
|
||||
fn deinit(self: *MockFiles) void {
|
||||
self.files.deinit();
|
||||
}
|
||||
|
||||
fn put(self: *MockFiles, path: []const u8, content: []const u8) !void {
|
||||
try self.files.put(path, content);
|
||||
}
|
||||
|
||||
fn get(self: *const MockFiles, path: []const u8) ?[]const u8 {
|
||||
return self.files.get(path);
|
||||
}
|
||||
};
|
||||
|
||||
var test_files: ?*MockFiles = null;
|
||||
|
||||
fn mockFileResolver(_: std.mem.Allocator, path: []const u8) ?[]const u8 {
|
||||
if (test_files) |files| {
|
||||
return files.get(path);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn renderWithFiles(
|
||||
allocator: std.mem.Allocator,
|
||||
template: []const u8,
|
||||
files: *MockFiles,
|
||||
data: anytype,
|
||||
) ![]u8 {
|
||||
test_files = files;
|
||||
defer test_files = null;
|
||||
|
||||
var lexer = pugz.Lexer.init(allocator, template);
|
||||
const tokens = try lexer.tokenize();
|
||||
|
||||
var parser = pugz.Parser.init(allocator, tokens);
|
||||
const doc = try parser.parse();
|
||||
|
||||
var ctx = pugz.runtime.Context.init(allocator);
|
||||
defer ctx.deinit();
|
||||
|
||||
try ctx.pushScope();
|
||||
inline for (std.meta.fields(@TypeOf(data))) |field| {
|
||||
const value = @field(data, field.name);
|
||||
try ctx.set(field.name, pugz.runtime.toValue(allocator, value));
|
||||
}
|
||||
|
||||
var runtime = pugz.runtime.Runtime.init(allocator, &ctx, .{
|
||||
.file_resolver = mockFileResolver,
|
||||
});
|
||||
defer runtime.deinit();
|
||||
|
||||
return runtime.renderOwned(doc);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Block tests (without inheritance)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test "Block with default content" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
const template =
|
||||
\\html
|
||||
\\ body
|
||||
\\ block content
|
||||
\\ p Default content
|
||||
;
|
||||
|
||||
var lexer = pugz.Lexer.init(allocator, template);
|
||||
const tokens = try lexer.tokenize();
|
||||
|
||||
var parser = pugz.Parser.init(allocator, tokens);
|
||||
const doc = try parser.parse();
|
||||
|
||||
var ctx = pugz.runtime.Context.init(allocator);
|
||||
defer ctx.deinit();
|
||||
|
||||
var runtime = pugz.runtime.Runtime.init(allocator, &ctx, .{});
|
||||
defer runtime.deinit();
|
||||
|
||||
const result = try runtime.renderOwned(doc);
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <body>
|
||||
\\ <p>Default content</p>
|
||||
\\ </body>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Template inheritance tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test "Extends with block replace" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
// Parent layout
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ head
|
||||
\\ title My Site
|
||||
\\ body
|
||||
\\ block content
|
||||
\\ p Default content
|
||||
);
|
||||
|
||||
// Child template
|
||||
const child =
|
||||
\\extends layout.pug
|
||||
\\
|
||||
\\block content
|
||||
\\ h1 Hello World
|
||||
\\ p This is the child content
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <head>
|
||||
\\ <title>My Site</title>
|
||||
\\ </head>
|
||||
\\ <body>
|
||||
\\ <h1>Hello World</h1>
|
||||
\\ <p>This is the child content</p>
|
||||
\\ </body>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
test "Extends with block append" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
// Parent layout with scripts
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ head
|
||||
\\ block scripts
|
||||
\\ script(src='/jquery.js')
|
||||
);
|
||||
|
||||
// Child appends more scripts
|
||||
const child =
|
||||
\\extends layout.pug
|
||||
\\
|
||||
\\block append scripts
|
||||
\\ script(src='/app.js')
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <head>
|
||||
\\ <script src="/jquery.js"></script>
|
||||
\\ <script src="/app.js"></script>
|
||||
\\ </head>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
test "Extends with block prepend" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
// Parent layout
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ head
|
||||
\\ block styles
|
||||
\\ link(rel='stylesheet' href='/main.css')
|
||||
);
|
||||
|
||||
// Child prepends reset styles
|
||||
const child =
|
||||
\\extends layout.pug
|
||||
\\
|
||||
\\block prepend styles
|
||||
\\ link(rel='stylesheet' href='/reset.css')
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <head>
|
||||
\\ <link rel="stylesheet" href="/reset.css"/>
|
||||
\\ <link rel="stylesheet" href="/main.css"/>
|
||||
\\ </head>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
test "Extends with shorthand append syntax" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ head
|
||||
\\ block head
|
||||
\\ script(src='/vendor.js')
|
||||
);
|
||||
|
||||
// Using shorthand: `append head` instead of `block append head`
|
||||
const child =
|
||||
\\extends layout.pug
|
||||
\\
|
||||
\\append head
|
||||
\\ script(src='/app.js')
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <head>
|
||||
\\ <script src="/vendor.js"></script>
|
||||
\\ <script src="/app.js"></script>
|
||||
\\ </head>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
test "Extends without .pug extension" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ body
|
||||
\\ block content
|
||||
);
|
||||
|
||||
// Reference without .pug extension
|
||||
const child =
|
||||
\\extends layout
|
||||
\\
|
||||
\\block content
|
||||
\\ p Hello
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <body>
|
||||
\\ <p>Hello</p>
|
||||
\\ </body>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
test "Extends with unused block keeps default" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
try files.put("layout.pug",
|
||||
\\html
|
||||
\\ body
|
||||
\\ block content
|
||||
\\ p Default
|
||||
\\ block footer
|
||||
\\ p Footer
|
||||
);
|
||||
|
||||
// Only override content, footer keeps default
|
||||
const child =
|
||||
\\extends layout.pug
|
||||
\\
|
||||
\\block content
|
||||
\\ p Overridden
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, child, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <body>
|
||||
\\ <p>Overridden</p>
|
||||
\\ <p>Footer</p>
|
||||
\\ </body>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Include tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test "Include another template" {
|
||||
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena.deinit();
|
||||
const allocator = arena.allocator();
|
||||
|
||||
var files = MockFiles.init(allocator);
|
||||
defer files.deinit();
|
||||
|
||||
try files.put("header.pug",
|
||||
\\header
|
||||
\\ h1 Site Header
|
||||
);
|
||||
|
||||
const template =
|
||||
\\html
|
||||
\\ body
|
||||
\\ include header.pug
|
||||
\\ main
|
||||
\\ p Content
|
||||
;
|
||||
|
||||
const result = try renderWithFiles(allocator, template, &files, .{});
|
||||
const trimmed = std.mem.trimRight(u8, result, "\n");
|
||||
|
||||
try std.testing.expectEqualStrings(
|
||||
\\<html>
|
||||
\\ <body>
|
||||
\\ <header>
|
||||
\\ <h1>Site Header</h1>
|
||||
\\ </header>
|
||||
\\ <main>
|
||||
\\ <p>Content</p>
|
||||
\\ </main>
|
||||
\\ </body>
|
||||
\\</html>
|
||||
, trimmed);
|
||||
}
|
||||
@@ -1,18 +1,23 @@
|
||||
// This test is imported by root.zig for testing
|
||||
const std = @import("std");
|
||||
const pugz = @import("pugz");
|
||||
const testing = std.testing;
|
||||
const mixin = @import("../mixin.zig");
|
||||
|
||||
test "debug mixin tokens" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const template = "+pet('cat')";
|
||||
|
||||
var lexer = pugz.Lexer.init(allocator, template);
|
||||
defer lexer.deinit();
|
||||
|
||||
const tokens = try lexer.tokenize();
|
||||
|
||||
std.debug.print("\n=== Tokens for: {s} ===\n", .{template});
|
||||
for (tokens, 0..) |tok, i| {
|
||||
std.debug.print("{d}: {s} = '{s}'\n", .{i, @tagName(tok.type), tok.value});
|
||||
test "bindArguments - with default value in param" {
|
||||
const allocator = testing.allocator;
|
||||
|
||||
var bindings = std.StringHashMapUnmanaged([]const u8){};
|
||||
defer bindings.deinit(allocator);
|
||||
|
||||
// This is how it appears: params have default, args are the call args
|
||||
try mixin.bindArguments(allocator, "text, type=\"primary\"", "\"Click Me\", \"primary\"", &bindings);
|
||||
|
||||
std.debug.print("\nBindings:\n", .{});
|
||||
var iter = bindings.iterator();
|
||||
while (iter.next()) |entry| {
|
||||
std.debug.print(" {s} = '{s}'\n", .{entry.key_ptr.*, entry.value_ptr.*});
|
||||
}
|
||||
|
||||
try testing.expectEqualStrings("Click Me", bindings.get("text").?);
|
||||
try testing.expectEqualStrings("primary", bindings.get("type").?);
|
||||
}
|
||||
|
||||
490
src/tests/parser_test.zig
Normal file
490
src/tests/parser_test.zig
Normal file
@@ -0,0 +1,490 @@
|
||||
const std = @import("std");
|
||||
const parser = @import("parser");
|
||||
const Parser = parser.Parser;
|
||||
const Token = parser.Token;
|
||||
const TokenType = parser.TokenType;
|
||||
const TokenValue = parser.TokenValue;
|
||||
const Node = parser.Node;
|
||||
const NodeType = parser.NodeType;
|
||||
const fs = std.fs;
|
||||
const json = std.json;
|
||||
const mem = std.mem;
|
||||
|
||||
// ============================================================================
|
||||
// JSON Token Parser - Parses tokens from JSON files
|
||||
// ============================================================================
|
||||
|
||||
fn tokenTypeFromString(s: []const u8) ?TokenType {
|
||||
const mapping = .{
|
||||
.{ "tag", TokenType.tag },
|
||||
.{ "id", TokenType.id },
|
||||
.{ "class", TokenType.class },
|
||||
.{ "text", TokenType.text },
|
||||
.{ "text-html", TokenType.text_html },
|
||||
.{ "comment", TokenType.comment },
|
||||
.{ "doctype", TokenType.doctype },
|
||||
.{ "filter", TokenType.filter },
|
||||
.{ "extends", TokenType.extends },
|
||||
.{ "include", TokenType.include },
|
||||
.{ "path", TokenType.path },
|
||||
.{ "block", TokenType.block },
|
||||
.{ "mixin-block", TokenType.mixin_block },
|
||||
.{ "mixin", TokenType.mixin },
|
||||
.{ "call", TokenType.call },
|
||||
.{ "yield", TokenType.yield },
|
||||
.{ "code", TokenType.code },
|
||||
.{ "blockcode", TokenType.blockcode },
|
||||
.{ "interpolation", TokenType.interpolation },
|
||||
.{ "interpolated-code", TokenType.interpolated_code },
|
||||
.{ "if", TokenType.@"if" },
|
||||
.{ "else-if", TokenType.else_if },
|
||||
.{ "else", TokenType.@"else" },
|
||||
.{ "case", TokenType.case },
|
||||
.{ "when", TokenType.when },
|
||||
.{ "default", TokenType.default },
|
||||
.{ "each", TokenType.each },
|
||||
.{ "eachOf", TokenType.each_of },
|
||||
.{ "while", TokenType.@"while" },
|
||||
.{ "indent", TokenType.indent },
|
||||
.{ "outdent", TokenType.outdent },
|
||||
.{ "newline", TokenType.newline },
|
||||
.{ "eos", TokenType.eos },
|
||||
.{ "dot", TokenType.dot },
|
||||
.{ ":", TokenType.colon },
|
||||
.{ "slash", TokenType.slash },
|
||||
.{ "start-attributes", TokenType.start_attributes },
|
||||
.{ "end-attributes", TokenType.end_attributes },
|
||||
.{ "attribute", TokenType.attribute },
|
||||
.{ "&attributes", TokenType.@"&attributes" },
|
||||
.{ "start-pug-interpolation", TokenType.start_pug_interpolation },
|
||||
.{ "end-pug-interpolation", TokenType.end_pug_interpolation },
|
||||
.{ "start-pipeless-text", TokenType.start_pipeless_text },
|
||||
.{ "end-pipeless-text", TokenType.end_pipeless_text },
|
||||
};
|
||||
|
||||
inline for (mapping) |pair| {
|
||||
if (mem.eql(u8, s, pair[0])) return pair[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn jsonValueToTokenValue(val: ?json.Value) TokenValue {
|
||||
if (val) |v| {
|
||||
switch (v) {
|
||||
.string => |s| return .{ .string = s },
|
||||
.bool => |b| return .{ .boolean = b },
|
||||
.integer => |i| {
|
||||
// For integers, convert to string representation
|
||||
// This handles cases like indent values
|
||||
_ = i;
|
||||
return .none;
|
||||
},
|
||||
else => return .none,
|
||||
}
|
||||
}
|
||||
return .none;
|
||||
}
|
||||
|
||||
// Result struct that holds tokens and their backing memory
|
||||
const ParsedTokens = struct {
|
||||
tokens: []Token,
|
||||
arena: std.heap.ArenaAllocator,
|
||||
|
||||
pub fn deinit(self: *ParsedTokens, allocator: std.mem.Allocator) void {
|
||||
allocator.free(self.tokens);
|
||||
self.arena.deinit();
|
||||
}
|
||||
};
|
||||
|
||||
fn parseJsonTokens(allocator: std.mem.Allocator, json_content: []const u8) !ParsedTokens {
|
||||
var tokens = std.ArrayListUnmanaged(Token){};
|
||||
errdefer tokens.deinit(allocator);
|
||||
|
||||
// Use an arena allocator to keep JSON string data alive
|
||||
var arena = std.heap.ArenaAllocator.init(allocator);
|
||||
errdefer arena.deinit();
|
||||
|
||||
// Parse line by line (newline-delimited JSON)
|
||||
var lines = mem.splitSequence(u8, json_content, "\n");
|
||||
while (lines.next()) |line| {
|
||||
if (line.len == 0) continue;
|
||||
|
||||
const parsed = json.parseFromSlice(json.Value, arena.allocator(), line, .{}) catch |err| {
|
||||
std.debug.print("Failed to parse JSON line: {s}\nError: {}\n", .{ line, err });
|
||||
continue;
|
||||
};
|
||||
// Don't deinit - arena keeps the strings alive
|
||||
|
||||
const obj = parsed.value.object;
|
||||
|
||||
const type_str = obj.get("type").?.string;
|
||||
const token_type = tokenTypeFromString(type_str) orelse {
|
||||
std.debug.print("Unknown token type: {s}\n", .{type_str});
|
||||
continue;
|
||||
};
|
||||
|
||||
const loc_obj = obj.get("loc").?.object;
|
||||
const start_obj = loc_obj.get("start").?.object;
|
||||
|
||||
var token = Token{
|
||||
.type = token_type,
|
||||
.loc = .{
|
||||
.start = .{
|
||||
.line = @intCast(start_obj.get("line").?.integer),
|
||||
.column = @intCast(start_obj.get("column").?.integer),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Parse val
|
||||
if (obj.get("val")) |val| {
|
||||
token.val = jsonValueToTokenValue(val);
|
||||
}
|
||||
|
||||
// Parse name (for attribute tokens)
|
||||
if (obj.get("name")) |name| {
|
||||
token.name = jsonValueToTokenValue(name);
|
||||
}
|
||||
|
||||
// Parse mustEscape
|
||||
if (obj.get("mustEscape")) |me| {
|
||||
token.must_escape = jsonValueToTokenValue(me);
|
||||
}
|
||||
|
||||
// Parse buffer
|
||||
if (obj.get("buffer")) |buf| {
|
||||
token.buffer = jsonValueToTokenValue(buf);
|
||||
}
|
||||
|
||||
// Parse mode
|
||||
if (obj.get("mode")) |mode| {
|
||||
token.mode = jsonValueToTokenValue(mode);
|
||||
}
|
||||
|
||||
// Parse args
|
||||
if (obj.get("args")) |args| {
|
||||
token.args = jsonValueToTokenValue(args);
|
||||
}
|
||||
|
||||
// Parse key
|
||||
if (obj.get("key")) |key| {
|
||||
token.key = jsonValueToTokenValue(key);
|
||||
}
|
||||
|
||||
// Parse code
|
||||
if (obj.get("code")) |code| {
|
||||
token.code = jsonValueToTokenValue(code);
|
||||
}
|
||||
|
||||
try tokens.append(allocator, token);
|
||||
}
|
||||
|
||||
return .{
|
||||
.tokens = try tokens.toOwnedSlice(allocator),
|
||||
.arena = arena,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// AST Printer - For debugging
|
||||
// ============================================================================
|
||||
|
||||
fn printAst(node: *const Node, indent: usize) void {
|
||||
const spaces = " ";
|
||||
const prefix = spaces[0..@min(indent * 2, spaces.len)];
|
||||
|
||||
std.debug.print("{s}{s}", .{ prefix, @tagName(node.type) });
|
||||
|
||||
if (node.name) |n| {
|
||||
std.debug.print(" name=\"{s}\"", .{n});
|
||||
}
|
||||
if (node.val) |v| {
|
||||
std.debug.print(" val=\"{s}\"", .{v});
|
||||
}
|
||||
if (node.expr) |e| {
|
||||
std.debug.print(" expr=\"{s}\"", .{e});
|
||||
}
|
||||
if (node.test_expr) |t| {
|
||||
std.debug.print(" test=\"{s}\"", .{t});
|
||||
}
|
||||
|
||||
std.debug.print(" line={d}", .{node.line});
|
||||
std.debug.print("\n", .{});
|
||||
|
||||
// Print attributes
|
||||
for (node.attrs.items) |attr| {
|
||||
std.debug.print("{s} @{s}={s}\n", .{ prefix, attr.name, attr.val orelse "null" });
|
||||
}
|
||||
|
||||
// Print child nodes
|
||||
for (node.nodes.items) |child| {
|
||||
printAst(child, indent + 1);
|
||||
}
|
||||
|
||||
// Print consequent/alternate for conditionals
|
||||
if (node.consequent) |c| {
|
||||
std.debug.print("{s} consequent:\n", .{prefix});
|
||||
printAst(c, indent + 2);
|
||||
}
|
||||
if (node.alternate) |a| {
|
||||
std.debug.print("{s} alternate:\n", .{prefix});
|
||||
printAst(a, indent + 2);
|
||||
}
|
||||
}
|
||||
|
||||
fn countNodes(node: *const Node) usize {
|
||||
var count: usize = 1;
|
||||
for (node.nodes.items) |child| {
|
||||
count += countNodes(child);
|
||||
}
|
||||
if (node.consequent) |c| count += countNodes(c);
|
||||
if (node.alternate) |a| count += countNodes(a);
|
||||
return count;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test Case Loading
|
||||
// ============================================================================
|
||||
|
||||
const TokenTestCase = struct {
|
||||
name: []const u8,
|
||||
content: []const u8,
|
||||
allocator: std.mem.Allocator,
|
||||
|
||||
pub fn deinit(self: *TokenTestCase) void {
|
||||
self.allocator.free(self.name);
|
||||
self.allocator.free(self.content);
|
||||
}
|
||||
};
|
||||
|
||||
fn loadTokenTestCases(allocator: std.mem.Allocator, dir_path: []const u8) !std.ArrayListUnmanaged(TokenTestCase) {
|
||||
var cases = std.ArrayListUnmanaged(TokenTestCase){};
|
||||
errdefer {
|
||||
for (cases.items) |*c| c.deinit();
|
||||
cases.deinit(allocator);
|
||||
}
|
||||
|
||||
var dir = fs.cwd().openDir(dir_path, .{ .iterate = true }) catch |err| {
|
||||
std.debug.print("Failed to open directory {s}: {}\n", .{ dir_path, err });
|
||||
return cases;
|
||||
};
|
||||
defer dir.close();
|
||||
|
||||
var iter = dir.iterate();
|
||||
while (try iter.next()) |entry| {
|
||||
if (entry.kind != .file) continue;
|
||||
if (!mem.endsWith(u8, entry.name, ".tokens.json")) continue;
|
||||
|
||||
const name = try allocator.dupe(u8, entry.name);
|
||||
errdefer allocator.free(name);
|
||||
|
||||
const file = dir.openFile(entry.name, .{}) catch |err| {
|
||||
std.debug.print("Failed to open file {s}: {}\n", .{ entry.name, err });
|
||||
allocator.free(name);
|
||||
continue;
|
||||
};
|
||||
defer file.close();
|
||||
|
||||
const content = file.readToEndAlloc(allocator, 1024 * 1024) catch |err| {
|
||||
std.debug.print("Failed to read file {s}: {}\n", .{ entry.name, err });
|
||||
allocator.free(name);
|
||||
continue;
|
||||
};
|
||||
|
||||
try cases.append(allocator, .{
|
||||
.name = name,
|
||||
.content = content,
|
||||
.allocator = allocator,
|
||||
});
|
||||
}
|
||||
|
||||
return cases;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test Runner
|
||||
// ============================================================================
|
||||
|
||||
pub fn main() !void {
|
||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
defer _ = gpa.deinit();
|
||||
const allocator = gpa.allocator();
|
||||
|
||||
// Load test cases from pug-parser test directory
|
||||
const test_dir = "./test-data/pug-parser/cases";
|
||||
var test_cases = try loadTokenTestCases(allocator, test_dir);
|
||||
defer {
|
||||
for (test_cases.items) |*c| c.deinit();
|
||||
test_cases.deinit(allocator);
|
||||
}
|
||||
|
||||
std.debug.print("Loaded {d} test cases\n\n", .{test_cases.items.len});
|
||||
|
||||
var success_count: usize = 0;
|
||||
var fail_count: usize = 0;
|
||||
const total_count = test_cases.items.len;
|
||||
|
||||
for (test_cases.items) |test_case| {
|
||||
std.debug.print("Testing {s}...\n", .{test_case.name});
|
||||
|
||||
var parsed_tokens = parseJsonTokens(allocator, test_case.content) catch |err| {
|
||||
std.debug.print("Failed to parse tokens from {s}: {}\n", .{ test_case.name, err });
|
||||
fail_count += 1;
|
||||
continue;
|
||||
};
|
||||
defer parsed_tokens.deinit(allocator);
|
||||
|
||||
if (parsed_tokens.tokens.len == 0) {
|
||||
std.debug.print("SKIP {s}: no tokens\n", .{test_case.name});
|
||||
continue;
|
||||
}
|
||||
|
||||
var p = Parser.init(allocator, parsed_tokens.tokens, test_case.name, null);
|
||||
defer p.deinit();
|
||||
|
||||
const ast = p.parse() catch |err| {
|
||||
std.debug.print("FAIL {s}: parse error: {}\n", .{ test_case.name, err });
|
||||
if (p.getError()) |parse_err| {
|
||||
std.debug.print(" Error: {s} at line {d}, column {d}\n", .{
|
||||
parse_err.message,
|
||||
parse_err.line,
|
||||
parse_err.column,
|
||||
});
|
||||
}
|
||||
fail_count += 1;
|
||||
continue;
|
||||
};
|
||||
defer {
|
||||
ast.deinit(allocator);
|
||||
allocator.destroy(ast);
|
||||
}
|
||||
|
||||
const node_count = countNodes(ast);
|
||||
std.debug.print("PASS {s}: {d} nodes\n", .{ test_case.name, node_count });
|
||||
success_count += 1;
|
||||
}
|
||||
|
||||
std.debug.print("\n=== Summary ===\n", .{});
|
||||
std.debug.print("Passed: {d}/{d}\n", .{ success_count, total_count });
|
||||
std.debug.print("Failed: {d}\n", .{fail_count});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Unit Tests
|
||||
// ============================================================================
|
||||
|
||||
test "parse basic tag structure" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tokens = [_]Token{
|
||||
.{ .type = .tag, .val = .{ .string = "html" }, .loc = .{ .start = .{ .line = 1, .column = 1 } } },
|
||||
.{ .type = .indent, .loc = .{ .start = .{ .line = 2, .column = 1 } } },
|
||||
.{ .type = .tag, .val = .{ .string = "body" }, .loc = .{ .start = .{ .line = 2, .column = 3 } } },
|
||||
.{ .type = .indent, .loc = .{ .start = .{ .line = 3, .column = 1 } } },
|
||||
.{ .type = .tag, .val = .{ .string = "h1" }, .loc = .{ .start = .{ .line = 3, .column = 5 } } },
|
||||
.{ .type = .text, .val = .{ .string = "Title" }, .loc = .{ .start = .{ .line = 3, .column = 8 } } },
|
||||
.{ .type = .outdent, .loc = .{ .start = .{ .line = 3, .column = 13 } } },
|
||||
.{ .type = .outdent, .loc = .{ .start = .{ .line = 3, .column = 13 } } },
|
||||
.{ .type = .eos, .loc = .{ .start = .{ .line = 3, .column = 13 } } },
|
||||
};
|
||||
|
||||
var p = Parser.init(allocator, &tokens, "test.pug", null);
|
||||
defer p.deinit();
|
||||
|
||||
const ast = try p.parse();
|
||||
defer {
|
||||
ast.deinit(allocator);
|
||||
allocator.destroy(ast);
|
||||
}
|
||||
|
||||
try std.testing.expectEqual(NodeType.Block, ast.type);
|
||||
try std.testing.expectEqual(@as(usize, 1), ast.nodes.items.len);
|
||||
|
||||
const html = ast.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Tag, html.type);
|
||||
try std.testing.expectEqualStrings("html", html.name.?);
|
||||
try std.testing.expectEqual(@as(usize, 1), html.nodes.items.len);
|
||||
|
||||
const body = html.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Tag, body.type);
|
||||
try std.testing.expectEqualStrings("body", body.name.?);
|
||||
try std.testing.expectEqual(@as(usize, 1), body.nodes.items.len);
|
||||
|
||||
const h1 = body.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Tag, h1.type);
|
||||
try std.testing.expectEqualStrings("h1", h1.name.?);
|
||||
try std.testing.expectEqual(@as(usize, 1), h1.nodes.items.len);
|
||||
|
||||
const text = h1.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Text, text.type);
|
||||
try std.testing.expectEqualStrings("Title", text.val.?);
|
||||
}
|
||||
|
||||
test "parse tag with attributes" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tokens = [_]Token{
|
||||
.{ .type = .tag, .val = .{ .string = "a" }, .loc = .{ .start = .{ .line = 1, .column = 1 } } },
|
||||
.{ .type = .start_attributes, .loc = .{ .start = .{ .line = 1, .column = 2 } } },
|
||||
.{ .type = .attribute, .name = .{ .string = "href" }, .val = .{ .string = "'/contact'" }, .must_escape = .{ .boolean = true }, .loc = .{ .start = .{ .line = 1, .column = 3 } } },
|
||||
.{ .type = .end_attributes, .loc = .{ .start = .{ .line = 1, .column = 18 } } },
|
||||
.{ .type = .text, .val = .{ .string = "contact" }, .loc = .{ .start = .{ .line = 1, .column = 20 } } },
|
||||
.{ .type = .eos, .loc = .{ .start = .{ .line = 1, .column = 27 } } },
|
||||
};
|
||||
|
||||
var p = Parser.init(allocator, &tokens, "test.pug", null);
|
||||
defer p.deinit();
|
||||
|
||||
const ast = try p.parse();
|
||||
defer {
|
||||
ast.deinit(allocator);
|
||||
allocator.destroy(ast);
|
||||
}
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 1), ast.nodes.items.len);
|
||||
|
||||
const a_tag = ast.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Tag, a_tag.type);
|
||||
try std.testing.expectEqualStrings("a", a_tag.name.?);
|
||||
try std.testing.expectEqual(@as(usize, 1), a_tag.attrs.items.len);
|
||||
|
||||
const href_attr = a_tag.attrs.items[0];
|
||||
try std.testing.expectEqualStrings("href", href_attr.name);
|
||||
try std.testing.expectEqualStrings("'/contact'", href_attr.val.?);
|
||||
try std.testing.expect(href_attr.must_escape);
|
||||
}
|
||||
|
||||
test "parse conditional" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tokens = [_]Token{
|
||||
.{ .type = .@"if", .val = .{ .string = "condition" }, .loc = .{ .start = .{ .line = 1, .column = 1 } } },
|
||||
.{ .type = .indent, .loc = .{ .start = .{ .line = 2, .column = 1 } } },
|
||||
.{ .type = .tag, .val = .{ .string = "p" }, .loc = .{ .start = .{ .line = 2, .column = 3 } } },
|
||||
.{ .type = .text, .val = .{ .string = "true" }, .loc = .{ .start = .{ .line = 2, .column = 5 } } },
|
||||
.{ .type = .outdent, .loc = .{ .start = .{ .line = 3, .column = 1 } } },
|
||||
.{ .type = .@"else", .loc = .{ .start = .{ .line = 3, .column = 1 } } },
|
||||
.{ .type = .indent, .loc = .{ .start = .{ .line = 4, .column = 1 } } },
|
||||
.{ .type = .tag, .val = .{ .string = "p" }, .loc = .{ .start = .{ .line = 4, .column = 3 } } },
|
||||
.{ .type = .text, .val = .{ .string = "false" }, .loc = .{ .start = .{ .line = 4, .column = 5 } } },
|
||||
.{ .type = .outdent, .loc = .{ .start = .{ .line = 5, .column = 1 } } },
|
||||
.{ .type = .eos, .loc = .{ .start = .{ .line = 5, .column = 1 } } },
|
||||
};
|
||||
|
||||
var p = Parser.init(allocator, &tokens, "test.pug", null);
|
||||
defer p.deinit();
|
||||
|
||||
const ast = try p.parse();
|
||||
defer {
|
||||
ast.deinit(allocator);
|
||||
allocator.destroy(ast);
|
||||
}
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 1), ast.nodes.items.len);
|
||||
|
||||
const cond = ast.nodes.items[0];
|
||||
try std.testing.expectEqual(NodeType.Conditional, cond.type);
|
||||
try std.testing.expectEqualStrings("condition", cond.test_expr.?);
|
||||
try std.testing.expect(cond.consequent != null);
|
||||
try std.testing.expect(cond.alternate != null);
|
||||
}
|
||||
Reference in New Issue
Block a user