fuzzer: share zig to html rendering with autodocs

lib/compiler/std-docs.zig

@@ -275,10 +275,6 @@ fn buildWasmBinary(
 ) ![]const u8 {
     const gpa = context.gpa;
 
-    const main_src_path = try std.fs.path.join(arena, &.{
-        context.zig_lib_directory, "docs", "wasm", "main.zig",
-    });
-
     var argv: std.ArrayListUnmanaged([]const u8) = .{};
 
     try argv.appendSlice(arena, &.{
@@ -298,7 +294,10 @@ fn buildWasmBinary(
         "--name",
         "autodoc",
         "-rdynamic",
-        main_src_path,
+        "--dep",
+        "Walk",
+        try std.fmt.allocPrint(arena, "-Mroot={s}/docs/wasm/main.zig", .{context.zig_lib_directory}),
+        try std.fmt.allocPrint(arena, "-MWalk={s}/docs/wasm/Walk.zig", .{context.zig_lib_directory}),
         "--listen=-",
     });
 

lib/docs/wasm/Decl.zig

@@ -1,3 +1,12 @@
+const Decl = @This();
+const std = @import("std");
+const Ast = std.zig.Ast;
+const Walk = @import("Walk.zig");
+const gpa = std.heap.wasm_allocator;
+const assert = std.debug.assert;
+const log = std.log;
+const Oom = error{OutOfMemory};
+
 ast_node: Ast.Node.Index,
 file: Walk.File.Index,
 /// The decl whose namespace this is in.
@@ -215,12 +224,3 @@ pub fn find(search_string: []const u8) Decl.Index {
     }
     return current_decl_index;
 }
-
-const Decl = @This();
-const std = @import("std");
-const Ast = std.zig.Ast;
-const Walk = @import("Walk.zig");
-const gpa = std.heap.wasm_allocator;
-const assert = std.debug.assert;
-const log = std.log;
-const Oom = error{OutOfMemory};

lib/docs/wasm/Walk.zig

@@ -1,4 +1,15 @@
 //! Find and annotate identifiers with links to their declarations.
+
+const Walk = @This();
+const std = @import("std");
+const Ast = std.zig.Ast;
+const assert = std.debug.assert;
+const log = std.log;
+const gpa = std.heap.wasm_allocator;
+const Oom = error{OutOfMemory};
+
+pub const Decl = @import("Decl.zig");
+
 pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
 pub var decls: std.ArrayListUnmanaged(Decl) = .{};
 pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};
@@ -1120,15 +1131,6 @@ pub fn isPrimitiveNonType(name: []const u8) bool {
 //    try w.root();
 //}
 
-const Walk = @This();
-const std = @import("std");
-const Ast = std.zig.Ast;
-const assert = std.debug.assert;
-const Decl = @import("Decl.zig");
-const log = std.log;
-const gpa = std.heap.wasm_allocator;
-const Oom = error{OutOfMemory};
-
 fn shrinkToFit(m: anytype) void {
     m.shrinkAndFree(gpa, m.entries.len);
 }

lib/docs/wasm/html_render.zig

@@ -0,0 +1,388 @@
+const std = @import("std");
+const Ast = std.zig.Ast;
+const assert = std.debug.assert;
+
+const Walk = @import("Walk");
+const Decl = Walk.Decl;
+
+const gpa = std.heap.wasm_allocator;
+const Oom = error{OutOfMemory};
+
+/// Delete this to find out where URL escaping needs to be added.
+pub const missing_feature_url_escape = true;
+
+pub const RenderSourceOptions = struct {
+    skip_doc_comments: bool = false,
+    skip_comments: bool = false,
+    collapse_whitespace: bool = false,
+    fn_link: Decl.Index = .none,
+};
+
+pub fn fileSourceHtml(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    root_node: Ast.Node.Index,
+    options: RenderSourceOptions,
+) !void {
+    const ast = file_index.get_ast();
+    const file = file_index.get();
+
+    const g = struct {
+        var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
+    };
+
+    const token_tags = ast.tokens.items(.tag);
+    const token_starts = ast.tokens.items(.start);
+    const main_tokens = ast.nodes.items(.main_token);
+
+    const start_token = ast.firstToken(root_node);
+    const end_token = ast.lastToken(root_node) + 1;
+
+    var cursor: usize = token_starts[start_token];
+
+    var indent: usize = 0;
+    if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
+        for (ast.source[newline_index + 1 .. cursor]) |c| {
+            if (c == ' ') {
+                indent += 1;
+            } else {
+                break;
+            }
+        }
+    }
+
+    for (
+        token_tags[start_token..end_token],
+        token_starts[start_token..end_token],
+        start_token..,
+    ) |tag, start, token_index| {
+        const between = ast.source[cursor..start];
+        if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
+            if (!options.skip_comments) {
+                try out.appendSlice(gpa, "<span class=\"tok-comment\">");
+                try appendUnindented(out, between, indent);
+                try out.appendSlice(gpa, "</span>");
+            }
+        } else if (between.len > 0) {
+            if (options.collapse_whitespace) {
+                if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
+                    try out.append(gpa, ' ');
+            } else {
+                try appendUnindented(out, between, indent);
+            }
+        }
+        if (tag == .eof) break;
+        const slice = ast.tokenSlice(token_index);
+        cursor = start + slice.len;
+        switch (tag) {
+            .eof => unreachable,
+
+            .keyword_addrspace,
+            .keyword_align,
+            .keyword_and,
+            .keyword_asm,
+            .keyword_async,
+            .keyword_await,
+            .keyword_break,
+            .keyword_catch,
+            .keyword_comptime,
+            .keyword_const,
+            .keyword_continue,
+            .keyword_defer,
+            .keyword_else,
+            .keyword_enum,
+            .keyword_errdefer,
+            .keyword_error,
+            .keyword_export,
+            .keyword_extern,
+            .keyword_for,
+            .keyword_if,
+            .keyword_inline,
+            .keyword_noalias,
+            .keyword_noinline,
+            .keyword_nosuspend,
+            .keyword_opaque,
+            .keyword_or,
+            .keyword_orelse,
+            .keyword_packed,
+            .keyword_anyframe,
+            .keyword_pub,
+            .keyword_resume,
+            .keyword_return,
+            .keyword_linksection,
+            .keyword_callconv,
+            .keyword_struct,
+            .keyword_suspend,
+            .keyword_switch,
+            .keyword_test,
+            .keyword_threadlocal,
+            .keyword_try,
+            .keyword_union,
+            .keyword_unreachable,
+            .keyword_usingnamespace,
+            .keyword_var,
+            .keyword_volatile,
+            .keyword_allowzero,
+            .keyword_while,
+            .keyword_anytype,
+            .keyword_fn,
+            => {
+                try out.appendSlice(gpa, "<span class=\"tok-kw\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .string_literal,
+            .char_literal,
+            .multiline_string_literal_line,
+            => {
+                try out.appendSlice(gpa, "<span class=\"tok-str\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .builtin => {
+                try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .doc_comment,
+            .container_doc_comment,
+            => {
+                if (!options.skip_doc_comments) {
+                    try out.appendSlice(gpa, "<span class=\"tok-comment\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                }
+            },
+
+            .identifier => i: {
+                if (options.fn_link != .none) {
+                    const fn_link = options.fn_link.get();
+                    const fn_token = main_tokens[fn_link.ast_node];
+                    if (token_index == fn_token + 1) {
+                        try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
+                        _ = missing_feature_url_escape;
+                        try fn_link.fqn(out);
+                        try out.appendSlice(gpa, "\">");
+                        try appendEscaped(out, slice);
+                        try out.appendSlice(gpa, "</a>");
+                        break :i;
+                    }
+                }
+
+                if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
+                    try out.appendSlice(gpa, "<span class=\"tok-fn\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                    break :i;
+                }
+
+                if (Walk.isPrimitiveNonType(slice)) {
+                    try out.appendSlice(gpa, "<span class=\"tok-null\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                    break :i;
+                }
+
+                if (std.zig.primitives.isPrimitive(slice)) {
+                    try out.appendSlice(gpa, "<span class=\"tok-type\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                    break :i;
+                }
+
+                if (file.token_parents.get(token_index)) |field_access_node| {
+                    g.field_access_buffer.clearRetainingCapacity();
+                    try walkFieldAccesses(file_index, &g.field_access_buffer, field_access_node);
+                    if (g.field_access_buffer.items.len > 0) {
+                        try out.appendSlice(gpa, "<a href=\"#");
+                        _ = missing_feature_url_escape;
+                        try out.appendSlice(gpa, g.field_access_buffer.items);
+                        try out.appendSlice(gpa, "\">");
+                        try appendEscaped(out, slice);
+                        try out.appendSlice(gpa, "</a>");
+                    } else {
+                        try appendEscaped(out, slice);
+                    }
+                    break :i;
+                }
+
+                {
+                    g.field_access_buffer.clearRetainingCapacity();
+                    try resolveIdentLink(file_index, &g.field_access_buffer, token_index);
+                    if (g.field_access_buffer.items.len > 0) {
+                        try out.appendSlice(gpa, "<a href=\"#");
+                        _ = missing_feature_url_escape;
+                        try out.appendSlice(gpa, g.field_access_buffer.items);
+                        try out.appendSlice(gpa, "\">");
+                        try appendEscaped(out, slice);
+                        try out.appendSlice(gpa, "</a>");
+                        break :i;
+                    }
+                }
+
+                try appendEscaped(out, slice);
+            },
+
+            .number_literal => {
+                try out.appendSlice(gpa, "<span class=\"tok-number\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .bang,
+            .pipe,
+            .pipe_pipe,
+            .pipe_equal,
+            .equal,
+            .equal_equal,
+            .equal_angle_bracket_right,
+            .bang_equal,
+            .l_paren,
+            .r_paren,
+            .semicolon,
+            .percent,
+            .percent_equal,
+            .l_brace,
+            .r_brace,
+            .l_bracket,
+            .r_bracket,
+            .period,
+            .period_asterisk,
+            .ellipsis2,
+            .ellipsis3,
+            .caret,
+            .caret_equal,
+            .plus,
+            .plus_plus,
+            .plus_equal,
+            .plus_percent,
+            .plus_percent_equal,
+            .plus_pipe,
+            .plus_pipe_equal,
+            .minus,
+            .minus_equal,
+            .minus_percent,
+            .minus_percent_equal,
+            .minus_pipe,
+            .minus_pipe_equal,
+            .asterisk,
+            .asterisk_equal,
+            .asterisk_asterisk,
+            .asterisk_percent,
+            .asterisk_percent_equal,
+            .asterisk_pipe,
+            .asterisk_pipe_equal,
+            .arrow,
+            .colon,
+            .slash,
+            .slash_equal,
+            .comma,
+            .ampersand,
+            .ampersand_equal,
+            .question_mark,
+            .angle_bracket_left,
+            .angle_bracket_left_equal,
+            .angle_bracket_angle_bracket_left,
+            .angle_bracket_angle_bracket_left_equal,
+            .angle_bracket_angle_bracket_left_pipe,
+            .angle_bracket_angle_bracket_left_pipe_equal,
+            .angle_bracket_right,
+            .angle_bracket_right_equal,
+            .angle_bracket_angle_bracket_right,
+            .angle_bracket_angle_bracket_right_equal,
+            .tilde,
+            => try appendEscaped(out, slice),
+
+            .invalid, .invalid_periodasterisks => return error.InvalidToken,
+        }
+    }
+}
+
+fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
+    var it = std.mem.splitScalar(u8, s, '\n');
+    var is_first_line = true;
+    while (it.next()) |line| {
+        if (is_first_line) {
+            try appendEscaped(out, line);
+            is_first_line = false;
+        } else {
+            try out.appendSlice(gpa, "\n");
+            try appendEscaped(out, unindent(line, indent));
+        }
+    }
+}
+
+pub fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
+    for (s) |c| {
+        try out.ensureUnusedCapacity(gpa, 6);
+        switch (c) {
+            '&' => out.appendSliceAssumeCapacity("&amp;"),
+            '<' => out.appendSliceAssumeCapacity("&lt;"),
+            '>' => out.appendSliceAssumeCapacity("&gt;"),
+            '"' => out.appendSliceAssumeCapacity("&quot;"),
+            else => out.appendAssumeCapacity(c),
+        }
+    }
+}
+
+fn walkFieldAccesses(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    node: Ast.Node.Index,
+) Oom!void {
+    const ast = file_index.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    assert(node_tags[node] == .field_access);
+    const node_datas = ast.nodes.items(.data);
+    const main_tokens = ast.nodes.items(.main_token);
+    const object_node = node_datas[node].lhs;
+    const dot_token = main_tokens[node];
+    const field_ident = dot_token + 1;
+    switch (node_tags[object_node]) {
+        .identifier => {
+            const lhs_ident = main_tokens[object_node];
+            try resolveIdentLink(file_index, out, lhs_ident);
+        },
+        .field_access => {
+            try walkFieldAccesses(file_index, out, object_node);
+        },
+        else => {},
+    }
+    if (out.items.len > 0) {
+        try out.append(gpa, '.');
+        try out.appendSlice(gpa, ast.tokenSlice(field_ident));
+    }
+}
+
+fn resolveIdentLink(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    ident_token: Ast.TokenIndex,
+) Oom!void {
+    const decl_index = file_index.get().lookup_token(ident_token);
+    if (decl_index == .none) return;
+    try resolveDeclLink(decl_index, out);
+}
+
+fn unindent(s: []const u8, indent: usize) []const u8 {
+    var indent_idx: usize = 0;
+    for (s) |c| {
+        if (c == ' ' and indent_idx < indent) {
+            indent_idx += 1;
+        } else {
+            break;
+        }
+    }
+    return s[indent_idx..];
+}
+
+pub fn resolveDeclLink(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
+    const decl = decl_index.get();
+    switch (decl.categorize()) {
+        .alias => |alias_decl| try alias_decl.get().fqn(out),
+        else => try decl.fqn(out),
+    }
+}

lib/docs/wasm/main.zig

@@ -1,15 +1,17 @@
-/// Delete this to find out where URL escaping needs to be added.
-const missing_feature_url_escape = true;
-
-const gpa = std.heap.wasm_allocator;
-
 const std = @import("std");
 const log = std.log;
 const assert = std.debug.assert;
 const Ast = std.zig.Ast;
-const Walk = @import("Walk.zig");
+const Walk = @import("Walk");
 const markdown = @import("markdown.zig");
-const Decl = @import("Decl.zig");
+const Decl = Walk.Decl;
+
+const fileSourceHtml = @import("html_render.zig").fileSourceHtml;
+const appendEscaped = @import("html_render.zig").appendEscaped;
+const resolveDeclLink = @import("html_render.zig").resolveDeclLink;
+const missing_feature_url_escape = @import("html_render.zig").missing_feature_url_escape;
+
+const gpa = std.heap.wasm_allocator;
 
 const js = struct {
     extern "js" fn log(ptr: [*]const u8, len: usize) void;
@@ -439,7 +441,7 @@ fn decl_field_html_fallible(
     const decl = decl_index.get();
     const ast = decl.file.get_ast();
     try out.appendSlice(gpa, "<pre><code>");
-    try file_source_html(decl.file, out, field_node, .{});
+    try fileSourceHtml(decl.file, out, field_node, .{});
     try out.appendSlice(gpa, "</code></pre>");
 
     const field = ast.fullContainerField(field_node).?;
@@ -478,7 +480,7 @@ fn decl_param_html_fallible(
     try out.appendSlice(gpa, "<pre><code>");
     try appendEscaped(out, name);
     try out.appendSlice(gpa, ": ");
-    try file_source_html(decl.file, out, param_node, .{});
+    try fileSourceHtml(decl.file, out, param_node, .{});
     try out.appendSlice(gpa, "</code></pre>");
 
     if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
@@ -506,7 +508,7 @@ export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) Stri
     };
 
     string_result.clearRetainingCapacity();
-    file_source_html(decl.file, &string_result, proto_node, .{
+    fileSourceHtml(decl.file, &string_result, proto_node, .{
         .skip_doc_comments = true,
         .skip_comments = true,
         .collapse_whitespace = true,
@@ -521,7 +523,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {
     const decl = decl_index.get();
 
     string_result.clearRetainingCapacity();
-    file_source_html(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
+    fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
         fatal("unable to render source: {s}", .{@errorName(err)});
     };
     return String.init(string_result.items);
@@ -533,7 +535,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
         return String.init("");
 
     string_result.clearRetainingCapacity();
-    file_source_html(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
+    fileSourceHtml(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
         fatal("unable to render source: {s}", .{@errorName(err)});
     };
     return String.init(string_result.items);
@@ -691,7 +693,7 @@ fn render_docs(
                         const content = doc.string(data.text.content);
                         if (resolve_decl_path(r.context, content)) |resolved_decl_index| {
                             g.link_buffer.clearRetainingCapacity();
-                            try resolve_decl_link(resolved_decl_index, &g.link_buffer);
+                            try resolveDeclLink(resolved_decl_index, &g.link_buffer);
 
                             try writer.writeAll("<a href=\"#");
                             _ = missing_feature_url_escape;
@@ -734,7 +736,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {
         if (ast.fullVarDecl(decl.ast_node)) |var_decl| {
             if (var_decl.ast.type_node != 0) {
                 string_result.appendSlice(gpa, "<code>") catch @panic("OOM");
-                file_source_html(decl.file, &string_result, var_decl.ast.type_node, .{
+                fileSourceHtml(decl.file, &string_result, var_decl.ast.type_node, .{
                     .skip_comments = true,
                     .collapse_whitespace = true,
                 }) catch |e| {
@@ -902,382 +904,6 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec
     return Slice(Decl.Index).init(g.members.items);
 }
 
-const RenderSourceOptions = struct {
-    skip_doc_comments: bool = false,
-    skip_comments: bool = false,
-    collapse_whitespace: bool = false,
-    fn_link: Decl.Index = .none,
-};
-
-fn file_source_html(
-    file_index: Walk.File.Index,
-    out: *std.ArrayListUnmanaged(u8),
-    root_node: Ast.Node.Index,
-    options: RenderSourceOptions,
-) !void {
-    const ast = file_index.get_ast();
-    const file = file_index.get();
-
-    const g = struct {
-        var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
-    };
-
-    const token_tags = ast.tokens.items(.tag);
-    const token_starts = ast.tokens.items(.start);
-    const main_tokens = ast.nodes.items(.main_token);
-
-    const start_token = ast.firstToken(root_node);
-    const end_token = ast.lastToken(root_node) + 1;
-
-    var cursor: usize = token_starts[start_token];
-
-    var indent: usize = 0;
-    if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
-        for (ast.source[newline_index + 1 .. cursor]) |c| {
-            if (c == ' ') {
-                indent += 1;
-            } else {
-                break;
-            }
-        }
-    }
-
-    for (
-        token_tags[start_token..end_token],
-        token_starts[start_token..end_token],
-        start_token..,
-    ) |tag, start, token_index| {
-        const between = ast.source[cursor..start];
-        if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
-            if (!options.skip_comments) {
-                try out.appendSlice(gpa, "<span class=\"tok-comment\">");
-                try appendUnindented(out, between, indent);
-                try out.appendSlice(gpa, "</span>");
-            }
-        } else if (between.len > 0) {
-            if (options.collapse_whitespace) {
-                if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
-                    try out.append(gpa, ' ');
-            } else {
-                try appendUnindented(out, between, indent);
-            }
-        }
-        if (tag == .eof) break;
-        const slice = ast.tokenSlice(token_index);
-        cursor = start + slice.len;
-        switch (tag) {
-            .eof => unreachable,
-
-            .keyword_addrspace,
-            .keyword_align,
-            .keyword_and,
-            .keyword_asm,
-            .keyword_async,
-            .keyword_await,
-            .keyword_break,
-            .keyword_catch,
-            .keyword_comptime,
-            .keyword_const,
-            .keyword_continue,
-            .keyword_defer,
-            .keyword_else,
-            .keyword_enum,
-            .keyword_errdefer,
-            .keyword_error,
-            .keyword_export,
-            .keyword_extern,
-            .keyword_for,
-            .keyword_if,
-            .keyword_inline,
-            .keyword_noalias,
-            .keyword_noinline,
-            .keyword_nosuspend,
-            .keyword_opaque,
-            .keyword_or,
-            .keyword_orelse,
-            .keyword_packed,
-            .keyword_anyframe,
-            .keyword_pub,
-            .keyword_resume,
-            .keyword_return,
-            .keyword_linksection,
-            .keyword_callconv,
-            .keyword_struct,
-            .keyword_suspend,
-            .keyword_switch,
-            .keyword_test,
-            .keyword_threadlocal,
-            .keyword_try,
-            .keyword_union,
-            .keyword_unreachable,
-            .keyword_usingnamespace,
-            .keyword_var,
-            .keyword_volatile,
-            .keyword_allowzero,
-            .keyword_while,
-            .keyword_anytype,
-            .keyword_fn,
-            => {
-                try out.appendSlice(gpa, "<span class=\"tok-kw\">");
-                try appendEscaped(out, slice);
-                try out.appendSlice(gpa, "</span>");
-            },
-
-            .string_literal,
-            .char_literal,
-            .multiline_string_literal_line,
-            => {
-                try out.appendSlice(gpa, "<span class=\"tok-str\">");
-                try appendEscaped(out, slice);
-                try out.appendSlice(gpa, "</span>");
-            },
-
-            .builtin => {
-                try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
-                try appendEscaped(out, slice);
-                try out.appendSlice(gpa, "</span>");
-            },
-
-            .doc_comment,
-            .container_doc_comment,
-            => {
-                if (!options.skip_doc_comments) {
-                    try out.appendSlice(gpa, "<span class=\"tok-comment\">");
-                    try appendEscaped(out, slice);
-                    try out.appendSlice(gpa, "</span>");
-                }
-            },
-
-            .identifier => i: {
-                if (options.fn_link != .none) {
-                    const fn_link = options.fn_link.get();
-                    const fn_token = main_tokens[fn_link.ast_node];
-                    if (token_index == fn_token + 1) {
-                        try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
-                        _ = missing_feature_url_escape;
-                        try fn_link.fqn(out);
-                        try out.appendSlice(gpa, "\">");
-                        try appendEscaped(out, slice);
-                        try out.appendSlice(gpa, "</a>");
-                        break :i;
-                    }
-                }
-
-                if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
-                    try out.appendSlice(gpa, "<span class=\"tok-fn\">");
-                    try appendEscaped(out, slice);
-                    try out.appendSlice(gpa, "</span>");
-                    break :i;
-                }
-
-                if (Walk.isPrimitiveNonType(slice)) {
-                    try out.appendSlice(gpa, "<span class=\"tok-null\">");
-                    try appendEscaped(out, slice);
-                    try out.appendSlice(gpa, "</span>");
-                    break :i;
-                }
-
-                if (std.zig.primitives.isPrimitive(slice)) {
-                    try out.appendSlice(gpa, "<span class=\"tok-type\">");
-                    try appendEscaped(out, slice);
-                    try out.appendSlice(gpa, "</span>");
-                    break :i;
-                }
-
-                if (file.token_parents.get(token_index)) |field_access_node| {
-                    g.field_access_buffer.clearRetainingCapacity();
-                    try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
-                    if (g.field_access_buffer.items.len > 0) {
-                        try out.appendSlice(gpa, "<a href=\"#");
-                        _ = missing_feature_url_escape;
-                        try out.appendSlice(gpa, g.field_access_buffer.items);
-                        try out.appendSlice(gpa, "\">");
-                        try appendEscaped(out, slice);
-                        try out.appendSlice(gpa, "</a>");
-                    } else {
-                        try appendEscaped(out, slice);
-                    }
-                    break :i;
-                }
-
-                {
-                    g.field_access_buffer.clearRetainingCapacity();
-                    try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
-                    if (g.field_access_buffer.items.len > 0) {
-                        try out.appendSlice(gpa, "<a href=\"#");
-                        _ = missing_feature_url_escape;
-                        try out.appendSlice(gpa, g.field_access_buffer.items);
-                        try out.appendSlice(gpa, "\">");
-                        try appendEscaped(out, slice);
-                        try out.appendSlice(gpa, "</a>");
-                        break :i;
-                    }
-                }
-
-                try appendEscaped(out, slice);
-            },
-
-            .number_literal => {
-                try out.appendSlice(gpa, "<span class=\"tok-number\">");
-                try appendEscaped(out, slice);
-                try out.appendSlice(gpa, "</span>");
-            },
-
-            .bang,
-            .pipe,
-            .pipe_pipe,
-            .pipe_equal,
-            .equal,
-            .equal_equal,
-            .equal_angle_bracket_right,
-            .bang_equal,
-            .l_paren,
-            .r_paren,
-            .semicolon,
-            .percent,
-            .percent_equal,
-            .l_brace,
-            .r_brace,
-            .l_bracket,
-            .r_bracket,
-            .period,
-            .period_asterisk,
-            .ellipsis2,
-            .ellipsis3,
-            .caret,
-            .caret_equal,
-            .plus,
-            .plus_plus,
-            .plus_equal,
-            .plus_percent,
-            .plus_percent_equal,
-            .plus_pipe,
-            .plus_pipe_equal,
-            .minus,
-            .minus_equal,
-            .minus_percent,
-            .minus_percent_equal,
-            .minus_pipe,
-            .minus_pipe_equal,
-            .asterisk,
-            .asterisk_equal,
-            .asterisk_asterisk,
-            .asterisk_percent,
-            .asterisk_percent_equal,
-            .asterisk_pipe,
-            .asterisk_pipe_equal,
-            .arrow,
-            .colon,
-            .slash,
-            .slash_equal,
-            .comma,
-            .ampersand,
-            .ampersand_equal,
-            .question_mark,
-            .angle_bracket_left,
-            .angle_bracket_left_equal,
-            .angle_bracket_angle_bracket_left,
-            .angle_bracket_angle_bracket_left_equal,
-            .angle_bracket_angle_bracket_left_pipe,
-            .angle_bracket_angle_bracket_left_pipe_equal,
-            .angle_bracket_right,
-            .angle_bracket_right_equal,
-            .angle_bracket_angle_bracket_right,
-            .angle_bracket_angle_bracket_right_equal,
-            .tilde,
-            => try appendEscaped(out, slice),
-
-            .invalid, .invalid_periodasterisks => return error.InvalidToken,
-        }
-    }
-}
-
-fn unindent(s: []const u8, indent: usize) []const u8 {
-    var indent_idx: usize = 0;
-    for (s) |c| {
-        if (c == ' ' and indent_idx < indent) {
-            indent_idx += 1;
-        } else {
-            break;
-        }
-    }
-    return s[indent_idx..];
-}
-
-fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
-    var it = std.mem.splitScalar(u8, s, '\n');
-    var is_first_line = true;
-    while (it.next()) |line| {
-        if (is_first_line) {
-            try appendEscaped(out, line);
-            is_first_line = false;
-        } else {
-            try out.appendSlice(gpa, "\n");
-            try appendEscaped(out, unindent(line, indent));
-        }
-    }
-}
-
-fn resolve_ident_link(
-    file_index: Walk.File.Index,
-    out: *std.ArrayListUnmanaged(u8),
-    ident_token: Ast.TokenIndex,
-) Oom!void {
-    const decl_index = file_index.get().lookup_token(ident_token);
-    if (decl_index == .none) return;
-    try resolve_decl_link(decl_index, out);
-}
-
-fn resolve_decl_link(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
-    const decl = decl_index.get();
-    switch (decl.categorize()) {
-        .alias => |alias_decl| try alias_decl.get().fqn(out),
-        else => try decl.fqn(out),
-    }
-}
-
-fn walk_field_accesses(
-    file_index: Walk.File.Index,
-    out: *std.ArrayListUnmanaged(u8),
-    node: Ast.Node.Index,
-) Oom!void {
-    const ast = file_index.get_ast();
-    const node_tags = ast.nodes.items(.tag);
-    assert(node_tags[node] == .field_access);
-    const node_datas = ast.nodes.items(.data);
-    const main_tokens = ast.nodes.items(.main_token);
-    const object_node = node_datas[node].lhs;
-    const dot_token = main_tokens[node];
-    const field_ident = dot_token + 1;
-    switch (node_tags[object_node]) {
-        .identifier => {
-            const lhs_ident = main_tokens[object_node];
-            try resolve_ident_link(file_index, out, lhs_ident);
-        },
-        .field_access => {
-            try walk_field_accesses(file_index, out, object_node);
-        },
-        else => {},
-    }
-    if (out.items.len > 0) {
-        try out.append(gpa, '.');
-        try out.appendSlice(gpa, ast.tokenSlice(field_ident));
-    }
-}
-
-fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
-    for (s) |c| {
-        try out.ensureUnusedCapacity(gpa, 6);
-        switch (c) {
-            '&' => out.appendSliceAssumeCapacity("&amp;"),
-            '<' => out.appendSliceAssumeCapacity("&lt;"),
-            '>' => out.appendSliceAssumeCapacity("&gt;"),
-            '"' => out.appendSliceAssumeCapacity("&quot;"),
-            else => out.appendAssumeCapacity(c),
-        }
-    }
-}
-
 fn count_scalar(haystack: []const u8, needle: u8) usize {
     var total: usize = 0;
     for (haystack) |elem| {

lib/fuzzer/index.html

@@ -2,12 +2,56 @@
 <html>
   <head>
     <meta charset="utf-8">
-    <title>Zig Documentation</title>
+    <title>Zig Build System Interface</title>
     <style type="text/css">
       body {
         font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;
         color: #000000;
       }
+      .hidden {
+        display: none;
+      }
+      table {
+        width: 100%;
+      }
+      a {
+        color: #2A6286;
+      }
+      pre{
+        font-family:"Source Code Pro",monospace;
+        font-size:1em;
+        background-color:#F5F5F5;
+        padding: 1em;
+        margin: 0;
+        overflow-x: auto;
+      }
+      :not(pre) > code {
+        white-space: break-spaces;
+      }
+      code {
+        font-family:"Source Code Pro",monospace;
+        font-size: 0.9em;
+      }
+      code a {
+        color: #000000;
+      }
+      kbd {
+        color: #000;
+        background-color: #fafbfc;
+        border-color: #d1d5da;
+        border-bottom-color: #c6cbd1;
+        box-shadow-color: #c6cbd1;
+        display: inline-block;
+        padding: 0.3em 0.2em;
+        font: 1.2em monospace;
+        line-height: 0.8em;
+        vertical-align: middle;
+        border: solid 1px;
+        border-radius: 3px;
+        box-shadow: inset 0 -1px 0;
+        cursor: default;
+      }
+
       .tok-kw {
           color: #333;
           font-weight: bold;
@@ -42,6 +86,16 @@
           background-color: #111;
           color: #bbb;
         }
+        pre {
+          background-color: #222;
+          color: #ccc;
+        }
+        a {
+          color: #88f;
+        }
+        code a {
+          color: #ccc;
+        }
         .tok-kw {
             color: #eee;
         }
@@ -70,6 +124,10 @@
     </style>
   </head>
   <body>
+    <div id="sectSource" class="hidden">
+      <h2>Source Code</h2>
+      <pre><code id="sourceText"></code></pre>
+    </div>
     <script src="main.js"></script>
   </body>
 </html>

lib/fuzzer/main.js

@@ -1,4 +1,7 @@
 (function() {
+    const domSectSource = document.getElementById("sectSource");
+    const domSourceText = document.getElementById("sourceText");
+
     let wasm_promise = fetch("main.wasm");
     let sources_promise = fetch("sources.tar").then(function(response) {
       if (!response.ok) throw new Error("unable to download sources");
@@ -30,11 +33,56 @@
         const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
         wasm_array.set(js_array);
         wasm_exports.unpack(ptr, js_array.length);
+
+        render();
       });
     });
 
+    function render() {
+        domSectSource.classList.add("hidden");
+
+        // TODO this is temporary debugging data
+        renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
+    }
+
+    function renderSource(path) {
+      const decl_index = findFileRoot(path);
+      if (decl_index == null) throw new Error("file not found: " + path);
+
+      const h2 = domSectSource.children[0];
+      h2.innerText = path;
+      domSourceText.innerHTML = declSourceHtml(decl_index);
+
+      domSectSource.classList.remove("hidden");
+    }
+
+    function findFileRoot(path) {
+      setInputString(path);
+      const result = wasm_exports.find_file_root();
+      if (result === -1) return null;
+      return result;
+    }
+
     function decodeString(ptr, len) {
       if (len === 0) return "";
       return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
     }
+
+    function setInputString(s) {
+      const jsArray = text_encoder.encode(s);
+      const len = jsArray.length;
+      const ptr = wasm_exports.set_input_string(len);
+      const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
+      wasmArray.set(jsArray);
+    }
+
+    function declSourceHtml(decl_index) {
+      return unwrapString(wasm_exports.decl_source_html(decl_index));
+    }
+
+    function unwrapString(bigint) {
+      const ptr = Number(bigint & 0xffffffffn);
+      const len = Number(bigint >> 32n);
+      return decodeString(ptr, len);
+    }
 })();

lib/fuzzer/wasm/main.zig

@@ -2,6 +2,8 @@ const std = @import("std");
 const assert = std.debug.assert;
 
 const Walk = @import("Walk");
+const Decl = Walk.Decl;
+const html_render = @import("html_render");
 
 const gpa = std.heap.wasm_allocator;
 const log = std.log;
@@ -52,6 +54,48 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
     };
 }
 
+/// Set by `set_input_string`.
+var input_string: std.ArrayListUnmanaged(u8) = .{};
+var string_result: std.ArrayListUnmanaged(u8) = .{};
+
+export fn set_input_string(len: usize) [*]u8 {
+    input_string.resize(gpa, len) catch @panic("OOM");
+    return input_string.items.ptr;
+}
+
+/// Looks up the root struct decl corresponding to a file by path.
+/// Uses `input_string`.
+export fn find_file_root() Decl.Index {
+    const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none);
+    return file.findRootDecl();
+}
+
+export fn decl_source_html(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+
+    string_result.clearRetainingCapacity();
+    html_render.fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
+        fatal("unable to render source: {s}", .{@errorName(err)});
+    };
+    return String.init(string_result.items);
+}
+
+const String = Slice(u8);
+
+fn Slice(T: type) type {
+    return packed struct(u64) {
+        ptr: u32,
+        len: u32,
+
+        fn init(s: []const T) @This() {
+            return .{
+                .ptr = @intFromPtr(s.ptr),
+                .len = s.len,
+            };
+        }
+    };
+}
+
 fn unpackInner(tar_bytes: []u8) !void {
     var fbs = std.io.fixedBufferStream(tar_bytes);
     var file_name_buffer: [1024]u8 = undefined;

lib/std/Build/Fuzz.zig

@@ -235,30 +235,29 @@ pub const WebServer = struct {
             .root_dir = ws.zig_lib_directory,
             .sub_path = "docs/wasm/Walk.zig",
         };
+        const html_render_src_path: Build.Cache.Path = .{
+            .root_dir = ws.zig_lib_directory,
+            .sub_path = "docs/wasm/html_render.zig",
+        };
 
         var argv: std.ArrayListUnmanaged([]const u8) = .{};
 
         try argv.appendSlice(arena, &.{
-            ws.zig_exe_path,
+            ws.zig_exe_path, "build-exe", //
-            "build-exe",
+            "-fno-entry", //
-            "-fno-entry",
+            "-O", @tagName(optimize_mode), //
-            "-O",
+            "-target", "wasm32-freestanding", //
-            @tagName(optimize_mode),
+            "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
-            "-target",
+            "--cache-dir", ws.global_cache_directory.path orelse ".", //
-            "wasm32-freestanding",
+            "--global-cache-dir", ws.global_cache_directory.path orelse ".", //
-            "-mcpu",
+            "--name", "fuzzer", //
-            "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",
+            "-rdynamic", //
-            "--cache-dir",
+            "--dep", "Walk", //
-            ws.global_cache_directory.path orelse ".",
+            "--dep", "html_render", //
-            "--global-cache-dir",
+            try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
-            ws.global_cache_directory.path orelse ".",
+            try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
-            "--name",
+            "--dep", "Walk", //
-            "fuzzer",
+            try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
-            "-rdynamic",
-            "--dep",
-            "Walk",
-            try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}),
-            try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}),
             "--listen=-",
         });