upstream new autodocs implementation

lib/docs/wasm/Decl.zig

@@ -0,0 +1,180 @@
+ast_node: Ast.Node.Index,
+file: Walk.File.Index,
+/// The decl whose namespace this is in.
+parent: Index,
+
+pub const ExtraInfo = struct {
+    is_pub: bool,
+    name: []const u8,
+    /// This might not be a doc_comment token in which case there are no doc comments.
+    first_doc_comment: Ast.TokenIndex,
+};
+
+pub const Index = enum(u32) {
+    none = std.math.maxInt(u32),
+    _,
+
+    pub fn get(i: Index) *Decl {
+        return &Walk.decls.items[@intFromEnum(i)];
+    }
+};
+
+pub fn is_pub(d: *const Decl) bool {
+    return d.extra_info().is_pub;
+}
+
+pub fn extra_info(d: *const Decl) ExtraInfo {
+    const ast = d.file.get_ast();
+    const token_tags = ast.tokens.items(.tag);
+    const node_tags = ast.nodes.items(.tag);
+    switch (node_tags[d.ast_node]) {
+        .root => return .{
+            .name = "",
+            .is_pub = true,
+            .first_doc_comment = if (token_tags[0] == .container_doc_comment)
+                0
+            else
+                token_tags.len - 1,
+        },
+
+        .global_var_decl,
+        .local_var_decl,
+        .simple_var_decl,
+        .aligned_var_decl,
+        => {
+            const var_decl = ast.fullVarDecl(d.ast_node).?;
+            const name_token = var_decl.ast.mut_token + 1;
+            assert(token_tags[name_token] == .identifier);
+            const ident_name = ast.tokenSlice(name_token);
+            return .{
+                .name = ident_name,
+                .is_pub = var_decl.visib_token != null,
+                .first_doc_comment = findFirstDocComment(ast, var_decl.firstToken()),
+            };
+        },
+
+        .fn_proto,
+        .fn_proto_multi,
+        .fn_proto_one,
+        .fn_proto_simple,
+        .fn_decl,
+        => {
+            var buf: [1]Ast.Node.Index = undefined;
+            const fn_proto = ast.fullFnProto(&buf, d.ast_node).?;
+            const name_token = fn_proto.name_token.?;
+            assert(token_tags[name_token] == .identifier);
+            const ident_name = ast.tokenSlice(name_token);
+            return .{
+                .name = ident_name,
+                .is_pub = fn_proto.visib_token != null,
+                .first_doc_comment = findFirstDocComment(ast, fn_proto.firstToken()),
+            };
+        },
+
+        else => |t| {
+            log.debug("hit '{s}'", .{@tagName(t)});
+            unreachable;
+        },
+    }
+}
+
+pub fn value_node(d: *const Decl) ?Ast.Node.Index {
+    const ast = d.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const token_tags = ast.tokens.items(.tag);
+    return switch (node_tags[d.ast_node]) {
+        .fn_proto,
+        .fn_proto_multi,
+        .fn_proto_one,
+        .fn_proto_simple,
+        .fn_decl,
+        .root,
+        => d.ast_node,
+
+        .global_var_decl,
+        .local_var_decl,
+        .simple_var_decl,
+        .aligned_var_decl,
+        => {
+            const var_decl = ast.fullVarDecl(d.ast_node).?;
+            if (token_tags[var_decl.ast.mut_token] == .keyword_const)
+                return var_decl.ast.init_node;
+
+            return null;
+        },
+
+        else => null,
+    };
+}
+
+pub fn categorize(decl: *const Decl) Walk.Category {
+    return decl.file.categorize_decl(decl.ast_node);
+}
+
+pub fn fqn(decl: *const Decl, out: *std.ArrayListUnmanaged(u8)) Oom!void {
+    try decl.reset_with_path(out);
+    if (decl.parent != .none) {
+        try append_parent_ns(out, decl.parent);
+        try out.appendSlice(gpa, decl.extra_info().name);
+    } else {
+        out.items.len -= 1; // remove the trailing '.'
+    }
+}
+
+pub fn reset_with_path(decl: *const Decl, list: *std.ArrayListUnmanaged(u8)) Oom!void {
+    list.clearRetainingCapacity();
+
+    // Prefer the package name alias.
+    for (Walk.packages.keys(), Walk.packages.values()) |pkg_name, pkg_file| {
+        if (pkg_file == decl.file) {
+            try list.ensureUnusedCapacity(gpa, pkg_name.len + 1);
+            list.appendSliceAssumeCapacity(pkg_name);
+            list.appendAssumeCapacity('.');
+            return;
+        }
+    }
+
+    const file_path = decl.file.path();
+    try list.ensureUnusedCapacity(gpa, file_path.len + 1);
+    list.appendSliceAssumeCapacity(file_path);
+    for (list.items) |*byte| switch (byte.*) {
+        '/' => byte.* = '.',
+        else => continue,
+    };
+    if (std.mem.endsWith(u8, list.items, ".zig")) {
+        list.items.len -= 3;
+    } else {
+        list.appendAssumeCapacity('.');
+    }
+}
+
+pub fn append_parent_ns(list: *std.ArrayListUnmanaged(u8), parent: Decl.Index) Oom!void {
+    assert(parent != .none);
+    const decl = parent.get();
+    if (decl.parent != .none) {
+        try append_parent_ns(list, decl.parent);
+        try list.appendSlice(gpa, decl.extra_info().name);
+        try list.append(gpa, '.');
+    }
+}
+
+pub fn findFirstDocComment(ast: *const Ast, token: Ast.TokenIndex) Ast.TokenIndex {
+    const token_tags = ast.tokens.items(.tag);
+    var it = token;
+    while (it > 0) {
+        it -= 1;
+        if (token_tags[it] != .doc_comment) {
+            return it + 1;
+        }
+    }
+    return it;
+}
+
+const Decl = @This();
+const std = @import("std");
+const Ast = std.zig.Ast;
+const Walk = @import("Walk.zig");
+const gpa = std.heap.wasm_allocator;
+const assert = std.debug.assert;
+const log = std.log;
+const Oom = error{OutOfMemory};

lib/docs/wasm/Walk.zig

@@ -0,0 +1,928 @@
+//! Find and annotate identifiers with links to their declarations.
+pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
+pub var decls: std.ArrayListUnmanaged(Decl) = .{};
+pub var packages: std.StringArrayHashMapUnmanaged(File.Index) = .{};
+
+arena: std.mem.Allocator,
+file: File.Index,
+
+/// keep in sync with "CAT_" constants in main.js
+pub const Category = union(enum(u8)) {
+    namespace: Ast.Node.Index,
+    global_variable: Ast.Node.Index,
+    function: Ast.Node.Index,
+    primitive: Ast.Node.Index,
+    error_set: Ast.Node.Index,
+    global_const: Ast.Node.Index,
+    alias: Decl.Index,
+    type,
+
+    pub const Tag = @typeInfo(Category).Union.tag_type.?;
+};
+
+pub const File = struct {
+    ast: Ast,
+    /// Maps identifiers to the declarations they point to.
+    ident_decls: std.AutoArrayHashMapUnmanaged(Ast.TokenIndex, Ast.Node.Index) = .{},
+    /// Maps field access identifiers to the containing field access node.
+    token_parents: std.AutoArrayHashMapUnmanaged(Ast.TokenIndex, Ast.Node.Index) = .{},
+    /// Maps declarations to their global index.
+    node_decls: std.AutoArrayHashMapUnmanaged(Ast.Node.Index, Decl.Index) = .{},
+    /// Maps function declarations to doctests.
+    doctests: std.AutoArrayHashMapUnmanaged(Ast.Node.Index, Ast.Node.Index) = .{},
+
+    pub fn lookup_token(file: *File, token: Ast.TokenIndex) Decl.Index {
+        const decl_node = file.ident_decls.get(token) orelse return .none;
+        return file.node_decls.get(decl_node) orelse return .none;
+    }
+
+    pub const Index = enum(u32) {
+        _,
+
+        fn add_decl(i: Index, node: Ast.Node.Index, parent_decl: Decl.Index) Oom!Decl.Index {
+            try decls.append(gpa, .{
+                .ast_node = node,
+                .file = i,
+                .parent = parent_decl,
+            });
+            const decl_index: Decl.Index = @enumFromInt(decls.items.len - 1);
+            try i.get().node_decls.put(gpa, node, decl_index);
+            return decl_index;
+        }
+
+        pub fn get(i: File.Index) *File {
+            return &files.values()[@intFromEnum(i)];
+        }
+
+        pub fn get_ast(i: File.Index) *Ast {
+            return &i.get().ast;
+        }
+
+        pub fn path(i: File.Index) []const u8 {
+            return files.keys()[@intFromEnum(i)];
+        }
+
+        pub fn findRootDecl(file_index: File.Index) Decl.Index {
+            return file_index.get().node_decls.values()[0];
+        }
+
+        pub fn categorize_decl(file_index: File.Index, node: Ast.Node.Index) Category {
+            const ast = file_index.get_ast();
+            const node_tags = ast.nodes.items(.tag);
+            const token_tags = ast.tokens.items(.tag);
+            switch (node_tags[node]) {
+                .root => return .{ .namespace = node },
+
+                .global_var_decl,
+                .local_var_decl,
+                .simple_var_decl,
+                .aligned_var_decl,
+                => {
+                    const var_decl = ast.fullVarDecl(node).?;
+                    if (token_tags[var_decl.ast.mut_token] == .keyword_var)
+                        return .{ .global_variable = node };
+
+                    return categorize_expr(file_index, var_decl.ast.init_node);
+                },
+
+                .fn_proto,
+                .fn_proto_multi,
+                .fn_proto_one,
+                .fn_proto_simple,
+                .fn_decl,
+                => return .{ .function = node },
+
+                else => unreachable,
+            }
+        }
+
+        pub fn categorize_expr(file_index: File.Index, node: Ast.Node.Index) Category {
+            const ast = file_index.get_ast();
+            const node_tags = ast.nodes.items(.tag);
+            const node_datas = ast.nodes.items(.data);
+            return switch (node_tags[node]) {
+                .container_decl,
+                .container_decl_trailing,
+                .container_decl_arg,
+                .container_decl_arg_trailing,
+                .container_decl_two,
+                .container_decl_two_trailing,
+                .tagged_union,
+                .tagged_union_trailing,
+                .tagged_union_enum_tag,
+                .tagged_union_enum_tag_trailing,
+                .tagged_union_two,
+                .tagged_union_two_trailing,
+                => .{ .namespace = node },
+
+                .error_set_decl,
+                => .{ .error_set = node },
+
+                .identifier => {
+                    const name_token = ast.nodes.items(.main_token)[node];
+                    const ident_name = ast.tokenSlice(name_token);
+                    if (std.zig.primitives.isPrimitive(ident_name)) {
+                        return .{ .primitive = node };
+                    }
+
+                    const decl_index = file_index.get().lookup_token(name_token);
+                    if (decl_index != .none) return .{ .alias = decl_index };
+
+                    return .{ .global_const = node };
+                },
+
+                .field_access => {
+                    // TODO:
+                    //return .alias;
+                    return .{ .global_const = node };
+                },
+
+                .builtin_call_two, .builtin_call_two_comma => {
+                    if (node_datas[node].lhs == 0) {
+                        const params = [_]Ast.Node.Index{};
+                        return categorize_builtin_call(file_index, node, &params);
+                    } else if (node_datas[node].rhs == 0) {
+                        const params = [_]Ast.Node.Index{node_datas[node].lhs};
+                        return categorize_builtin_call(file_index, node, &params);
+                    } else {
+                        const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
+                        return categorize_builtin_call(file_index, node, &params);
+                    }
+                },
+                .builtin_call, .builtin_call_comma => {
+                    const params = ast.extra_data[node_datas[node].lhs..node_datas[node].rhs];
+                    return categorize_builtin_call(file_index, node, params);
+                },
+
+                else => .{ .global_const = node },
+            };
+        }
+
+        fn categorize_builtin_call(
+            file_index: File.Index,
+            node: Ast.Node.Index,
+            params: []const Ast.Node.Index,
+        ) Category {
+            const ast = file_index.get_ast();
+            const main_tokens = ast.nodes.items(.main_token);
+            const builtin_token = main_tokens[node];
+            const builtin_name = ast.tokenSlice(builtin_token);
+            if (std.mem.eql(u8, builtin_name, "@import")) {
+                const str_lit_token = main_tokens[params[0]];
+                const str_bytes = ast.tokenSlice(str_lit_token);
+                const file_path = std.zig.string_literal.parseAlloc(gpa, str_bytes) catch @panic("OOM");
+                defer gpa.free(file_path);
+                const base_path = file_index.path();
+                const resolved_path = std.fs.path.resolvePosix(gpa, &.{
+                    base_path, "..", file_path,
+                }) catch @panic("OOM");
+                defer gpa.free(resolved_path);
+                log.debug("from '{s}' @import '{s}' resolved='{s}'", .{
+                    base_path, file_path, resolved_path,
+                });
+                if (Walk.files.getIndex(resolved_path)) |imported_file_index| {
+                    return .{ .alias = Walk.File.Index.findRootDecl(@enumFromInt(imported_file_index)) };
+                } else {
+                    log.warn("import target '{s}' did not resolve to any file", .{resolved_path});
+                }
+            } else if (std.mem.eql(u8, builtin_name, "@This")) {
+                if (file_index.get().node_decls.get(node)) |decl_index| {
+                    return .{ .alias = decl_index };
+                } else {
+                    log.warn("@This() is missing link to Decl.Index", .{});
+                }
+            }
+
+            return .{ .global_const = node };
+        }
+    };
+};
+
+pub const PackageIndex = enum(u32) {
+    _,
+};
+
+pub fn add_file(file_name: []const u8, bytes: []u8) !File.Index {
+    const ast = try parse(bytes);
+    const file_index: File.Index = @enumFromInt(files.entries.len);
+    try files.put(gpa, file_name, .{ .ast = ast });
+
+    if (ast.errors.len > 0) {
+        // TODO: expose this in the UI
+        log.err("can't index '{s}' because it has syntax errors", .{file_index.path()});
+        return file_index;
+    }
+
+    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+    defer arena_instance.deinit();
+
+    var w: Walk = .{
+        .arena = arena_instance.allocator(),
+        .file = file_index,
+    };
+    var scope: Scope = .{ .tag = .top };
+
+    const decl_index = try file_index.add_decl(0, .none);
+    try struct_decl(&w, &scope, decl_index, ast.containerDeclRoot());
+
+    const file = file_index.get();
+    shrinkToFit(&file.ident_decls);
+    shrinkToFit(&file.token_parents);
+    shrinkToFit(&file.node_decls);
+    shrinkToFit(&file.doctests);
+
+    return file_index;
+}
+
+fn parse(source: []u8) Oom!Ast {
+    // Require every source file to end with a newline so that Zig's tokenizer
+    // can continue to require null termination and Autodoc implementation can
+    // avoid copying source bytes from the decompressed tar file buffer.
+    const adjusted_source: [:0]const u8 = s: {
+        if (source.len == 0)
+            break :s "";
+
+        assert(source[source.len - 1] == '\n');
+        source[source.len - 1] = 0;
+        break :s source[0 .. source.len - 1 :0];
+    };
+
+    return Ast.parse(gpa, adjusted_source, .zig);
+}
+
+const Scope = struct {
+    tag: Tag,
+
+    const Tag = enum { top, local, namespace };
+
+    const Local = struct {
+        base: Scope = .{ .tag = .local },
+        parent: *Scope,
+        var_node: Ast.Node.Index,
+    };
+
+    const Namespace = struct {
+        base: Scope = .{ .tag = .namespace },
+        parent: *Scope,
+        names: std.StringArrayHashMapUnmanaged(Ast.Node.Index) = .{},
+        doctests: std.StringArrayHashMapUnmanaged(Ast.Node.Index) = .{},
+        decl_index: Decl.Index,
+    };
+
+    fn getNamespaceDecl(start_scope: *Scope) Decl.Index {
+        var it: *Scope = start_scope;
+        while (true) switch (it.tag) {
+            .top => unreachable,
+            .local => {
+                const local = @fieldParentPtr(Local, "base", it);
+                it = local.parent;
+            },
+            .namespace => {
+                const namespace = @fieldParentPtr(Namespace, "base", it);
+                return namespace.decl_index;
+            },
+        };
+    }
+
+    fn lookup(start_scope: *Scope, ast: *const Ast, name: []const u8) ?Ast.Node.Index {
+        const main_tokens = ast.nodes.items(.main_token);
+        var it: *Scope = start_scope;
+        while (true) switch (it.tag) {
+            .top => break,
+            .local => {
+                const local = @fieldParentPtr(Local, "base", it);
+                const name_token = main_tokens[local.var_node] + 1;
+                const ident_name = ast.tokenSlice(name_token);
+                if (std.mem.eql(u8, ident_name, name)) {
+                    return local.var_node;
+                }
+                it = local.parent;
+            },
+            .namespace => {
+                const namespace = @fieldParentPtr(Namespace, "base", it);
+                if (namespace.names.get(name)) |node| {
+                    return node;
+                }
+                it = namespace.parent;
+            },
+        };
+        return null;
+    }
+};
+
+fn struct_decl(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    container_decl: Ast.full.ContainerDecl,
+) Oom!void {
+    const ast = w.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+
+    var namespace: Scope.Namespace = .{
+        .parent = scope,
+        .decl_index = parent_decl,
+    };
+    try w.scanDecls(&namespace, container_decl.ast.members);
+
+    for (container_decl.ast.members) |member| switch (node_tags[member]) {
+        .container_field_init,
+        .container_field_align,
+        .container_field,
+        => try w.container_field(&namespace.base, parent_decl, ast.fullContainerField(member).?),
+
+        .fn_proto,
+        .fn_proto_multi,
+        .fn_proto_one,
+        .fn_proto_simple,
+        .fn_decl,
+        => {
+            var buf: [1]Ast.Node.Index = undefined;
+            const full = ast.fullFnProto(&buf, member).?;
+            const fn_name_token = full.ast.fn_token + 1;
+            const fn_name = ast.tokenSlice(fn_name_token);
+            if (namespace.doctests.get(fn_name)) |doctest_node| {
+                try w.file.get().doctests.put(gpa, member, doctest_node);
+            }
+            const decl_index = try w.file.add_decl(member, parent_decl);
+            const body = if (node_tags[member] == .fn_decl) node_datas[member].rhs else 0;
+            try w.fn_decl(&namespace.base, decl_index, body, full);
+        },
+
+        .global_var_decl,
+        .local_var_decl,
+        .simple_var_decl,
+        .aligned_var_decl,
+        => {
+            const decl_index = try w.file.add_decl(member, parent_decl);
+            try w.global_var_decl(&namespace.base, decl_index, ast.fullVarDecl(member).?);
+        },
+
+        .@"comptime",
+        .@"usingnamespace",
+        => try w.expr(&namespace.base, parent_decl, node_datas[member].lhs),
+
+        .test_decl => try w.expr(&namespace.base, parent_decl, node_datas[member].rhs),
+
+        else => unreachable,
+    };
+}
+
+fn comptime_decl(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    full: Ast.full.VarDecl,
+) Oom!void {
+    try w.expr(scope, parent_decl, full.ast.type_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.align_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.addrspace_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.section_node);
+    try w.expr(scope, parent_decl, full.ast.init_node);
+}
+
+fn global_var_decl(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    full: Ast.full.VarDecl,
+) Oom!void {
+    try w.maybe_expr(scope, parent_decl, full.ast.type_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.align_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.addrspace_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.section_node);
+    try w.maybe_expr(scope, parent_decl, full.ast.init_node);
+}
+
+fn container_field(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    full: Ast.full.ContainerField,
+) Oom!void {
+    try w.maybe_expr(scope, parent_decl, full.ast.type_expr);
+    try w.maybe_expr(scope, parent_decl, full.ast.align_expr);
+    try w.maybe_expr(scope, parent_decl, full.ast.value_expr);
+}
+
+fn fn_decl(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    body: Ast.Node.Index,
+    full: Ast.full.FnProto,
+) Oom!void {
+    for (full.ast.params) |param| {
+        try expr(w, scope, parent_decl, param);
+    }
+    try expr(w, scope, parent_decl, full.ast.return_type);
+    try maybe_expr(w, scope, parent_decl, full.ast.align_expr);
+    try maybe_expr(w, scope, parent_decl, full.ast.addrspace_expr);
+    try maybe_expr(w, scope, parent_decl, full.ast.section_expr);
+    try maybe_expr(w, scope, parent_decl, full.ast.callconv_expr);
+    try maybe_expr(w, scope, parent_decl, body);
+}
+
+fn maybe_expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, node: Ast.Node.Index) Oom!void {
+    if (node != 0) return expr(w, scope, parent_decl, node);
+}
+
+fn expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, node: Ast.Node.Index) Oom!void {
+    assert(node != 0);
+    const ast = w.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+    const main_tokens = ast.nodes.items(.main_token);
+    switch (node_tags[node]) {
+        .root => unreachable, // Top-level declaration.
+        .@"usingnamespace" => unreachable, // Top-level declaration.
+        .test_decl => unreachable, // Top-level declaration.
+        .container_field_init => unreachable, // Top-level declaration.
+        .container_field_align => unreachable, // Top-level declaration.
+        .container_field => unreachable, // Top-level declaration.
+        .fn_decl => unreachable, // Top-level declaration.
+
+        .global_var_decl => unreachable, // Handled in `block`.
+        .local_var_decl => unreachable, // Handled in `block`.
+        .simple_var_decl => unreachable, // Handled in `block`.
+        .aligned_var_decl => unreachable, // Handled in `block`.
+        .@"defer" => unreachable, // Handled in `block`.
+        .@"errdefer" => unreachable, // Handled in `block`.
+
+        .switch_case => unreachable, // Handled in `switchExpr`.
+        .switch_case_inline => unreachable, // Handled in `switchExpr`.
+        .switch_case_one => unreachable, // Handled in `switchExpr`.
+        .switch_case_inline_one => unreachable, // Handled in `switchExpr`.
+
+        .asm_output => unreachable, // Handled in `asmExpr`.
+        .asm_input => unreachable, // Handled in `asmExpr`.
+
+        .for_range => unreachable, // Handled in `forExpr`.
+
+        .assign,
+        .assign_shl,
+        .assign_shl_sat,
+        .assign_shr,
+        .assign_bit_and,
+        .assign_bit_or,
+        .assign_bit_xor,
+        .assign_div,
+        .assign_sub,
+        .assign_sub_wrap,
+        .assign_sub_sat,
+        .assign_mod,
+        .assign_add,
+        .assign_add_wrap,
+        .assign_add_sat,
+        .assign_mul,
+        .assign_mul_wrap,
+        .assign_mul_sat,
+        .shl,
+        .shr,
+        .add,
+        .add_wrap,
+        .add_sat,
+        .sub,
+        .sub_wrap,
+        .sub_sat,
+        .mul,
+        .mul_wrap,
+        .mul_sat,
+        .div,
+        .mod,
+        .shl_sat,
+
+        .bit_and,
+        .bit_or,
+        .bit_xor,
+        .bang_equal,
+        .equal_equal,
+        .greater_than,
+        .greater_or_equal,
+        .less_than,
+        .less_or_equal,
+        .array_cat,
+
+        .array_mult,
+        .error_union,
+        .merge_error_sets,
+        .bool_and,
+        .bool_or,
+        .@"catch",
+        .@"orelse",
+        .array_type,
+        .array_access,
+        .switch_range,
+        => {
+            try expr(w, scope, parent_decl, node_datas[node].lhs);
+            try expr(w, scope, parent_decl, node_datas[node].rhs);
+        },
+
+        .assign_destructure => {
+            const extra_index = node_datas[node].lhs;
+            const lhs_count = ast.extra_data[extra_index];
+            const lhs_nodes: []const Ast.Node.Index = @ptrCast(ast.extra_data[extra_index + 1 ..][0..lhs_count]);
+            const rhs = node_datas[node].rhs;
+            for (lhs_nodes) |lhs_node| try expr(w, scope, parent_decl, lhs_node);
+            _ = try expr(w, scope, parent_decl, rhs);
+        },
+
+        .bool_not,
+        .bit_not,
+        .negation,
+        .negation_wrap,
+        .@"return",
+        .deref,
+        .address_of,
+        .optional_type,
+        .unwrap_optional,
+        .grouped_expression,
+        .@"comptime",
+        .@"nosuspend",
+        .@"suspend",
+        .@"await",
+        .@"resume",
+        .@"try",
+        => try maybe_expr(w, scope, parent_decl, node_datas[node].lhs),
+
+        .anyframe_type,
+        .@"break",
+        => try maybe_expr(w, scope, parent_decl, node_datas[node].rhs),
+
+        .identifier => {
+            const ident_token = main_tokens[node];
+            const ident_name = ast.tokenSlice(ident_token);
+            if (scope.lookup(ast, ident_name)) |var_node| {
+                try w.file.get().ident_decls.put(gpa, ident_token, var_node);
+            }
+        },
+        .field_access => {
+            const object_node = node_datas[node].lhs;
+            const dot_token = main_tokens[node];
+            const field_ident = dot_token + 1;
+            try w.file.get().token_parents.put(gpa, field_ident, node);
+            // This will populate the left-most field object if it is an
+            // identifier, allowing rendering code to piece together the link.
+            try expr(w, scope, parent_decl, object_node);
+        },
+
+        .string_literal,
+        .multiline_string_literal,
+        .number_literal,
+        .unreachable_literal,
+        .enum_literal,
+        .error_value,
+        .anyframe_literal,
+        .@"continue",
+        .char_literal,
+        .error_set_decl,
+        => {},
+
+        .asm_simple,
+        .@"asm",
+        => {
+            const full = ast.fullAsm(node).?;
+            for (full.ast.items) |n| {
+                // TODO handle .asm_input, .asm_output
+                _ = n;
+            }
+            try expr(w, scope, parent_decl, full.ast.template);
+        },
+
+        .builtin_call_two, .builtin_call_two_comma => {
+            if (node_datas[node].lhs == 0) {
+                const params = [_]Ast.Node.Index{};
+                return builtin_call(w, scope, parent_decl, node, &params);
+            } else if (node_datas[node].rhs == 0) {
+                const params = [_]Ast.Node.Index{node_datas[node].lhs};
+                return builtin_call(w, scope, parent_decl, node, &params);
+            } else {
+                const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
+                return builtin_call(w, scope, parent_decl, node, &params);
+            }
+        },
+        .builtin_call, .builtin_call_comma => {
+            const params = ast.extra_data[node_datas[node].lhs..node_datas[node].rhs];
+            return builtin_call(w, scope, parent_decl, node, params);
+        },
+
+        .call_one,
+        .call_one_comma,
+        .async_call_one,
+        .async_call_one_comma,
+        .call,
+        .call_comma,
+        .async_call,
+        .async_call_comma,
+        => {
+            var buf: [1]Ast.Node.Index = undefined;
+            const full = ast.fullCall(&buf, node).?;
+            try expr(w, scope, parent_decl, full.ast.fn_expr);
+            for (full.ast.params) |param| {
+                try expr(w, scope, parent_decl, param);
+            }
+        },
+
+        .if_simple,
+        .@"if",
+        => {
+            const full = ast.fullIf(node).?;
+            try expr(w, scope, parent_decl, full.ast.cond_expr);
+            try expr(w, scope, parent_decl, full.ast.then_expr);
+            try maybe_expr(w, scope, parent_decl, full.ast.else_expr);
+        },
+
+        .while_simple,
+        .while_cont,
+        .@"while",
+        => {
+            try while_expr(w, scope, parent_decl, ast.fullWhile(node).?);
+        },
+
+        .for_simple, .@"for" => {
+            const full = ast.fullFor(node).?;
+            for (full.ast.inputs) |input| {
+                if (node_tags[input] == .for_range) {
+                    try expr(w, scope, parent_decl, node_datas[input].lhs);
+                    try maybe_expr(w, scope, parent_decl, node_datas[input].rhs);
+                } else {
+                    try expr(w, scope, parent_decl, input);
+                }
+            }
+            try expr(w, scope, parent_decl, full.ast.then_expr);
+            try maybe_expr(w, scope, parent_decl, full.ast.else_expr);
+        },
+
+        .slice => return slice(w, scope, parent_decl, ast.slice(node)),
+        .slice_open => return slice(w, scope, parent_decl, ast.sliceOpen(node)),
+        .slice_sentinel => return slice(w, scope, parent_decl, ast.sliceSentinel(node)),
+
+        .block_two, .block_two_semicolon => {
+            const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
+            if (node_datas[node].lhs == 0) {
+                return block(w, scope, parent_decl, statements[0..0]);
+            } else if (node_datas[node].rhs == 0) {
+                return block(w, scope, parent_decl, statements[0..1]);
+            } else {
+                return block(w, scope, parent_decl, statements[0..2]);
+            }
+        },
+        .block, .block_semicolon => {
+            const statements = ast.extra_data[node_datas[node].lhs..node_datas[node].rhs];
+            return block(w, scope, parent_decl, statements);
+        },
+
+        .ptr_type_aligned,
+        .ptr_type_sentinel,
+        .ptr_type,
+        .ptr_type_bit_range,
+        => {
+            const full = ast.fullPtrType(node).?;
+            try maybe_expr(w, scope, parent_decl, full.ast.align_node);
+            try maybe_expr(w, scope, parent_decl, full.ast.addrspace_node);
+            try maybe_expr(w, scope, parent_decl, full.ast.sentinel);
+            try maybe_expr(w, scope, parent_decl, full.ast.bit_range_start);
+            try maybe_expr(w, scope, parent_decl, full.ast.bit_range_end);
+            try expr(w, scope, parent_decl, full.ast.child_type);
+        },
+
+        .container_decl,
+        .container_decl_trailing,
+        .container_decl_arg,
+        .container_decl_arg_trailing,
+        .container_decl_two,
+        .container_decl_two_trailing,
+        .tagged_union,
+        .tagged_union_trailing,
+        .tagged_union_enum_tag,
+        .tagged_union_enum_tag_trailing,
+        .tagged_union_two,
+        .tagged_union_two_trailing,
+        => {
+            var buf: [2]Ast.Node.Index = undefined;
+            return struct_decl(w, scope, parent_decl, ast.fullContainerDecl(&buf, node).?);
+        },
+
+        .array_type_sentinel => {
+            const extra = ast.extraData(node_datas[node].rhs, Ast.Node.ArrayTypeSentinel);
+            try expr(w, scope, parent_decl, node_datas[node].lhs);
+            try expr(w, scope, parent_decl, extra.elem_type);
+            try expr(w, scope, parent_decl, extra.sentinel);
+        },
+        .@"switch", .switch_comma => {
+            const operand_node = node_datas[node].lhs;
+            try expr(w, scope, parent_decl, operand_node);
+            const extra = ast.extraData(node_datas[node].rhs, Ast.Node.SubRange);
+            const case_nodes = ast.extra_data[extra.start..extra.end];
+            for (case_nodes) |case_node| {
+                const case = ast.fullSwitchCase(case_node).?;
+                for (case.ast.values) |value_node| {
+                    try expr(w, scope, parent_decl, value_node);
+                }
+                try expr(w, scope, parent_decl, case.ast.target_expr);
+            }
+        },
+
+        .array_init_one,
+        .array_init_one_comma,
+        .array_init_dot_two,
+        .array_init_dot_two_comma,
+        .array_init_dot,
+        .array_init_dot_comma,
+        .array_init,
+        .array_init_comma,
+        => {
+            var buf: [2]Ast.Node.Index = undefined;
+            const full = ast.fullArrayInit(&buf, node).?;
+            try maybe_expr(w, scope, parent_decl, full.ast.type_expr);
+            for (full.ast.elements) |elem| {
+                try expr(w, scope, parent_decl, elem);
+            }
+        },
+
+        .struct_init_one,
+        .struct_init_one_comma,
+        .struct_init_dot_two,
+        .struct_init_dot_two_comma,
+        .struct_init_dot,
+        .struct_init_dot_comma,
+        .struct_init,
+        .struct_init_comma,
+        => {
+            var buf: [2]Ast.Node.Index = undefined;
+            const full = ast.fullStructInit(&buf, node).?;
+            try maybe_expr(w, scope, parent_decl, full.ast.type_expr);
+            for (full.ast.fields) |field| {
+                try expr(w, scope, parent_decl, field);
+            }
+        },
+
+        .fn_proto_simple,
+        .fn_proto_multi,
+        .fn_proto_one,
+        .fn_proto,
+        => {
+            var buf: [1]Ast.Node.Index = undefined;
+            return fn_decl(w, scope, parent_decl, 0, ast.fullFnProto(&buf, node).?);
+        },
+    }
+}
+
+fn slice(w: *Walk, scope: *Scope, parent_decl: Decl.Index, full: Ast.full.Slice) Oom!void {
+    try expr(w, scope, parent_decl, full.ast.sliced);
+    try expr(w, scope, parent_decl, full.ast.start);
+    try maybe_expr(w, scope, parent_decl, full.ast.end);
+    try maybe_expr(w, scope, parent_decl, full.ast.sentinel);
+}
+
+fn builtin_call(
+    w: *Walk,
+    scope: *Scope,
+    parent_decl: Decl.Index,
+    node: Ast.Node.Index,
+    params: []const Ast.Node.Index,
+) Oom!void {
+    const ast = w.file.get_ast();
+    const main_tokens = ast.nodes.items(.main_token);
+    const builtin_token = main_tokens[node];
+    const builtin_name = ast.tokenSlice(builtin_token);
+    if (std.mem.eql(u8, builtin_name, "@This")) {
+        try w.file.get().node_decls.put(gpa, node, scope.getNamespaceDecl());
+    }
+
+    for (params) |param| {
+        try expr(w, scope, parent_decl, param);
+    }
+}
+
+fn block(
+    w: *Walk,
+    parent_scope: *Scope,
+    parent_decl: Decl.Index,
+    statements: []const Ast.Node.Index,
+) Oom!void {
+    const ast = w.file.get_ast();
+    const arena = w.arena;
+    const node_tags = ast.nodes.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+
+    var scope = parent_scope;
+
+    for (statements) |node| {
+        switch (node_tags[node]) {
+            .global_var_decl,
+            .local_var_decl,
+            .simple_var_decl,
+            .aligned_var_decl,
+            => {
+                const full = ast.fullVarDecl(node).?;
+                try global_var_decl(w, scope, parent_decl, full);
+                const local = try arena.create(Scope.Local);
+                local.* = .{
+                    .parent = scope,
+                    .var_node = node,
+                };
+                scope = &local.base;
+            },
+
+            .assign_destructure => {
+                // TODO
+            },
+
+            .grouped_expression => try expr(w, scope, parent_decl, node_datas[node].lhs),
+
+            .@"defer",
+            .@"errdefer",
+            => try expr(w, scope, parent_decl, node_datas[node].rhs),
+
+            else => try expr(w, scope, parent_decl, node),
+        }
+    }
+}
+
+fn while_expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, full: Ast.full.While) Oom!void {
+    try expr(w, scope, parent_decl, full.ast.cond_expr);
+    try maybe_expr(w, scope, parent_decl, full.ast.cont_expr);
+    try expr(w, scope, parent_decl, full.ast.then_expr);
+    try maybe_expr(w, scope, parent_decl, full.ast.else_expr);
+}
+
+fn scanDecls(w: *Walk, namespace: *Scope.Namespace, members: []const Ast.Node.Index) Oom!void {
+    const arena = w.arena;
+    const ast = w.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const main_tokens = ast.nodes.items(.main_token);
+    const token_tags = ast.tokens.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+
+    for (members) |member_node| {
+        const name_token = switch (node_tags[member_node]) {
+            .global_var_decl,
+            .local_var_decl,
+            .simple_var_decl,
+            .aligned_var_decl,
+            => main_tokens[member_node] + 1,
+
+            .fn_proto_simple,
+            .fn_proto_multi,
+            .fn_proto_one,
+            .fn_proto,
+            .fn_decl,
+            => blk: {
+                const ident = main_tokens[member_node] + 1;
+                if (token_tags[ident] != .identifier) continue;
+                break :blk ident;
+            },
+
+            .test_decl => {
+                const ident_token = node_datas[member_node].lhs;
+                const is_doctest = token_tags[ident_token] == .identifier;
+                if (is_doctest) {
+                    const token_bytes = ast.tokenSlice(ident_token);
+                    try namespace.doctests.put(arena, token_bytes, member_node);
+                }
+                continue;
+            },
+
+            else => continue,
+        };
+
+        const token_bytes = ast.tokenSlice(name_token);
+        try namespace.names.put(arena, token_bytes, member_node);
+    }
+}
+
+//test {
+//    const gpa = std.testing.allocator;
+//
+//    var arena_instance = std.heap.ArenaAllocator.init(gpa);
+//    defer arena_instance.deinit();
+//    const arena = arena_instance.allocator();
+//
+//    // example test command:
+//    // zig test --dep input.zig -Mroot=src/Walk.zig -Minput.zig=/home/andy/dev/zig/lib/std/fs/File/zig
+//    var ast = try Ast.parse(gpa, @embedFile("input.zig"), .zig);
+//    defer ast.deinit(gpa);
+//
+//    var w: Walk = .{
+//        .arena = arena,
+//        .token_links = .{},
+//        .ast = &ast,
+//    };
+//
+//    try w.root();
+//}
+
+const Walk = @This();
+const std = @import("std");
+const Ast = std.zig.Ast;
+const assert = std.debug.assert;
+const Decl = @import("Decl.zig");
+const log = std.log;
+const gpa = std.heap.wasm_allocator;
+const Oom = error{OutOfMemory};
+
+fn shrinkToFit(m: anytype) void {
+    m.shrinkAndFree(gpa, m.entries.len);
+}

lib/docs/wasm/main.zig

@@ -0,0 +1,920 @@
+const gpa = std.heap.wasm_allocator;
+
+const std = @import("std");
+const log = std.log;
+const assert = std.debug.assert;
+const Ast = std.zig.Ast;
+const Walk = @import("Walk.zig");
+const markdown = @import("markdown.zig");
+const Decl = @import("Decl.zig");
+
+const js = struct {
+    extern "js" fn log(ptr: [*]const u8, len: usize) void;
+    extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;
+};
+
+pub const std_options: std.Options = .{
+    .logFn = logFn,
+    //.log_level = .debug,
+};
+
+pub fn panic(msg: []const u8, st: ?*std.builtin.StackTrace, addr: ?usize) noreturn {
+    _ = st;
+    _ = addr;
+    log.err("panic: {s}", .{msg});
+    @trap();
+}
+
+fn logFn(
+    comptime message_level: log.Level,
+    comptime scope: @TypeOf(.enum_literal),
+    comptime format: []const u8,
+    args: anytype,
+) void {
+    const level_txt = comptime message_level.asText();
+    const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
+    var buf: [500]u8 = undefined;
+    const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: {
+        buf[buf.len - 3 ..][0..3].* = "...".*;
+        break :l &buf;
+    };
+    js.log(line.ptr, line.len);
+}
+
+export fn alloc(n: usize) [*]u8 {
+    const slice = gpa.alloc(u8, n) catch @panic("OOM");
+    return slice.ptr;
+}
+
+export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
+    const tar_bytes = tar_ptr[0..tar_len];
+    //log.debug("received {d} bytes of tar file", .{tar_bytes.len});
+
+    unpack_inner(tar_bytes) catch |err| {
+        fatal("unable to unpack tar: {s}", .{@errorName(err)});
+    };
+}
+
+var query_string: std.ArrayListUnmanaged(u8) = .{};
+var query_results: std.ArrayListUnmanaged(Decl.Index) = .{};
+
+/// Resizes the query string to be the correct length; returns the pointer to
+/// the query string.
+export fn query_begin(query_string_len: usize) [*]u8 {
+    query_string.resize(gpa, query_string_len) catch @panic("OOM");
+    return query_string.items.ptr;
+}
+
+/// Executes the query. Returns the pointer to the query results which is an
+/// array of u32.
+/// The first element is the length of the array.
+/// Subsequent elements are Decl.Index values which are all public
+/// declarations.
+export fn query_exec(ignore_case: bool) [*]Decl.Index {
+    const query = query_string.items;
+    log.debug("querying '{s}'", .{query});
+    query_exec_fallible(query, ignore_case) catch |err| switch (err) {
+        error.OutOfMemory => @panic("OOM"),
+    };
+    query_results.items[0] = @enumFromInt(query_results.items.len - 1);
+    return query_results.items.ptr;
+}
+
+const max_matched_items = 2000;
+
+fn query_exec_fallible(query: []const u8, ignore_case: bool) !void {
+    const Score = packed struct(u32) {
+        points: u16,
+        segments: u16,
+    };
+    const g = struct {
+        var full_path_search_text: std.ArrayListUnmanaged(u8) = .{};
+        var full_path_search_text_lower: std.ArrayListUnmanaged(u8) = .{};
+        var doc_search_text: std.ArrayListUnmanaged(u8) = .{};
+        /// Each element matches a corresponding query_results element.
+        var scores: std.ArrayListUnmanaged(Score) = .{};
+    };
+
+    // First element stores the size of the list.
+    try query_results.resize(gpa, 1);
+    // Corresponding point value is meaningless and therefore undefined.
+    try g.scores.resize(gpa, 1);
+
+    decl_loop: for (Walk.decls.items, 0..) |*decl, decl_index| {
+        const info = decl.extra_info();
+        if (!info.is_pub) continue;
+
+        try decl.reset_with_path(&g.full_path_search_text);
+        if (decl.parent != .none)
+            try Decl.append_parent_ns(&g.full_path_search_text, decl.parent);
+        try g.full_path_search_text.appendSlice(gpa, info.name);
+
+        try g.full_path_search_text_lower.resize(gpa, g.full_path_search_text.items.len);
+        @memcpy(g.full_path_search_text_lower.items, g.full_path_search_text.items);
+
+        const ast = decl.file.get_ast();
+        try collect_docs(&g.doc_search_text, ast, info.first_doc_comment);
+
+        if (ignore_case) {
+            ascii_lower(g.full_path_search_text_lower.items);
+            ascii_lower(g.doc_search_text.items);
+        }
+
+        var it = std.mem.tokenizeScalar(u8, query, ' ');
+        var points: u16 = 0;
+        var bypass_limit = false;
+        while (it.next()) |term| {
+            // exact, case sensitive match of full decl path
+            if (std.mem.eql(u8, g.full_path_search_text.items, term)) {
+                points += 4;
+                bypass_limit = true;
+                continue;
+            }
+            // exact, case sensitive match of just decl name
+            if (std.mem.eql(u8, info.name, term)) {
+                points += 3;
+                bypass_limit = true;
+                continue;
+            }
+            // substring, case insensitive match of full decl path
+            if (std.mem.indexOf(u8, g.full_path_search_text_lower.items, term) != null) {
+                points += 2;
+                continue;
+            }
+            if (std.mem.indexOf(u8, g.doc_search_text.items, term) != null) {
+                points += 1;
+                continue;
+            }
+            continue :decl_loop;
+        }
+
+        if (query_results.items.len < max_matched_items or bypass_limit) {
+            try query_results.append(gpa, @enumFromInt(decl_index));
+            try g.scores.append(gpa, .{
+                .points = points,
+                .segments = @intCast(count_scalar(g.full_path_search_text.items, '.')),
+            });
+        }
+    }
+
+    const sort_context: struct {
+        pub fn swap(sc: @This(), a_index: usize, b_index: usize) void {
+            _ = sc;
+            std.mem.swap(Score, &g.scores.items[a_index], &g.scores.items[b_index]);
+            std.mem.swap(Decl.Index, &query_results.items[a_index], &query_results.items[b_index]);
+        }
+
+        pub fn lessThan(sc: @This(), a_index: usize, b_index: usize) bool {
+            _ = sc;
+            const a_score = g.scores.items[a_index];
+            const b_score = g.scores.items[b_index];
+            if (b_score.points < a_score.points) {
+                return true;
+            } else if (b_score.points > a_score.points) {
+                return false;
+            } else if (a_score.segments < b_score.segments) {
+                return true;
+            } else if (a_score.segments > b_score.segments) {
+                return false;
+            } else {
+                const a_decl = query_results.items[a_index];
+                const b_decl = query_results.items[b_index];
+                const a_file_path = a_decl.get().file.path();
+                const b_file_path = b_decl.get().file.path();
+                // TODO Also check the local namespace  inside the file
+                return std.mem.lessThan(u8, b_file_path, a_file_path);
+            }
+        }
+    } = .{};
+
+    std.mem.sortUnstableContext(1, query_results.items.len, sort_context);
+
+    if (query_results.items.len > max_matched_items)
+        query_results.shrinkRetainingCapacity(max_matched_items);
+}
+
+const String = Slice(u8);
+
+fn Slice(T: type) type {
+    return packed struct(u64) {
+        ptr: u32,
+        len: u32,
+
+        fn init(s: []const T) @This() {
+            return .{
+                .ptr = @intFromPtr(s.ptr),
+                .len = s.len,
+            };
+        }
+    };
+}
+
+var string_result: std.ArrayListUnmanaged(u8) = .{};
+
+export fn decl_fields(decl_index: Decl.Index) Slice(Ast.Node.Index) {
+    return Slice(Ast.Node.Index).init(decl_fields_fallible(decl_index) catch @panic("OOM"));
+}
+
+fn decl_fields_fallible(decl_index: Decl.Index) ![]Ast.Node.Index {
+    const g = struct {
+        var result: std.ArrayListUnmanaged(Ast.Node.Index) = .{};
+    };
+    g.result.clearRetainingCapacity();
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const value_node = decl.value_node() orelse return &.{};
+    var buf: [2]Ast.Node.Index = undefined;
+    const container_decl = ast.fullContainerDecl(&buf, value_node) orelse return &.{};
+    for (container_decl.ast.members) |member_node| switch (node_tags[member_node]) {
+        .container_field_init,
+        .container_field_align,
+        .container_field,
+        => try g.result.append(gpa, member_node),
+
+        else => continue,
+    };
+    return g.result.items;
+}
+
+export fn decl_field_html(decl_index: Decl.Index, field_node: Ast.Node.Index) String {
+    string_result.clearRetainingCapacity();
+    decl_field_html_fallible(&string_result, decl_index, field_node) catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+fn decl_field_html_fallible(
+    out: *std.ArrayListUnmanaged(u8),
+    decl_index: Decl.Index,
+    field_node: Ast.Node.Index,
+) !void {
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    try out.appendSlice(gpa, "<pre><code>");
+    try file_source_html(decl.file, out, field_node);
+    try out.appendSlice(gpa, "</code></pre>");
+
+    const field = ast.fullContainerField(field_node).?;
+    const first_doc_comment = Decl.findFirstDocComment(ast, field.firstToken());
+
+    if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
+        try out.appendSlice(gpa, "<div class=\"fieldDocs\">");
+        try render_docs(out, ast, first_doc_comment, false);
+        try out.appendSlice(gpa, "</div>");
+    }
+}
+
+export fn decl_fn_proto_html(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    const node_datas = ast.nodes.items(.data);
+    const proto_node = switch (node_tags[decl.ast_node]) {
+        .fn_decl => node_datas[decl.ast_node].lhs,
+
+        .fn_proto,
+        .fn_proto_one,
+        .fn_proto_simple,
+        .fn_proto_multi,
+        => decl.ast_node,
+
+        else => unreachable,
+    };
+
+    string_result.clearRetainingCapacity();
+    file_source_html(decl.file, &string_result, proto_node) catch |err| {
+        fatal("unable to render source: {s}", .{@errorName(err)});
+    };
+    return String.init(string_result.items);
+}
+
+export fn decl_source_html(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+
+    string_result.clearRetainingCapacity();
+    file_source_html(decl.file, &string_result, decl.ast_node) catch |err| {
+        fatal("unable to render source: {s}", .{@errorName(err)});
+    };
+    return String.init(string_result.items);
+}
+
+export fn decl_doctest_html(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    const doctest_ast_node = decl.file.get().doctests.get(decl.ast_node) orelse
+        return String.init("");
+
+    string_result.clearRetainingCapacity();
+    file_source_html(decl.file, &string_result, doctest_ast_node) catch |err| {
+        fatal("unable to render source: {s}", .{@errorName(err)});
+    };
+    return String.init(string_result.items);
+}
+
+export fn decl_fqn(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    decl.fqn(&string_result) catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+export fn decl_parent(decl_index: Decl.Index) Decl.Index {
+    const decl = decl_index.get();
+    return decl.parent;
+}
+
+export fn decl_file_path(decl_index: Decl.Index) String {
+    string_result.clearRetainingCapacity();
+    string_result.appendSlice(gpa, decl_index.get().file.path()) catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+export fn decl_category_name(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    const token_tags = ast.tokens.items(.tag);
+    const name = switch (decl.categorize()) {
+        .namespace => |node| {
+            const node_tags = ast.nodes.items(.tag);
+            if (node_tags[decl.ast_node] == .root)
+                return String.init("struct");
+            string_result.clearRetainingCapacity();
+            var buf: [2]Ast.Node.Index = undefined;
+            const container_decl = ast.fullContainerDecl(&buf, node).?;
+            if (container_decl.layout_token) |t| {
+                if (token_tags[t] == .keyword_extern) {
+                    string_result.appendSlice(gpa, "extern ") catch @panic("OOM");
+                }
+            }
+            const main_token_tag = token_tags[container_decl.ast.main_token];
+            string_result.appendSlice(gpa, main_token_tag.lexeme().?) catch @panic("OOM");
+            return String.init(string_result.items);
+        },
+        .global_variable => "Global Variable",
+        .function => "Function",
+        .type => "Type",
+        .error_set => "Error Set",
+        .global_const => "Constant",
+        .primitive => "Primitive Value",
+        .alias => "Alias",
+    };
+    return String.init(name);
+}
+
+export fn decl_name(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    string_result.clearRetainingCapacity();
+    const name = n: {
+        if (decl.parent == .none) {
+            // Then it is the root struct of a file.
+            break :n std.fs.path.stem(decl.file.path());
+        }
+        break :n decl.extra_info().name;
+    };
+    string_result.appendSlice(gpa, name) catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+export fn decl_docs_html(decl_index: Decl.Index, short: bool) String {
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    string_result.clearRetainingCapacity();
+    render_docs(&string_result, ast, decl.extra_info().first_doc_comment, short) catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+fn collect_docs(
+    list: *std.ArrayListUnmanaged(u8),
+    ast: *const Ast,
+    first_doc_comment: Ast.TokenIndex,
+) Oom!void {
+    const token_tags = ast.tokens.items(.tag);
+    list.clearRetainingCapacity();
+    var it = first_doc_comment;
+    while (true) : (it += 1) switch (token_tags[it]) {
+        .doc_comment, .container_doc_comment => {
+            // It is tempting to trim this string but think carefully about how
+            // that will affect the markdown parser.
+            const line = ast.tokenSlice(it)[3..];
+            try list.appendSlice(gpa, line);
+        },
+        else => break,
+    };
+}
+
+fn render_docs(
+    out: *std.ArrayListUnmanaged(u8),
+    ast: *const Ast,
+    first_doc_comment: Ast.TokenIndex,
+    short: bool,
+) Oom!void {
+    const token_tags = ast.tokens.items(.tag);
+
+    var parser = try markdown.Parser.init(gpa);
+    defer parser.deinit();
+    var it = first_doc_comment;
+    while (true) : (it += 1) switch (token_tags[it]) {
+        .doc_comment, .container_doc_comment => {
+            const line = ast.tokenSlice(it)[3..];
+            if (short and line.len == 0) break;
+            try parser.feedLine(line);
+        },
+        else => break,
+    };
+
+    var parsed_doc = try parser.endInput();
+    defer parsed_doc.deinit(gpa);
+
+    const Writer = std.ArrayListUnmanaged(u8).Writer;
+    const Renderer = markdown.Renderer(Writer, void);
+    const renderer: Renderer = .{
+        .context = {},
+        .renderFn = struct {
+            fn render(
+                r: Renderer,
+                doc: markdown.Document,
+                node: markdown.Document.Node.Index,
+                writer: Writer,
+            ) !void {
+                const data = doc.nodes.items(.data)[@intFromEnum(node)];
+                switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
+                    // TODO: detect identifier references (dotted paths) in
+                    // these three node types and render them appropriately.
+                    // Also, syntax highlighting can be applied in code blocks
+                    // unless the tag says otherwise.
+                    .code_block => {
+                        const tag = doc.string(data.code_block.tag);
+                        _ = tag;
+                        const content = doc.string(data.code_block.content);
+                        try writer.print("<pre><code>{}</code></pre>\n", .{markdown.fmtHtml(content)});
+                    },
+                    .code_span => {
+                        const content = doc.string(data.text.content);
+                        try writer.print("<code>{}</code>", .{markdown.fmtHtml(content)});
+                    },
+                    .text => {
+                        const content = doc.string(data.text.content);
+                        try writer.print("{}", .{markdown.fmtHtml(content)});
+                    },
+
+                    else => try Renderer.renderDefault(r, doc, node, writer),
+                }
+            }
+        }.render,
+    };
+    try renderer.render(parsed_doc, out.writer(gpa));
+}
+
+export fn decl_type_html(decl_index: Decl.Index) String {
+    const decl = decl_index.get();
+    const ast = decl.file.get_ast();
+    string_result.clearRetainingCapacity();
+    _ = ast; // TODO
+    string_result.appendSlice(gpa, "TODO_type_here") catch @panic("OOM");
+    return String.init(string_result.items);
+}
+
+const Oom = error{OutOfMemory};
+
+fn unpack_inner(tar_bytes: []u8) !void {
+    var fbs = std.io.fixedBufferStream(tar_bytes);
+    var file_name_buffer: [1024]u8 = undefined;
+    var link_name_buffer: [1024]u8 = undefined;
+    var it = std.tar.iterator(fbs.reader(), .{
+        .file_name_buffer = &file_name_buffer,
+        .link_name_buffer = &link_name_buffer,
+    });
+    while (try it.next()) |tar_file| {
+        switch (tar_file.kind) {
+            .normal => {
+                if (tar_file.size == 0 and tar_file.name.len == 0) break;
+                if (std.mem.endsWith(u8, tar_file.name, ".zig")) {
+                    log.debug("found file: '{s}'", .{tar_file.name});
+                    const file_name = try gpa.dupe(u8, tar_file.name);
+                    if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| {
+                        const pkg_name = file_name[0..pkg_name_end];
+                        const gop = try Walk.packages.getOrPut(gpa, pkg_name);
+                        const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len);
+                        if (!gop.found_existing or
+                            std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or
+                            std.mem.eql(u8, file_name[pkg_name_end + 1 .. file_name.len - ".zig".len], pkg_name))
+                        {
+                            gop.value_ptr.* = file;
+                        }
+                        const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)];
+                        assert(file == try Walk.add_file(file_name, file_bytes));
+                    }
+                } else {
+                    log.warn("skipping: '{s}' - the tar creation should have done that", .{
+                        tar_file.name,
+                    });
+                }
+                try tar_file.skip();
+            },
+            else => continue,
+        }
+    }
+}
+
+fn fatal(comptime format: []const u8, args: anytype) noreturn {
+    var buf: [500]u8 = undefined;
+    const line = std.fmt.bufPrint(&buf, format, args) catch l: {
+        buf[buf.len - 3 ..][0..3].* = "...".*;
+        break :l &buf;
+    };
+    js.panic(line.ptr, line.len);
+}
+
+fn ascii_lower(bytes: []u8) void {
+    for (bytes) |*b| b.* = std.ascii.toLower(b.*);
+}
+
+export fn package_name(index: u32) String {
+    const names = Walk.packages.keys();
+    return String.init(if (index >= names.len) "" else names[index]);
+}
+
+export fn find_package_root(pkg: Walk.PackageIndex) Decl.Index {
+    const root_file = Walk.packages.values()[@intFromEnum(pkg)];
+    const result = root_file.findRootDecl();
+    assert(result != .none);
+    return result;
+}
+
+/// Set by `set_input_string`.
+var input_string: std.ArrayListUnmanaged(u8) = .{};
+
+export fn set_input_string(len: usize) [*]u8 {
+    input_string.resize(gpa, len) catch @panic("OOM");
+    return input_string.items.ptr;
+}
+
+/// Looks up the root struct decl corresponding to a file by path.
+/// Uses `input_string`.
+export fn find_file_root() Decl.Index {
+    const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none);
+    return file.findRootDecl();
+}
+
+/// Uses `input_string`.
+export fn find_decl() Decl.Index {
+    const g = struct {
+        var match_fqn: std.ArrayListUnmanaged(u8) = .{};
+    };
+    for (Walk.decls.items, 0..) |*decl, decl_index| {
+        decl.fqn(&g.match_fqn) catch @panic("OOM");
+        if (std.mem.eql(u8, g.match_fqn.items, input_string.items)) {
+            //const path = @as(Decl.Index, @enumFromInt(decl_index)).get().file.path();
+            //log.debug("find_decl '{s}' found in {s}", .{ input_string.items, path });
+            return @enumFromInt(decl_index);
+        }
+    }
+    return .none;
+}
+
+/// Set only by `categorize_decl`; read only by `get_aliasee`, valid only
+/// when `categorize_decl` returns `.alias`.
+var global_aliasee: Decl.Index = .none;
+
+export fn get_aliasee() Decl.Index {
+    return global_aliasee;
+}
+export fn categorize_decl(decl_index: Decl.Index, resolve_alias_count: usize) Walk.Category.Tag {
+    global_aliasee = .none;
+    var chase_alias_n = resolve_alias_count;
+    var decl = decl_index.get();
+    while (true) {
+        const result = decl.categorize();
+        switch (decl.categorize()) {
+            .alias => |new_index| {
+                assert(new_index != .none);
+                global_aliasee = new_index;
+                if (chase_alias_n > 0) {
+                    chase_alias_n -= 1;
+                    decl = new_index.get();
+                    continue;
+                }
+            },
+            else => {},
+        }
+        return result;
+    }
+}
+
+export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Decl.Index) {
+    const g = struct {
+        var members: std.ArrayListUnmanaged(Decl.Index) = .{};
+    };
+
+    g.members.clearRetainingCapacity();
+
+    for (Walk.decls.items, 0..) |*decl, i| {
+        if (decl.parent == parent) {
+            if (include_private or decl.is_pub()) {
+                g.members.append(gpa, @enumFromInt(i)) catch @panic("OOM");
+            }
+        }
+    }
+
+    return Slice(Decl.Index).init(g.members.items);
+}
+
+fn file_source_html(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    root_node: Ast.Node.Index,
+) !void {
+    const ast = file_index.get_ast();
+    const file = file_index.get();
+
+    const g = struct {
+        var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
+    };
+
+    const token_tags = ast.tokens.items(.tag);
+    const token_starts = ast.tokens.items(.start);
+
+    const start_token = ast.firstToken(root_node);
+    const end_token = ast.lastToken(root_node) + 1;
+
+    var cursor: usize = token_starts[start_token];
+
+    for (
+        token_tags[start_token..end_token],
+        token_starts[start_token..end_token],
+        start_token..,
+    ) |tag, start, token_index| {
+        const between = ast.source[cursor..start];
+        try appendEscaped(out, between);
+        if (tag == .eof) break;
+        const slice = ast.tokenSlice(token_index);
+        cursor = start + slice.len;
+        switch (tag) {
+            .eof => unreachable,
+
+            .keyword_addrspace,
+            .keyword_align,
+            .keyword_and,
+            .keyword_asm,
+            .keyword_async,
+            .keyword_await,
+            .keyword_break,
+            .keyword_catch,
+            .keyword_comptime,
+            .keyword_const,
+            .keyword_continue,
+            .keyword_defer,
+            .keyword_else,
+            .keyword_enum,
+            .keyword_errdefer,
+            .keyword_error,
+            .keyword_export,
+            .keyword_extern,
+            .keyword_for,
+            .keyword_if,
+            .keyword_inline,
+            .keyword_noalias,
+            .keyword_noinline,
+            .keyword_nosuspend,
+            .keyword_opaque,
+            .keyword_or,
+            .keyword_orelse,
+            .keyword_packed,
+            .keyword_anyframe,
+            .keyword_pub,
+            .keyword_resume,
+            .keyword_return,
+            .keyword_linksection,
+            .keyword_callconv,
+            .keyword_struct,
+            .keyword_suspend,
+            .keyword_switch,
+            .keyword_test,
+            .keyword_threadlocal,
+            .keyword_try,
+            .keyword_union,
+            .keyword_unreachable,
+            .keyword_usingnamespace,
+            .keyword_var,
+            .keyword_volatile,
+            .keyword_allowzero,
+            .keyword_while,
+            .keyword_anytype,
+            .keyword_fn,
+            => {
+                try out.appendSlice(gpa, "<span class=\"tok-kw\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .string_literal,
+            .char_literal,
+            .multiline_string_literal_line,
+            => {
+                try out.appendSlice(gpa, "<span class=\"tok-str\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .builtin => {
+                try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .doc_comment,
+            .container_doc_comment,
+            => {
+                try out.appendSlice(gpa, "<span class=\"tok-comment\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .identifier => i: {
+                if (std.mem.eql(u8, slice, "undefined") or
+                    std.mem.eql(u8, slice, "null") or
+                    std.mem.eql(u8, slice, "true") or
+                    std.mem.eql(u8, slice, "false"))
+                {
+                    try out.appendSlice(gpa, "<span class=\"tok-null\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                    break :i;
+                }
+
+                if (std.zig.primitives.isPrimitive(slice)) {
+                    try out.appendSlice(gpa, "<span class=\"tok-type\">");
+                    try appendEscaped(out, slice);
+                    try out.appendSlice(gpa, "</span>");
+                    break :i;
+                }
+
+                if (file.token_parents.get(token_index)) |field_access_node| {
+                    g.field_access_buffer.clearRetainingCapacity();
+                    try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
+                    if (g.field_access_buffer.items.len > 0) {
+                        try out.appendSlice(gpa, "<a href=\"#");
+                        try out.appendSlice(gpa, g.field_access_buffer.items); // TODO url escape
+                        try out.appendSlice(gpa, "\">");
+                        try appendEscaped(out, slice);
+                        try out.appendSlice(gpa, "</a>");
+                    } else {
+                        try appendEscaped(out, slice);
+                    }
+                    break :i;
+                }
+
+                {
+                    g.field_access_buffer.clearRetainingCapacity();
+                    try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
+                    if (g.field_access_buffer.items.len > 0) {
+                        try out.appendSlice(gpa, "<a href=\"#");
+                        try out.appendSlice(gpa, g.field_access_buffer.items); // TODO url escape
+                        try out.appendSlice(gpa, "\">");
+                        try appendEscaped(out, slice);
+                        try out.appendSlice(gpa, "</a>");
+                        break :i;
+                    }
+                }
+
+                try appendEscaped(out, slice);
+            },
+
+            .number_literal => {
+                try out.appendSlice(gpa, "<span class=\"tok-number\">");
+                try appendEscaped(out, slice);
+                try out.appendSlice(gpa, "</span>");
+            },
+
+            .bang,
+            .pipe,
+            .pipe_pipe,
+            .pipe_equal,
+            .equal,
+            .equal_equal,
+            .equal_angle_bracket_right,
+            .bang_equal,
+            .l_paren,
+            .r_paren,
+            .semicolon,
+            .percent,
+            .percent_equal,
+            .l_brace,
+            .r_brace,
+            .l_bracket,
+            .r_bracket,
+            .period,
+            .period_asterisk,
+            .ellipsis2,
+            .ellipsis3,
+            .caret,
+            .caret_equal,
+            .plus,
+            .plus_plus,
+            .plus_equal,
+            .plus_percent,
+            .plus_percent_equal,
+            .plus_pipe,
+            .plus_pipe_equal,
+            .minus,
+            .minus_equal,
+            .minus_percent,
+            .minus_percent_equal,
+            .minus_pipe,
+            .minus_pipe_equal,
+            .asterisk,
+            .asterisk_equal,
+            .asterisk_asterisk,
+            .asterisk_percent,
+            .asterisk_percent_equal,
+            .asterisk_pipe,
+            .asterisk_pipe_equal,
+            .arrow,
+            .colon,
+            .slash,
+            .slash_equal,
+            .comma,
+            .ampersand,
+            .ampersand_equal,
+            .question_mark,
+            .angle_bracket_left,
+            .angle_bracket_left_equal,
+            .angle_bracket_angle_bracket_left,
+            .angle_bracket_angle_bracket_left_equal,
+            .angle_bracket_angle_bracket_left_pipe,
+            .angle_bracket_angle_bracket_left_pipe_equal,
+            .angle_bracket_right,
+            .angle_bracket_right_equal,
+            .angle_bracket_angle_bracket_right,
+            .angle_bracket_angle_bracket_right_equal,
+            .tilde,
+            => try appendEscaped(out, slice),
+
+            .invalid, .invalid_periodasterisks => return error.InvalidToken,
+        }
+    }
+}
+
+fn resolve_ident_link(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    ident_token: Ast.TokenIndex,
+) Oom!void {
+    const decl_index = file_index.get().lookup_token(ident_token);
+    if (decl_index == .none) return;
+
+    const decl = decl_index.get();
+    switch (decl.categorize()) {
+        .alias => |alias_decl| try alias_decl.get().fqn(out),
+        else => try decl.fqn(out),
+    }
+}
+
+fn walk_field_accesses(
+    file_index: Walk.File.Index,
+    out: *std.ArrayListUnmanaged(u8),
+    node: Ast.Node.Index,
+) Oom!void {
+    const ast = file_index.get_ast();
+    const node_tags = ast.nodes.items(.tag);
+    assert(node_tags[node] == .field_access);
+    const node_datas = ast.nodes.items(.data);
+    const main_tokens = ast.nodes.items(.main_token);
+    const object_node = node_datas[node].lhs;
+    const dot_token = main_tokens[node];
+    const field_ident = dot_token + 1;
+    switch (node_tags[object_node]) {
+        .identifier => {
+            const lhs_ident = main_tokens[object_node];
+            try resolve_ident_link(file_index, out, lhs_ident);
+        },
+        .field_access => {
+            try walk_field_accesses(file_index, out, object_node);
+        },
+        else => {},
+    }
+    if (out.items.len > 0) {
+        try out.append(gpa, '.');
+        try out.appendSlice(gpa, ast.tokenSlice(field_ident));
+    }
+}
+
+fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
+    for (s) |c| {
+        try out.ensureUnusedCapacity(gpa, 6);
+        switch (c) {
+            '&' => out.appendSliceAssumeCapacity("&amp;"),
+            '<' => out.appendSliceAssumeCapacity("&lt;"),
+            '>' => out.appendSliceAssumeCapacity("&gt;"),
+            '"' => out.appendSliceAssumeCapacity("&quot;"),
+            else => out.appendAssumeCapacity(c),
+        }
+    }
+}
+
+fn count_scalar(haystack: []const u8, needle: u8) usize {
+    var total: usize = 0;
+    for (haystack) |elem| {
+        if (elem == needle)
+            total += 1;
+    }
+    return total;
+}

lib/docs/wasm/markdown.zig

@@ -0,0 +1,933 @@
+//! Markdown parsing and rendering support.
+//!
+//! A Markdown document consists of a series of blocks. Depending on its type,
+//! each block may contain other blocks, inline content, or nothing. The
+//! supported blocks are as follows:
+//!
+//! - **List** - a sequence of list items of the same type.
+//!
+//! - **List item** - unordered list items start with `-`, `*`, or `+` followed
+//!   by a space. Ordered list items start with a number between 0 and
+//!   999,999,999, followed by a `.` or `)` and a space. The number of an
+//!   ordered list item only matters for the first item in the list (to
+//!   determine the starting number of the list). All subsequent ordered list
+//!   items will have sequentially increasing numbers.
+//!
+//!   All list items may contain block content. Any content indented at least as
+//!   far as the end of the list item marker (including the space after it) is
+//!   considered part of the list item.
+//!
+//!   Lists which have no blank lines between items or between direct children
+//!   of items are considered _tight_, and direct child paragraphs of tight list
+//!   items are rendered without `<p>` tags.
+//!
+//! - **Table** - a sequence of adjacent table row lines, where each line starts
+//!   and ends with a `|`, and cells within the row are delimited by `|`s.
+//!
+//!   The first or second row of a table may be a _header delimiter row_, which
+//!   is a row consisting of cells of the pattern `---` (for unset column
+//!   alignment), `:--` (for left alignment), `:-:` (for center alignment), or
+//!   `--:` (for right alignment). The number of `-`s must be at least one, but
+//!   is otherwise arbitrary. If there is a row just before the header delimiter
+//!   row, it becomes the header row for the table (a table need not have a
+//!   header row at all).
+//!
+//! - **Heading** - a sequence of between 1 and 6 `#` characters, followed by a
+//!   space and further inline content on the same line.
+//!
+//! - **Code block** - a sequence of at least 3 `` ` `` characters (a _fence_),
+//!   optionally followed by a "tag" on the same line, and continuing until a
+//!   line consisting only of a closing fence whose length matches the opening
+//!   fence, or until the end of the containing block.
+//!
+//!   The content of a code block is not parsed as inline content. It is
+//!   included verbatim in the output document (minus leading indentation up to
+//!   the position of the opening fence).
+//!
+//! - **Blockquote** - a sequence of lines preceded by `>` characters.
+//!
+//! - **Paragraph** - ordinary text, parsed as inline content, ending with a
+//!   blank line or the end of the containing block.
+//!
+//!   Paragraphs which are part of another block may be "lazily" continued by
+//!   subsequent paragraph lines even if those lines would not ordinarily be
+//!   considered part of the containing block. For example, this is a single
+//!   list item, not a list item followed by a paragraph:
+//!
+//!   ```markdown
+//!   - First line of content.
+//!   This content is still part of the paragraph,
+//!   even though it isn't indented far enough.
+//!   ```
+//!
+//! - **Thematic break** - a line consisting of at least three matching `-`,
+//!   `_`, or `*` characters and, optionally, spaces.
+//!
+//! Indentation may consist of spaces and tabs. The use of tabs is not
+//! recommended: a tab is treated the same as a single space for the purpose of
+//! determining the indentation level, and is not recognized as a space for
+//! block starters which require one (for example, `-` followed by a tab is not
+//! a valid list item).
+//!
+//! The supported inlines are as follows:
+//!
+//! - **Link** - of the format `[text](target)`. `text` may contain inline
+//!   content. `target` may contain `\`-escaped characters and balanced
+//!   parentheses.
+//!
+//! - **Image** - a link directly preceded by a `!`. The link text is
+//!   interpreted as the alt text of the image.
+//!
+//! - **Emphasis** - a run of `*` or `_` characters may be an emphasis opener,
+//!   closer, or both. For `*` characters, the run may be an opener as long as
+//!   it is not directly followed by a whitespace character (or the end of the
+//!   inline content) and a closer as long as it is not directly preceded by
+//!   one. For `_` characters, this rule is strengthened by requiring that the
+//!   run also be preceded by a whitespace or punctuation character (for
+//!   openers) or followed by one (for closers), to avoid mangling `snake_case`
+//!   words.
+//!
+//!   The rule for emphasis handling is greedy: any run that can close existing
+//!   emphasis will do so, otherwise it will open emphasis. A single run may
+//!   serve both functions: the middle `**` in the following example both closes
+//!   the initial emphasis and opens a new one:
+//!
+//!   ```markdown
+//!   *one**two*
+//!   ```
+//!
+//!   A single `*` or `_` is used for normal emphasis (HTML `<em>`), and a
+//!   double `**` or `__` is used for strong emphasis (HTML `<strong>`). Even
+//!   longer runs may be used to produce further nested emphasis (though only
+//!   `***` and `___` to produce `<em><strong>` is really useful).
+//!
+//! - **Code span** - a run of `` ` `` characters, terminated by a matching run
+//!   or the end of inline content. The content of a code span is not parsed
+//!   further.
+//!
+//! - **Text** - normal text is interpreted as-is, except that `\` may be used
+//!   to escape any punctuation character, preventing it from being interpreted
+//!   according to other syntax rules. A `\` followed by a line break within a
+//!   paragraph is interpreted as a hard line break.
+//!
+//!   Any null bytes or invalid UTF-8 bytes within text are replaced with Unicode
+//!   replacement characters, `U+FFFD`.
+
+const std = @import("std");
+const testing = std.testing;
+
+pub const Document = @import("markdown/Document.zig");
+pub const Parser = @import("markdown/Parser.zig");
+pub const Renderer = @import("markdown/renderer.zig").Renderer;
+pub const renderNodeInlineText = @import("markdown/renderer.zig").renderNodeInlineText;
+pub const fmtHtml = @import("markdown/renderer.zig").fmtHtml;
+
+// Avoid exposing main to other files merely importing this one.
+pub const main = if (@import("root") == @This())
+    mainImpl
+else
+    @compileError("only available as root source file");
+
+fn mainImpl() !void {
+    const gpa = std.heap.c_allocator;
+
+    var parser = try Parser.init(gpa);
+    defer parser.deinit();
+
+    var stdin_buf = std.io.bufferedReader(std.io.getStdIn().reader());
+    var line_buf = std.ArrayList(u8).init(gpa);
+    defer line_buf.deinit();
+    while (stdin_buf.reader().streamUntilDelimiter(line_buf.writer(), '\n', null)) {
+        if (line_buf.getLastOrNull() == '\r') _ = line_buf.pop();
+        try parser.feedLine(line_buf.items);
+        line_buf.clearRetainingCapacity();
+    } else |err| switch (err) {
+        error.EndOfStream => {},
+        else => |e| return e,
+    }
+
+    var doc = try parser.endInput();
+    defer doc.deinit(gpa);
+
+    var stdout_buf = std.io.bufferedWriter(std.io.getStdOut().writer());
+    try doc.render(stdout_buf.writer());
+    try stdout_buf.flush();
+}
+
+test "empty document" {
+    try testRender("", "");
+    try testRender("   ", "");
+    try testRender("\n \n\t\n   \n", "");
+}
+
+test "unordered lists" {
+    try testRender(
+        \\- Spam
+        \\- Spam
+        \\- Spam
+        \\- Eggs
+        \\- Bacon
+        \\- Spam
+        \\
+        \\* Spam
+        \\* Spam
+        \\* Spam
+        \\* Eggs
+        \\* Bacon
+        \\* Spam
+        \\
+        \\+ Spam
+        \\+ Spam
+        \\+ Spam
+        \\+ Eggs
+        \\+ Bacon
+        \\+ Spam
+        \\
+    ,
+        \\<ul>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Eggs</li>
+        \\<li>Bacon</li>
+        \\<li>Spam</li>
+        \\</ul>
+        \\<ul>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Eggs</li>
+        \\<li>Bacon</li>
+        \\<li>Spam</li>
+        \\</ul>
+        \\<ul>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Spam</li>
+        \\<li>Eggs</li>
+        \\<li>Bacon</li>
+        \\<li>Spam</li>
+        \\</ul>
+        \\
+    );
+}
+
+test "ordered lists" {
+    try testRender(
+        \\1. Breakfast
+        \\2. Second breakfast
+        \\3. Lunch
+        \\2. Afternoon snack
+        \\1. Dinner
+        \\6. Dessert
+        \\7. Midnight snack
+        \\
+        \\1) Breakfast
+        \\2) Second breakfast
+        \\3) Lunch
+        \\2) Afternoon snack
+        \\1) Dinner
+        \\6) Dessert
+        \\7) Midnight snack
+        \\
+        \\1001. Breakfast
+        \\2. Second breakfast
+        \\3. Lunch
+        \\2. Afternoon snack
+        \\1. Dinner
+        \\6. Dessert
+        \\7. Midnight snack
+        \\
+        \\1001) Breakfast
+        \\2) Second breakfast
+        \\3) Lunch
+        \\2) Afternoon snack
+        \\1) Dinner
+        \\6) Dessert
+        \\7) Midnight snack
+        \\
+    ,
+        \\<ol>
+        \\<li>Breakfast</li>
+        \\<li>Second breakfast</li>
+        \\<li>Lunch</li>
+        \\<li>Afternoon snack</li>
+        \\<li>Dinner</li>
+        \\<li>Dessert</li>
+        \\<li>Midnight snack</li>
+        \\</ol>
+        \\<ol>
+        \\<li>Breakfast</li>
+        \\<li>Second breakfast</li>
+        \\<li>Lunch</li>
+        \\<li>Afternoon snack</li>
+        \\<li>Dinner</li>
+        \\<li>Dessert</li>
+        \\<li>Midnight snack</li>
+        \\</ol>
+        \\<ol start="1001">
+        \\<li>Breakfast</li>
+        \\<li>Second breakfast</li>
+        \\<li>Lunch</li>
+        \\<li>Afternoon snack</li>
+        \\<li>Dinner</li>
+        \\<li>Dessert</li>
+        \\<li>Midnight snack</li>
+        \\</ol>
+        \\<ol start="1001">
+        \\<li>Breakfast</li>
+        \\<li>Second breakfast</li>
+        \\<li>Lunch</li>
+        \\<li>Afternoon snack</li>
+        \\<li>Dinner</li>
+        \\<li>Dessert</li>
+        \\<li>Midnight snack</li>
+        \\</ol>
+        \\
+    );
+}
+
+test "nested lists" {
+    try testRender(
+        \\- - Item 1.
+        \\  - Item 2.
+        \\Item 2 continued.
+        \\  * New list.
+        \\
+    ,
+        \\<ul>
+        \\<li><ul>
+        \\<li>Item 1.</li>
+        \\<li>Item 2.
+        \\Item 2 continued.</li>
+        \\</ul>
+        \\<ul>
+        \\<li>New list.</li>
+        \\</ul>
+        \\</li>
+        \\</ul>
+        \\
+    );
+}
+
+test "lists with block content" {
+    try testRender(
+        \\1. Item 1.
+        \\2. Item 2.
+        \\
+        \\   This one has another paragraph.
+        \\3. Item 3.
+        \\
+        \\- > Blockquote.
+        \\- - Sub-list.
+        \\  - Sub-list continued.
+        \\  * Different sub-list.
+        \\- ## Heading.
+        \\
+        \\  Some contents below the heading.
+        \\  1. Item 1.
+        \\  2. Item 2.
+        \\  3. Item 3.
+        \\
+    ,
+        \\<ol>
+        \\<li><p>Item 1.</p>
+        \\</li>
+        \\<li><p>Item 2.</p>
+        \\<p>This one has another paragraph.</p>
+        \\</li>
+        \\<li><p>Item 3.</p>
+        \\</li>
+        \\</ol>
+        \\<ul>
+        \\<li><blockquote>
+        \\<p>Blockquote.</p>
+        \\</blockquote>
+        \\</li>
+        \\<li><ul>
+        \\<li>Sub-list.</li>
+        \\<li>Sub-list continued.</li>
+        \\</ul>
+        \\<ul>
+        \\<li>Different sub-list.</li>
+        \\</ul>
+        \\</li>
+        \\<li><h2>Heading.</h2>
+        \\<p>Some contents below the heading.</p>
+        \\<ol>
+        \\<li>Item 1.</li>
+        \\<li>Item 2.</li>
+        \\<li>Item 3.</li>
+        \\</ol>
+        \\</li>
+        \\</ul>
+        \\
+    );
+}
+
+test "tables" {
+    try testRender(
+        \\| Operator | Meaning          |
+        \\| :------: | ---------------- |
+        \\| `+`      | Add              |
+        \\| `-`      | Subtract         |
+        \\| `*`      | Multiply         |
+        \\| `/`      | Divide           |
+        \\| `??`     | **Not sure yet** |
+        \\
+        \\| Item 1 | Value 1 |
+        \\| Item 2 | Value 2 |
+        \\| Item 3 | Value 3 |
+        \\| Item 4 | Value 4 |
+        \\
+        \\| :--- | :----: | ----: |
+        \\| Left | Center | Right |
+        \\
+    ,
+        \\<table>
+        \\<tr>
+        \\<th style="text-align: center">Operator</th>
+        \\<th>Meaning</th>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: center"><code>+</code></td>
+        \\<td>Add</td>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: center"><code>-</code></td>
+        \\<td>Subtract</td>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: center"><code>*</code></td>
+        \\<td>Multiply</td>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: center"><code>/</code></td>
+        \\<td>Divide</td>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: center"><code>??</code></td>
+        \\<td><strong>Not sure yet</strong></td>
+        \\</tr>
+        \\</table>
+        \\<table>
+        \\<tr>
+        \\<td>Item 1</td>
+        \\<td>Value 1</td>
+        \\</tr>
+        \\<tr>
+        \\<td>Item 2</td>
+        \\<td>Value 2</td>
+        \\</tr>
+        \\<tr>
+        \\<td>Item 3</td>
+        \\<td>Value 3</td>
+        \\</tr>
+        \\<tr>
+        \\<td>Item 4</td>
+        \\<td>Value 4</td>
+        \\</tr>
+        \\</table>
+        \\<table>
+        \\<tr>
+        \\<td style="text-align: left">Left</td>
+        \\<td style="text-align: center">Center</td>
+        \\<td style="text-align: right">Right</td>
+        \\</tr>
+        \\</table>
+        \\
+    );
+}
+
+test "table with uneven number of columns" {
+    try testRender(
+        \\| One |
+        \\| :-- | :--: |
+        \\| One | Two | Three |
+        \\
+    ,
+        \\<table>
+        \\<tr>
+        \\<th style="text-align: left">One</th>
+        \\</tr>
+        \\<tr>
+        \\<td style="text-align: left">One</td>
+        \\<td style="text-align: center">Two</td>
+        \\<td>Three</td>
+        \\</tr>
+        \\</table>
+        \\
+    );
+}
+
+test "table with escaped pipes" {
+    try testRender(
+        \\| One \| Two |
+        \\| --- | --- |
+        \\| One \| Two |
+        \\
+    ,
+        \\<table>
+        \\<tr>
+        \\<th>One | Two</th>
+        \\</tr>
+        \\<tr>
+        \\<td>One | Two</td>
+        \\</tr>
+        \\</table>
+        \\
+    );
+}
+
+test "table with pipes in code spans" {
+    try testRender(
+        \\| `|` | Bitwise _OR_ |
+        \\| `||` | Combines error sets |
+        \\| `` `||` `` | Escaped version |
+        \\| ` ``||`` ` | Another escaped version |
+        \\| `Oops unterminated code span |
+        \\
+    ,
+        \\<table>
+        \\<tr>
+        \\<td><code>|</code></td>
+        \\<td>Bitwise <em>OR</em></td>
+        \\</tr>
+        \\<tr>
+        \\<td><code>||</code></td>
+        \\<td>Combines error sets</td>
+        \\</tr>
+        \\<tr>
+        \\<td><code>`||`</code></td>
+        \\<td>Escaped version</td>
+        \\</tr>
+        \\<tr>
+        \\<td><code>``||``</code></td>
+        \\<td>Another escaped version</td>
+        \\</tr>
+        \\</table>
+        \\<p>| <code>Oops unterminated code span |</code></p>
+        \\
+    );
+}
+
+test "tables require leading and trailing pipes" {
+    try testRender(
+        \\Not | a | table
+        \\
+        \\| But | this | is |
+        \\
+    ,
+        \\<p>Not | a | table</p>
+        \\<table>
+        \\<tr>
+        \\<td>But</td>
+        \\<td>this</td>
+        \\<td>is</td>
+        \\</tr>
+        \\</table>
+        \\
+    );
+}
+
+test "headings" {
+    try testRender(
+        \\# Level one
+        \\## Level two
+        \\### Level three
+        \\#### Level four
+        \\##### Level five
+        \\###### Level six
+        \\####### Not a heading
+        \\
+    ,
+        \\<h1>Level one</h1>
+        \\<h2>Level two</h2>
+        \\<h3>Level three</h3>
+        \\<h4>Level four</h4>
+        \\<h5>Level five</h5>
+        \\<h6>Level six</h6>
+        \\<p>####### Not a heading</p>
+        \\
+    );
+}
+
+test "headings with inline content" {
+    try testRender(
+        \\# Outline of `std.zig`
+        \\## **Important** notes
+        \\### ***Nested* inline content**
+        \\
+    ,
+        \\<h1>Outline of <code>std.zig</code></h1>
+        \\<h2><strong>Important</strong> notes</h2>
+        \\<h3><strong><em>Nested</em> inline content</strong></h3>
+        \\
+    );
+}
+
+test "code blocks" {
+    try testRender(
+        \\```
+        \\Hello, world!
+        \\This is some code.
+        \\```
+        \\``` zig test
+        \\const std = @import("std");
+        \\
+        \\test {
+        \\    try std.testing.expect(2 + 2 == 4);
+        \\}
+        \\```
+        \\
+    ,
+        \\<pre><code>Hello, world!
+        \\This is some code.
+        \\</code></pre>
+        \\<pre><code class="zig test">const std = @import(&quot;std&quot;);
+        \\
+        \\test {
+        \\    try std.testing.expect(2 + 2 == 4);
+        \\}
+        \\</code></pre>
+        \\
+    );
+}
+
+test "blockquotes" {
+    try testRender(
+        \\> > You miss 100% of the shots you don't take.
+        \\> >
+        \\> > ~ Wayne Gretzky
+        \\>
+        \\> ~ Michael Scott
+        \\
+    ,
+        \\<blockquote>
+        \\<blockquote>
+        \\<p>You miss 100% of the shots you don't take.</p>
+        \\<p>~ Wayne Gretzky</p>
+        \\</blockquote>
+        \\<p>~ Michael Scott</p>
+        \\</blockquote>
+        \\
+    );
+}
+
+test "blockquote lazy continuation lines" {
+    try testRender(
+        \\>>>>Deeply nested blockquote
+        \\>>which continues on another line
+        \\and then yet another one.
+        \\>>
+        \\>> But now two of them have been closed.
+        \\
+        \\And then there were none.
+        \\
+    ,
+        \\<blockquote>
+        \\<blockquote>
+        \\<blockquote>
+        \\<blockquote>
+        \\<p>Deeply nested blockquote
+        \\which continues on another line
+        \\and then yet another one.</p>
+        \\</blockquote>
+        \\</blockquote>
+        \\<p>But now two of them have been closed.</p>
+        \\</blockquote>
+        \\</blockquote>
+        \\<p>And then there were none.</p>
+        \\
+    );
+}
+
+test "paragraphs" {
+    try testRender(
+        \\Paragraph one.
+        \\
+        \\Paragraph two.
+        \\Still in the paragraph.
+        \\    So is this.
+        \\
+        \\
+        \\
+        \\
+        \\ Last paragraph.
+        \\
+    ,
+        \\<p>Paragraph one.</p>
+        \\<p>Paragraph two.
+        \\Still in the paragraph.
+        \\So is this.</p>
+        \\<p>Last paragraph.</p>
+        \\
+    );
+}
+
+test "thematic breaks" {
+    try testRender(
+        \\---
+        \\***
+        \\___
+        \\          ---
+        \\ - - - - - - - - - - -
+        \\
+    ,
+        \\<hr />
+        \\<hr />
+        \\<hr />
+        \\<hr />
+        \\<hr />
+        \\
+    );
+}
+
+test "links" {
+    try testRender(
+        \\[Link](https://example.com)
+        \\[Link *with inlines*](https://example.com)
+        \\[Nested parens](https://example.com/nested(parens(inside)))
+        \\[Escaped parens](https://example.com/\)escaped\()
+        \\[Line break in target](test\
+        \\target)
+        \\
+    ,
+        \\<p><a href="https://example.com">Link</a>
+        \\<a href="https://example.com">Link <em>with inlines</em></a>
+        \\<a href="https://example.com/nested(parens(inside))">Nested parens</a>
+        \\<a href="https://example.com/)escaped(">Escaped parens</a>
+        \\<a href="test\
+        \\target">Line break in target</a></p>
+        \\
+    );
+}
+
+test "images" {
+    try testRender(
+        \\![Alt text](https://example.com/image.png)
+        \\![Alt text *with inlines*](https://example.com/image.png)
+        \\![Nested parens](https://example.com/nested(parens(inside)).png)
+        \\![Escaped parens](https://example.com/\)escaped\(.png)
+        \\![Line break in target](test\
+        \\target)
+        \\
+    ,
+        \\<p><img src="https://example.com/image.png" alt="Alt text" />
+        \\<img src="https://example.com/image.png" alt="Alt text with inlines" />
+        \\<img src="https://example.com/nested(parens(inside)).png" alt="Nested parens" />
+        \\<img src="https://example.com/)escaped(.png" alt="Escaped parens" />
+        \\<img src="test\
+        \\target" alt="Line break in target" /></p>
+        \\
+    );
+}
+
+test "emphasis" {
+    try testRender(
+        \\*Emphasis.*
+        \\**Strong.**
+        \\***Strong emphasis.***
+        \\****More...****
+        \\*****MORE...*****
+        \\******Even more...******
+        \\*******OK, this is enough.*******
+        \\
+    ,
+        \\<p><em>Emphasis.</em>
+        \\<strong>Strong.</strong>
+        \\<em><strong>Strong emphasis.</strong></em>
+        \\<em><strong><em>More...</em></strong></em>
+        \\<em><strong><strong>MORE...</strong></strong></em>
+        \\<em><strong><em><strong>Even more...</strong></em></strong></em>
+        \\<em><strong><em><strong><em>OK, this is enough.</em></strong></em></strong></em></p>
+        \\
+    );
+    try testRender(
+        \\_Emphasis._
+        \\__Strong.__
+        \\___Strong emphasis.___
+        \\____More...____
+        \\_____MORE..._____
+        \\______Even more...______
+        \\_______OK, this is enough._______
+        \\
+    ,
+        \\<p><em>Emphasis.</em>
+        \\<strong>Strong.</strong>
+        \\<em><strong>Strong emphasis.</strong></em>
+        \\<em><strong><em>More...</em></strong></em>
+        \\<em><strong><strong>MORE...</strong></strong></em>
+        \\<em><strong><em><strong>Even more...</strong></em></strong></em>
+        \\<em><strong><em><strong><em>OK, this is enough.</em></strong></em></strong></em></p>
+        \\
+    );
+}
+
+test "nested emphasis" {
+    try testRender(
+        \\**Hello, *world!***
+        \\*Hello, **world!***
+        \\**Hello, _world!_**
+        \\_Hello, **world!**_
+        \\*Hello, **nested** *world!**
+        \\***Hello,* world!**
+        \\__**Hello, world!**__
+        \\****Hello,** world!**
+        \\__Hello,_ world!_
+        \\*Test**123*
+        \\__Test____123__
+        \\
+    ,
+        \\<p><strong>Hello, <em>world!</em></strong>
+        \\<em>Hello, <strong>world!</strong></em>
+        \\<strong>Hello, <em>world!</em></strong>
+        \\<em>Hello, <strong>world!</strong></em>
+        \\<em>Hello, <strong>nested</strong> <em>world!</em></em>
+        \\<strong><em>Hello,</em> world!</strong>
+        \\<strong><strong>Hello, world!</strong></strong>
+        \\<strong><strong>Hello,</strong> world!</strong>
+        \\<em><em>Hello,</em> world!</em>
+        \\<em>Test</em><em>123</em>
+        \\<strong>Test____123</strong></p>
+        \\
+    );
+}
+
+test "emphasis precedence" {
+    try testRender(
+        \\*First one _wins*_.
+        \\_*No other __rule matters.*_
+        \\
+    ,
+        \\<p><em>First one _wins</em>_.
+        \\<em><em>No other __rule matters.</em></em></p>
+        \\
+    );
+}
+
+test "emphasis open and close" {
+    try testRender(
+        \\Cannot open: *
+        \\Cannot open: _
+        \\*Cannot close: *
+        \\_Cannot close: _
+        \\
+        \\foo*bar*baz
+        \\foo_bar_baz
+        \\foo**bar**baz
+        \\foo__bar__baz
+        \\
+    ,
+        \\<p>Cannot open: *
+        \\Cannot open: _
+        \\*Cannot close: *
+        \\_Cannot close: _</p>
+        \\<p>foo<em>bar</em>baz
+        \\foo_bar_baz
+        \\foo<strong>bar</strong>baz
+        \\foo__bar__baz</p>
+        \\
+    );
+}
+
+test "code spans" {
+    try testRender(
+        \\`Hello, world!`
+        \\```Multiple `backticks` can be used.```
+        \\`**This** does not produce emphasis.`
+        \\`` `Backtick enclosed string.` ``
+        \\`Delimiter lengths ```must``` match.`
+        \\
+        \\Unterminated ``code...
+        \\
+        \\Weird empty code span: `
+        \\
+        \\**Very important code: `hi`**
+        \\
+    ,
+        \\<p><code>Hello, world!</code>
+        \\<code>Multiple `backticks` can be used.</code>
+        \\<code>**This** does not produce emphasis.</code>
+        \\<code>`Backtick enclosed string.`</code>
+        \\<code>Delimiter lengths ```must``` match.</code></p>
+        \\<p>Unterminated <code>code...</code></p>
+        \\<p>Weird empty code span: <code></code></p>
+        \\<p><strong>Very important code: <code>hi</code></strong></p>
+        \\
+    );
+}
+
+test "backslash escapes" {
+    try testRender(
+        \\Not \*emphasized\*.
+        \\Literal \\backslashes\\.
+        \\Not code: \`hi\`.
+        \\\# Not a title.
+        \\#\# Also not a title.
+        \\\> Not a blockquote.
+        \\\- Not a list item.
+        \\\| Not a table. |
+        \\| Also not a table. \|
+        \\Any \punctuation\ characte\r can be escaped:
+        \\\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~
+        \\
+    ,
+        \\<p>Not *emphasized*.
+        \\Literal \backslashes\.
+        \\Not code: `hi`.
+        \\# Not a title.
+        \\## Also not a title.
+        \\&gt; Not a blockquote.
+        \\- Not a list item.
+        \\| Not a table. |
+        \\| Also not a table. |
+        \\Any \punctuation\ characte\r can be escaped:
+        \\!&quot;#$%&amp;'()*+,-./:;&lt;=&gt;?@[\]^_`{|}~</p>
+        \\
+    );
+}
+
+test "hard line breaks" {
+    try testRender(
+        \\The iguana sits\
+        \\Perched atop a short desk chair\
+        \\Writing code in Zig
+        \\
+    ,
+        \\<p>The iguana sits<br />
+        \\Perched atop a short desk chair<br />
+        \\Writing code in Zig</p>
+        \\
+    );
+}
+
+test "Unicode handling" {
+    // Null bytes must be replaced.
+    try testRender("\x00\x00\x00", "<p>\u{FFFD}\u{FFFD}\u{FFFD}</p>\n");
+
+    // Invalid UTF-8 must be replaced.
+    try testRender("\xC0\x80\xE0\x80\x80\xF0\x80\x80\x80", "<p>\u{FFFD}\u{FFFD}\u{FFFD}</p>\n");
+    try testRender("\xED\xA0\x80\xED\xBF\xBF", "<p>\u{FFFD}\u{FFFD}</p>\n");
+
+    // Incomplete UTF-8 must be replaced.
+    try testRender("\xE2\x82", "<p>\u{FFFD}</p>\n");
+}
+
+fn testRender(input: []const u8, expected: []const u8) !void {
+    var parser = try Parser.init(testing.allocator);
+    defer parser.deinit();
+
+    var lines = std.mem.split(u8, input, "\n");
+    while (lines.next()) |line| {
+        try parser.feedLine(line);
+    }
+    var doc = try parser.endInput();
+    defer doc.deinit(testing.allocator);
+
+    var actual = std.ArrayList(u8).init(testing.allocator);
+    defer actual.deinit();
+    try doc.render(actual.writer());
+
+    try testing.expectEqualStrings(expected, actual.items);
+}

lib/docs/wasm/markdown/Document.zig

@@ -0,0 +1,192 @@
+//! An abstract tree representation of a Markdown document.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+const Renderer = @import("renderer.zig").Renderer;
+
+nodes: Node.List.Slice,
+extra: []u32,
+string_bytes: []u8,
+
+const Document = @This();
+
+pub const Node = struct {
+    tag: Tag,
+    data: Data,
+
+    pub const Index = enum(u32) {
+        root = 0,
+        _,
+    };
+    pub const List = std.MultiArrayList(Node);
+
+    pub const Tag = enum {
+        /// Data is `container`.
+        root,
+
+        // Blocks
+        /// Data is `list`.
+        list,
+        /// Data is `list_item`.
+        list_item,
+        /// Data is `container`.
+        table,
+        /// Data is `container`.
+        table_row,
+        /// Data is `table_cell`.
+        table_cell,
+        /// Data is `heading`.
+        heading,
+        /// Data is `code_block`.
+        code_block,
+        /// Data is `container`.
+        blockquote,
+        /// Data is `container`.
+        paragraph,
+        /// Data is `none`.
+        thematic_break,
+
+        // Inlines
+        /// Data is `link`.
+        link,
+        /// Data is `link`.
+        image,
+        /// Data is `container`.
+        strong,
+        /// Data is `container`.
+        emphasis,
+        /// Data is `text`.
+        code_span,
+        /// Data is `text`.
+        text,
+        /// Data is `none`.
+        line_break,
+    };
+
+    pub const Data = union {
+        none: void,
+        container: struct {
+            children: ExtraIndex,
+        },
+        text: struct {
+            content: StringIndex,
+        },
+        list: struct {
+            start: ListStart,
+            children: ExtraIndex,
+        },
+        list_item: struct {
+            tight: bool,
+            children: ExtraIndex,
+        },
+        table_cell: struct {
+            info: packed struct {
+                alignment: TableCellAlignment,
+                header: bool,
+            },
+            children: ExtraIndex,
+        },
+        heading: struct {
+            /// Between 1 and 6, inclusive.
+            level: u3,
+            children: ExtraIndex,
+        },
+        code_block: struct {
+            tag: StringIndex,
+            content: StringIndex,
+        },
+        link: struct {
+            target: StringIndex,
+            children: ExtraIndex,
+        },
+
+        comptime {
+            // In Debug and ReleaseSafe builds, there may be hidden extra fields
+            // included for safety checks. Without such safety checks enabled,
+            // we always want this union to be 8 bytes.
+            if (builtin.mode != .Debug and builtin.mode != .ReleaseSafe) {
+                assert(@sizeOf(Data) == 8);
+            }
+        }
+    };
+
+    /// The starting number of a list. This is either a number between 0 and
+    /// 999,999,999, inclusive, or `unordered` to indicate an unordered list.
+    pub const ListStart = enum(u30) {
+        // When https://github.com/ziglang/zig/issues/104 is implemented, this
+        // type can be more naturally expressed as ?u30. As it is, we want
+        // values to fit within 4 bytes, so ?u30 does not yet suffice for
+        // storage.
+        unordered = std.math.maxInt(u30),
+        _,
+
+        pub fn asNumber(start: ListStart) ?u30 {
+            if (start == .unordered) return null;
+            assert(@intFromEnum(start) <= 999_999_999);
+            return @intFromEnum(start);
+        }
+    };
+
+    pub const TableCellAlignment = enum {
+        unset,
+        left,
+        center,
+        right,
+    };
+
+    /// Trailing: `len` times `Node.Index`
+    pub const Children = struct {
+        len: u32,
+    };
+};
+
+pub const ExtraIndex = enum(u32) { _ };
+
+/// The index of a null-terminated string in `string_bytes`.
+pub const StringIndex = enum(u32) {
+    empty = 0,
+    _,
+};
+
+pub fn deinit(doc: *Document, allocator: Allocator) void {
+    doc.nodes.deinit(allocator);
+    allocator.free(doc.extra);
+    allocator.free(doc.string_bytes);
+    doc.* = undefined;
+}
+
+/// Renders a document directly to a writer using the default renderer.
+pub fn render(doc: Document, writer: anytype) @TypeOf(writer).Error!void {
+    const renderer: Renderer(@TypeOf(writer), void) = .{ .context = {} };
+    try renderer.render(doc, writer);
+}
+
+pub fn ExtraData(comptime T: type) type {
+    return struct { data: T, end: usize };
+}
+
+pub fn extraData(doc: Document, comptime T: type, index: ExtraIndex) ExtraData(T) {
+    const fields = @typeInfo(T).Struct.fields;
+    var i: usize = @intFromEnum(index);
+    var result: T = undefined;
+    inline for (fields) |field| {
+        @field(result, field.name) = switch (field.type) {
+            u32 => doc.extra[i],
+            else => @compileError("bad field type"),
+        };
+        i += 1;
+    }
+    return .{ .data = result, .end = i };
+}
+
+pub fn extraChildren(doc: Document, index: ExtraIndex) []const Node.Index {
+    const children = doc.extraData(Node.Children, index);
+    return @ptrCast(doc.extra[children.end..][0..children.data.len]);
+}
+
+pub fn string(doc: Document, index: StringIndex) [:0]const u8 {
+    const start = @intFromEnum(index);
+    return std.mem.span(@as([*:0]u8, @ptrCast(doc.string_bytes[start..].ptr)));
+}

lib/docs/wasm/markdown/renderer.zig

@@ -0,0 +1,254 @@
+const std = @import("std");
+const Document = @import("Document.zig");
+const Node = Document.Node;
+
+/// A Markdown document renderer.
+///
+/// Each concrete `Renderer` type has a `renderDefault` function, with the
+/// intention that custom `renderFn` implementations can call `renderDefault`
+/// for node types for which they require no special rendering.
+pub fn Renderer(comptime Writer: type, comptime Context: type) type {
+    return struct {
+        renderFn: *const fn (
+            r: Self,
+            doc: Document,
+            node: Node.Index,
+            writer: Writer,
+        ) Writer.Error!void = renderDefault,
+        context: Context,
+
+        const Self = @This();
+
+        pub fn render(r: Self, doc: Document, writer: Writer) Writer.Error!void {
+            try r.renderFn(r, doc, .root, writer);
+        }
+
+        pub fn renderDefault(
+            r: Self,
+            doc: Document,
+            node: Node.Index,
+            writer: Writer,
+        ) Writer.Error!void {
+            const data = doc.nodes.items(.data)[@intFromEnum(node)];
+            switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
+                .root => {
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                },
+                .list => {
+                    if (data.list.start.asNumber()) |start| {
+                        if (start == 1) {
+                            try writer.writeAll("<ol>\n");
+                        } else {
+                            try writer.print("<ol start=\"{}\">\n", .{start});
+                        }
+                    } else {
+                        try writer.writeAll("<ul>\n");
+                    }
+                    for (doc.extraChildren(data.list.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    if (data.list.start.asNumber() != null) {
+                        try writer.writeAll("</ol>\n");
+                    } else {
+                        try writer.writeAll("</ul>\n");
+                    }
+                },
+                .list_item => {
+                    try writer.writeAll("<li>");
+                    for (doc.extraChildren(data.list_item.children)) |child| {
+                        if (data.list_item.tight and doc.nodes.items(.tag)[@intFromEnum(child)] == .paragraph) {
+                            const para_data = doc.nodes.items(.data)[@intFromEnum(child)];
+                            for (doc.extraChildren(para_data.container.children)) |para_child| {
+                                try r.renderFn(r, doc, para_child, writer);
+                            }
+                        } else {
+                            try r.renderFn(r, doc, child, writer);
+                        }
+                    }
+                    try writer.writeAll("</li>\n");
+                },
+                .table => {
+                    try writer.writeAll("<table>\n");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</table>\n");
+                },
+                .table_row => {
+                    try writer.writeAll("<tr>\n");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</tr>\n");
+                },
+                .table_cell => {
+                    if (data.table_cell.info.header) {
+                        try writer.writeAll("<th");
+                    } else {
+                        try writer.writeAll("<td");
+                    }
+                    switch (data.table_cell.info.alignment) {
+                        .unset => try writer.writeAll(">"),
+                        else => |a| try writer.print(" style=\"text-align: {s}\">", .{@tagName(a)}),
+                    }
+
+                    for (doc.extraChildren(data.table_cell.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+
+                    if (data.table_cell.info.header) {
+                        try writer.writeAll("</th>\n");
+                    } else {
+                        try writer.writeAll("</td>\n");
+                    }
+                },
+                .heading => {
+                    try writer.print("<h{}>", .{data.heading.level});
+                    for (doc.extraChildren(data.heading.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.print("</h{}>\n", .{data.heading.level});
+                },
+                .code_block => {
+                    const tag = doc.string(data.code_block.tag);
+                    const content = doc.string(data.code_block.content);
+                    if (tag.len > 0) {
+                        try writer.print("<pre><code class=\"{}\">{}</code></pre>\n", .{ fmtHtml(tag), fmtHtml(content) });
+                    } else {
+                        try writer.print("<pre><code>{}</code></pre>\n", .{fmtHtml(content)});
+                    }
+                },
+                .blockquote => {
+                    try writer.writeAll("<blockquote>\n");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</blockquote>\n");
+                },
+                .paragraph => {
+                    try writer.writeAll("<p>");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</p>\n");
+                },
+                .thematic_break => {
+                    try writer.writeAll("<hr />\n");
+                },
+                .link => {
+                    const target = doc.string(data.link.target);
+                    try writer.print("<a href=\"{}\">", .{fmtHtml(target)});
+                    for (doc.extraChildren(data.link.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</a>");
+                },
+                .image => {
+                    const target = doc.string(data.link.target);
+                    try writer.print("<img src=\"{}\" alt=\"", .{fmtHtml(target)});
+                    for (doc.extraChildren(data.link.children)) |child| {
+                        try renderInlineNodeText(doc, child, writer);
+                    }
+                    try writer.writeAll("\" />");
+                },
+                .strong => {
+                    try writer.writeAll("<strong>");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</strong>");
+                },
+                .emphasis => {
+                    try writer.writeAll("<em>");
+                    for (doc.extraChildren(data.container.children)) |child| {
+                        try r.renderFn(r, doc, child, writer);
+                    }
+                    try writer.writeAll("</em>");
+                },
+                .code_span => {
+                    const content = doc.string(data.text.content);
+                    try writer.print("<code>{}</code>", .{fmtHtml(content)});
+                },
+                .text => {
+                    const content = doc.string(data.text.content);
+                    try writer.print("{}", .{fmtHtml(content)});
+                },
+                .line_break => {
+                    try writer.writeAll("<br />\n");
+                },
+            }
+        }
+    };
+}
+
+/// Renders an inline node as plain text. Asserts that the node is an inline and
+/// has no non-inline children.
+pub fn renderInlineNodeText(
+    doc: Document,
+    node: Node.Index,
+    writer: anytype,
+) @TypeOf(writer).Error!void {
+    const data = doc.nodes.items(.data)[@intFromEnum(node)];
+    switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
+        .root,
+        .list,
+        .list_item,
+        .table,
+        .table_row,
+        .table_cell,
+        .heading,
+        .code_block,
+        .blockquote,
+        .paragraph,
+        .thematic_break,
+        => unreachable, // Blocks
+
+        .link, .image => {
+            for (doc.extraChildren(data.link.children)) |child| {
+                try renderInlineNodeText(doc, child, writer);
+            }
+        },
+        .strong => {
+            for (doc.extraChildren(data.container.children)) |child| {
+                try renderInlineNodeText(doc, child, writer);
+            }
+        },
+        .emphasis => {
+            for (doc.extraChildren(data.container.children)) |child| {
+                try renderInlineNodeText(doc, child, writer);
+            }
+        },
+        .code_span, .text => {
+            const content = doc.string(data.text.content);
+            try writer.print("{}", .{fmtHtml(content)});
+        },
+        .line_break => {
+            try writer.writeAll("\n");
+        },
+    }
+}
+
+pub fn fmtHtml(bytes: []const u8) std.fmt.Formatter(formatHtml) {
+    return .{ .data = bytes };
+}
+
+fn formatHtml(
+    bytes: []const u8,
+    comptime fmt: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    _ = fmt;
+    _ = options;
+    for (bytes) |b| {
+        switch (b) {
+            '<' => try writer.writeAll("&lt;"),
+            '>' => try writer.writeAll("&gt;"),
+            '&' => try writer.writeAll("&amp;"),
+            '"' => try writer.writeAll("&quot;"),
+            else => try writer.writeByte(b),
+        }
+    }
+}