update autodocs web application to latest

upstream commit 1f921d540e1a8bb40839be30239019c820eb663d

after this branch is merged, ziglang/zig becomes the new repository for
this code.
This commit is contained in:
Andrew Kelley committed 2024-03-10 18:13:25 -07:00
1 parent 6b8c7540a8
commit b13a55db97
8 files changed
+1074 -330

No files matched your search

+50 -4
View File
@@ -111,8 +111,35 @@ pub fn categorize(decl: *const Decl) Walk.Category {
return decl.file.categorize_decl(decl.ast_node);
}
/// Looks up a direct child of `decl` by name.
pub fn get_child(decl: *const Decl, name: []const u8) ?Decl.Index {
switch (decl.categorize()) {
.alias => |aliasee| return aliasee.get().get_child(name),
.namespace => |node| {
const file = decl.file.get();
const scope = file.scopes.get(node) orelse return null;
const child_node = scope.get_child(name) orelse return null;
return file.node_decls.get(child_node);
},
else => return null,
}
}
/// Looks up a decl by name accessible in `decl`'s namespace.
pub fn lookup(decl: *const Decl, name: []const u8) ?Decl.Index {
const namespace_node = switch (decl.categorize()) {
.namespace => |node| node,
else => decl.parent.get().ast_node,
};
const file = decl.file.get();
const scope = file.scopes.get(namespace_node) orelse return null;
const resolved_node = scope.lookup(&file.ast, name) orelse return null;
return file.node_decls.get(resolved_node);
}
/// Appends the fully qualified name to `out`.
pub fn fqn(decl: *const Decl, out: *std.ArrayListUnmanaged(u8)) Oom!void {
try decl.reset_with_path(out);
try decl.append_path(out);
if (decl.parent != .none) {
try append_parent_ns(out, decl.parent);
try out.appendSlice(gpa, decl.extra_info().name);
@@ -123,9 +150,13 @@ pub fn fqn(decl: *const Decl, out: *std.ArrayListUnmanaged(u8)) Oom!void {
pub fn reset_with_path(decl: *const Decl, list: *std.ArrayListUnmanaged(u8)) Oom!void {
list.clearRetainingCapacity();
try append_path(decl, list);
}
// Prefer the package name alias.
for (Walk.packages.keys(), Walk.packages.values()) |pkg_name, pkg_file| {
pub fn append_path(decl: *const Decl, list: *std.ArrayListUnmanaged(u8)) Oom!void {
const start = list.items.len;
// Prefer the module name alias.
for (Walk.modules.keys(), Walk.modules.values()) |pkg_name, pkg_file| {
if (pkg_file == decl.file) {
try list.ensureUnusedCapacity(gpa, pkg_name.len + 1);
list.appendSliceAssumeCapacity(pkg_name);
@@ -137,7 +168,7 @@ pub fn reset_with_path(decl: *const Decl, list: *std.ArrayListUnmanaged(u8)) Oom
const file_path = decl.file.path();
try list.ensureUnusedCapacity(gpa, file_path.len + 1);
list.appendSliceAssumeCapacity(file_path);
for (list.items) |*byte| switch (byte.*) {
for (list.items[start..]) |*byte| switch (byte.*) {
'/' => byte.* = '.',
else => continue,
};
@@ -170,6 +201,21 @@ pub fn findFirstDocComment(ast: *const Ast, token: Ast.TokenIndex) Ast.TokenInde
return it;
}
/// Successively looks up each component.
pub fn find(search_string: []const u8) Decl.Index {
var path_components = std.mem.splitScalar(u8, search_string, '.');
const file = Walk.modules.get(path_components.first()) orelse return .none;
var current_decl_index = file.findRootDecl();
while (path_components.next()) |component| {
while (true) switch (current_decl_index.get().categorize()) {
.alias => |aliasee| current_decl_index = aliasee,
else => break,
};
current_decl_index = current_decl_index.get().get_child(component) orelse return .none;
}
return current_decl_index;
}
const Decl = @This();
const std = @import("std");
const Ast = std.zig.Ast;
+226 -32
View File
@@ -1,21 +1,26 @@
//! Find and annotate identifiers with links to their declarations.
pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
pub var decls: std.ArrayListUnmanaged(Decl) = .{};
pub var packages: std.StringArrayHashMapUnmanaged(File.Index) = .{};
pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};
arena: std.mem.Allocator,
file: File.Index,
/// keep in sync with "CAT_" constants in main.js
pub const Category = union(enum(u8)) {
namespace: Ast.Node.Index,
global_variable: Ast.Node.Index,
/// A function that has not been detected as returning a type.
function: Ast.Node.Index,
primitive: Ast.Node.Index,
error_set: Ast.Node.Index,
global_const: Ast.Node.Index,
alias: Decl.Index,
/// A primitive identifier that is also a type.
type,
/// Specifically it is the literal `type`.
type_type,
/// A function that returns a type.
type_function: Ast.Node.Index,
pub const Tag = @typeInfo(Category).Union.tag_type.?;
};
@@ -30,12 +35,23 @@ pub const File = struct {
node_decls: std.AutoArrayHashMapUnmanaged(Ast.Node.Index, Decl.Index) = .{},
/// Maps function declarations to doctests.
doctests: std.AutoArrayHashMapUnmanaged(Ast.Node.Index, Ast.Node.Index) = .{},
/// root node => its namespace scope
/// struct/union/enum/opaque decl node => its namespace scope
/// local var decl node => its local variable scope
scopes: std.AutoArrayHashMapUnmanaged(Ast.Node.Index, *Scope) = .{},
pub fn lookup_token(file: *File, token: Ast.TokenIndex) Decl.Index {
const decl_node = file.ident_decls.get(token) orelse return .none;
return file.node_decls.get(decl_node) orelse return .none;
}
pub fn field_count(file: *const File, node: Ast.Node.Index) u32 {
const scope = file.scopes.get(node) orelse return 0;
if (scope.tag != .namespace) return 0;
const namespace = @fieldParentPtr(Scope.Namespace, "base", scope);
return namespace.field_count;
}
pub const Index = enum(u32) {
_,
@@ -90,16 +106,41 @@ pub const File = struct {
.fn_proto_one,
.fn_proto_simple,
.fn_decl,
=> return .{ .function = node },
=> {
var buf: [1]Ast.Node.Index = undefined;
const full = ast.fullFnProto(&buf, node).?;
return categorize_func(file_index, node, full);
},
else => unreachable,
}
}
pub fn categorize_func(
file_index: File.Index,
node: Ast.Node.Index,
full: Ast.full.FnProto,
) Category {
return switch (categorize_expr(file_index, full.ast.return_type)) {
.namespace, .error_set, .type_type => .{ .type_function = node },
else => .{ .function = node },
};
}
pub fn categorize_expr_deep(file_index: File.Index, node: Ast.Node.Index) Category {
return switch (categorize_expr(file_index, node)) {
.alias => |aliasee| aliasee.get().categorize(),
else => |result| result,
};
}
pub fn categorize_expr(file_index: File.Index, node: Ast.Node.Index) Category {
const file = file_index.get();
const ast = file_index.get_ast();
const node_tags = ast.nodes.items(.tag);
const node_datas = ast.nodes.items(.data);
const main_tokens = ast.nodes.items(.main_token);
//log.debug("categorize_expr tag {s}", .{@tagName(node_tags[node])});
return switch (node_tags[node]) {
.container_decl,
.container_decl_trailing,
@@ -116,24 +157,43 @@ pub const File = struct {
=> .{ .namespace = node },
.error_set_decl,
.merge_error_sets,
=> .{ .error_set = node },
.identifier => {
const name_token = ast.nodes.items(.main_token)[node];
const ident_name = ast.tokenSlice(name_token);
if (std.zig.primitives.isPrimitive(ident_name)) {
return .{ .primitive = node };
}
if (std.mem.eql(u8, ident_name, "type"))
return .type_type;
const decl_index = file_index.get().lookup_token(name_token);
if (decl_index != .none) return .{ .alias = decl_index };
if (isPrimitiveNonType(ident_name))
return .{ .primitive = node };
if (std.zig.primitives.isPrimitive(ident_name))
return .type;
if (file.ident_decls.get(name_token)) |decl_node| {
const decl_index = file.node_decls.get(decl_node) orelse .none;
if (decl_index != .none) return .{ .alias = decl_index };
return categorize_decl(file_index, decl_node);
}
return .{ .global_const = node };
},
.field_access => {
// TODO:
//return .alias;
const object_node = node_datas[node].lhs;
const dot_token = main_tokens[node];
const field_ident = dot_token + 1;
const field_name = ast.tokenSlice(field_ident);
switch (categorize_expr(file_index, object_node)) {
.alias => |aliasee| if (aliasee.get().get_child(field_name)) |decl_index| {
return .{ .alias = decl_index };
},
else => {},
}
return .{ .global_const = node };
},
@@ -154,10 +214,77 @@ pub const File = struct {
return categorize_builtin_call(file_index, node, params);
},
.call_one,
.call_one_comma,
.async_call_one,
.async_call_one_comma,
.call,
.call_comma,
.async_call,
.async_call_comma,
=> {
var buf: [1]Ast.Node.Index = undefined;
return categorize_call(file_index, node, ast.fullCall(&buf, node).?);
},
.if_simple,
.@"if",
=> {
const if_full = ast.fullIf(node).?;
if (if_full.ast.else_expr != 0) {
const then_cat = categorize_expr_deep(file_index, if_full.ast.then_expr);
const else_cat = categorize_expr_deep(file_index, if_full.ast.else_expr);
if (then_cat == .type_type and else_cat == .type_type) {
return .type_type;
} else if (then_cat == .error_set and else_cat == .error_set) {
return .{ .error_set = node };
} else if (then_cat == .type or else_cat == .type or
then_cat == .namespace or else_cat == .namespace or
then_cat == .error_set or else_cat == .error_set or
then_cat == .type_function or else_cat == .type_function)
{
return .type;
}
}
return .{ .global_const = node };
},
.@"switch", .switch_comma => return categorize_switch(file_index, node),
.optional_type,
.array_type,
.array_type_sentinel,
.ptr_type_aligned,
.ptr_type_sentinel,
.ptr_type,
.ptr_type_bit_range,
.anyframe_type,
=> .type,
else => .{ .global_const = node },
};
}
fn categorize_call(
file_index: File.Index,
node: Ast.Node.Index,
call: Ast.full.Call,
) Category {
return switch (categorize_expr(file_index, call.ast.fn_expr)) {
.type_function => .type,
.alias => |aliasee| categorize_decl_as_callee(aliasee, node),
else => .{ .global_const = node },
};
}
fn categorize_decl_as_callee(decl_index: Decl.Index, call_node: Ast.Node.Index) Category {
return switch (decl_index.get().categorize()) {
.type_function => .type,
.alias => |aliasee| categorize_decl_as_callee(aliasee, call_node),
else => .{ .global_const = call_node },
};
}
fn categorize_builtin_call(
file_index: File.Index,
node: Ast.Node.Index,
@@ -172,6 +299,9 @@ pub const File = struct {
const str_bytes = ast.tokenSlice(str_lit_token);
const file_path = std.zig.string_literal.parseAlloc(gpa, str_bytes) catch @panic("OOM");
defer gpa.free(file_path);
if (modules.get(file_path)) |imported_file_index| {
return .{ .alias = File.Index.findRootDecl(imported_file_index) };
}
const base_path = file_index.path();
const resolved_path = std.fs.path.resolvePosix(gpa, &.{
base_path, "..", file_path,
@@ -180,8 +310,8 @@ pub const File = struct {
log.debug("from '{s}' @import '{s}' resolved='{s}'", .{
base_path, file_path, resolved_path,
});
if (Walk.files.getIndex(resolved_path)) |imported_file_index| {
return .{ .alias = Walk.File.Index.findRootDecl(@enumFromInt(imported_file_index)) };
if (files.getIndex(resolved_path)) |imported_file_index| {
return .{ .alias = File.Index.findRootDecl(@enumFromInt(imported_file_index)) };
} else {
log.warn("import target '{s}' did not resolve to any file", .{resolved_path});
}
@@ -195,10 +325,47 @@ pub const File = struct {
return .{ .global_const = node };
}
fn categorize_switch(file_index: File.Index, node: Ast.Node.Index) Category {
const ast = file_index.get_ast();
const node_datas = ast.nodes.items(.data);
const extra = ast.extraData(node_datas[node].rhs, Ast.Node.SubRange);
const case_nodes = ast.extra_data[extra.start..extra.end];
var all_type_type = true;
var all_error_set = true;
var any_type = false;
if (case_nodes.len == 0) return .{ .global_const = node };
for (case_nodes) |case_node| {
const case = ast.fullSwitchCase(case_node).?;
switch (categorize_expr_deep(file_index, case.ast.target_expr)) {
.type_type => {
any_type = true;
all_error_set = false;
},
.error_set => {
any_type = true;
all_type_type = false;
},
.type, .namespace, .type_function => {
any_type = true;
all_error_set = false;
all_type_type = false;
},
else => {
all_error_set = false;
all_type_type = false;
},
}
}
if (all_type_type) return .type_type;
if (all_error_set) return .{ .error_set = node };
if (any_type) return .type;
return .{ .global_const = node };
}
};
};
pub const PackageIndex = enum(u32) {
pub const ModuleIndex = enum(u32) {
_,
};
@@ -208,28 +375,25 @@ pub fn add_file(file_name: []const u8, bytes: []u8) !File.Index {
try files.put(gpa, file_name, .{ .ast = ast });
if (ast.errors.len > 0) {
// TODO: expose this in the UI
log.err("can't index '{s}' because it has syntax errors", .{file_index.path()});
return file_index;
}
var arena_instance = std.heap.ArenaAllocator.init(gpa);
defer arena_instance.deinit();
var w: Walk = .{
.arena = arena_instance.allocator(),
.file = file_index,
};
var scope: Scope = .{ .tag = .top };
const scope = try gpa.create(Scope);
scope.* = .{ .tag = .top };
const decl_index = try file_index.add_decl(0, .none);
try struct_decl(&w, &scope, decl_index, ast.containerDeclRoot());
try struct_decl(&w, scope, decl_index, 0, ast.containerDeclRoot());
const file = file_index.get();
shrinkToFit(&file.ident_decls);
shrinkToFit(&file.token_parents);
shrinkToFit(&file.node_decls);
shrinkToFit(&file.doctests);
shrinkToFit(&file.scopes);
return file_index;
}
@@ -250,7 +414,7 @@ fn parse(source: []u8) Oom!Ast {
return Ast.parse(gpa, adjusted_source, .zig);
}
const Scope = struct {
pub const Scope = struct {
tag: Tag,
const Tag = enum { top, local, namespace };
@@ -267,6 +431,7 @@ const Scope = struct {
names: std.StringArrayHashMapUnmanaged(Ast.Node.Index) = .{},
doctests: std.StringArrayHashMapUnmanaged(Ast.Node.Index) = .{},
decl_index: Decl.Index,
field_count: u32,
};
fn getNamespaceDecl(start_scope: *Scope) Decl.Index {
@@ -284,7 +449,17 @@ const Scope = struct {
};
}
fn lookup(start_scope: *Scope, ast: *const Ast, name: []const u8) ?Ast.Node.Index {
pub fn get_child(scope: *Scope, name: []const u8) ?Ast.Node.Index {
switch (scope.tag) {
.top, .local => return null,
.namespace => {
const namespace = @fieldParentPtr(Namespace, "base", scope);
return namespace.names.get(name);
},
}
}
pub fn lookup(start_scope: *Scope, ast: *const Ast, name: []const u8) ?Ast.Node.Index {
const main_tokens = ast.nodes.items(.main_token);
var it: *Scope = start_scope;
while (true) switch (it.tag) {
@@ -314,17 +489,21 @@ fn struct_decl(
w: *Walk,
scope: *Scope,
parent_decl: Decl.Index,
node: Ast.Node.Index,
container_decl: Ast.full.ContainerDecl,
) Oom!void {
const ast = w.file.get_ast();
const node_tags = ast.nodes.items(.tag);
const node_datas = ast.nodes.items(.data);
var namespace: Scope.Namespace = .{
const namespace = try gpa.create(Scope.Namespace);
namespace.* = .{
.parent = scope,
.decl_index = parent_decl,
.field_count = 0,
};
try w.scanDecls(&namespace, container_decl.ast.members);
try w.file.get().scopes.putNoClobber(gpa, node, &namespace.base);
try w.scanDecls(namespace, container_decl.ast.members);
for (container_decl.ast.members) |member| switch (node_tags[member]) {
.container_field_init,
@@ -584,7 +763,8 @@ fn expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, node: Ast.Node.Index)
=> {
const full = ast.fullAsm(node).?;
for (full.ast.items) |n| {
// TODO handle .asm_input, .asm_output
// There is a missing call here to expr() for .asm_input and
// .asm_output nodes.
_ = n;
}
try expr(w, scope, parent_decl, full.ast.template);
@@ -701,7 +881,7 @@ fn expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, node: Ast.Node.Index)
.tagged_union_two_trailing,
=> {
var buf: [2]Ast.Node.Index = undefined;
return struct_decl(w, scope, parent_decl, ast.fullContainerDecl(&buf, node).?);
return struct_decl(w, scope, parent_decl, node, ast.fullContainerDecl(&buf, node).?);
},
.array_type_sentinel => {
@@ -803,7 +983,6 @@ fn block(
statements: []const Ast.Node.Index,
) Oom!void {
const ast = w.file.get_ast();
const arena = w.arena;
const node_tags = ast.nodes.items(.tag);
const node_datas = ast.nodes.items(.data);
@@ -818,16 +997,17 @@ fn block(
=> {
const full = ast.fullVarDecl(node).?;
try global_var_decl(w, scope, parent_decl, full);
const local = try arena.create(Scope.Local);
const local = try gpa.create(Scope.Local);
local.* = .{
.parent = scope,
.var_node = node,
};
try w.file.get().scopes.putNoClobber(gpa, node, &local.base);
scope = &local.base;
},
.assign_destructure => {
// TODO
log.debug("walk assign_destructure not implemented yet", .{});
},
.grouped_expression => try expr(w, scope, parent_decl, node_datas[node].lhs),
@@ -849,7 +1029,6 @@ fn while_expr(w: *Walk, scope: *Scope, parent_decl: Decl.Index, full: Ast.full.W
}
fn scanDecls(w: *Walk, namespace: *Scope.Namespace, members: []const Ast.Node.Index) Oom!void {
const arena = w.arena;
const ast = w.file.get_ast();
const node_tags = ast.nodes.items(.tag);
const main_tokens = ast.nodes.items(.main_token);
@@ -880,19 +1059,34 @@ fn scanDecls(w: *Walk, namespace: *Scope.Namespace, members: []const Ast.Node.In
const is_doctest = token_tags[ident_token] == .identifier;
if (is_doctest) {
const token_bytes = ast.tokenSlice(ident_token);
try namespace.doctests.put(arena, token_bytes, member_node);
try namespace.doctests.put(gpa, token_bytes, member_node);
}
continue;
},
.container_field_init,
.container_field_align,
.container_field,
=> {
namespace.field_count += 1;
continue;
},
else => continue,
};
const token_bytes = ast.tokenSlice(name_token);
try namespace.names.put(arena, token_bytes, member_node);
try namespace.names.put(gpa, token_bytes, member_node);
}
}
pub fn isPrimitiveNonType(name: []const u8) bool {
return std.mem.eql(u8, name, "undefined") or
std.mem.eql(u8, name, "null") or
std.mem.eql(u8, name, "true") or
std.mem.eql(u8, name, "false");
}
//test {
// const gpa = std.testing.allocator;
//
+387 -48
View File
@@ -1,3 +1,6 @@
/// Delete this to find out where URL escaping needs to be added.
const missing_feature_url_escape = true;
const gpa = std.heap.wasm_allocator;
const std = @import("std");
@@ -80,7 +83,7 @@ export fn query_exec(ignore_case: bool) [*]Decl.Index {
return query_results.items.ptr;
}
const max_matched_items = 2000;
const max_matched_items = 1000;
fn query_exec_fallible(query: []const u8, ignore_case: bool) !void {
const Score = packed struct(u32) {
@@ -181,7 +184,7 @@ fn query_exec_fallible(query: []const u8, ignore_case: bool) !void {
const b_decl = query_results.items[b_index];
const a_file_path = a_decl.get().file.path();
const b_file_path = b_decl.get().file.path();
// TODO Also check the local namespace inside the file
// This neglects to check the local namespace inside the file.
return std.mem.lessThan(u8, b_file_path, a_file_path);
}
}
@@ -209,12 +212,178 @@ fn Slice(T: type) type {
};
}
const ErrorIdentifier = packed struct(u64) {
token_index: Ast.TokenIndex,
decl_index: Decl.Index,
fn hasDocs(ei: ErrorIdentifier) bool {
const decl_index = ei.decl_index;
const ast = decl_index.get().file.get_ast();
const token_tags = ast.tokens.items(.tag);
const token_index = ei.token_index;
if (token_index == 0) return false;
return token_tags[token_index - 1] == .doc_comment;
}
fn html(ei: ErrorIdentifier, base_decl: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
const decl_index = ei.decl_index;
const ast = decl_index.get().file.get_ast();
const name = ast.tokenSlice(ei.token_index);
const first_doc_comment = Decl.findFirstDocComment(ast, ei.token_index);
const has_docs = ast.tokens.items(.tag)[first_doc_comment] == .doc_comment;
const has_link = base_decl != decl_index;
try out.appendSlice(gpa, "<dt>");
try out.appendSlice(gpa, name);
if (has_link) {
try out.appendSlice(gpa, " <a href=\"#");
_ = missing_feature_url_escape;
try decl_index.get().fqn(out);
try out.appendSlice(gpa, "\">");
try out.appendSlice(gpa, decl_index.get().extra_info().name);
try out.appendSlice(gpa, "</a>");
}
try out.appendSlice(gpa, "</dt>");
if (has_docs) {
try out.appendSlice(gpa, "<dd>");
try render_docs(out, decl_index, first_doc_comment, false);
try out.appendSlice(gpa, "</dd>");
}
}
};
var string_result: std.ArrayListUnmanaged(u8) = .{};
var error_set_result: std.StringArrayHashMapUnmanaged(ErrorIdentifier) = .{};
export fn decl_error_set(decl_index: Decl.Index) Slice(ErrorIdentifier) {
return Slice(ErrorIdentifier).init(decl_error_set_fallible(decl_index) catch @panic("OOM"));
}
export fn error_set_node_list(base_decl: Decl.Index, node: Ast.Node.Index) Slice(ErrorIdentifier) {
error_set_result.clearRetainingCapacity();
addErrorsFromExpr(base_decl, &error_set_result, node) catch @panic("OOM");
sort_error_set_result();
return Slice(ErrorIdentifier).init(error_set_result.values());
}
export fn fn_error_set_decl(decl_index: Decl.Index, node: Ast.Node.Index) Decl.Index {
return switch (decl_index.get().file.categorize_expr(node)) {
.alias => |aliasee| fn_error_set_decl(aliasee, aliasee.get().ast_node),
else => decl_index,
};
}
export fn decl_field_count(decl_index: Decl.Index) u32 {
switch (decl_index.get().categorize()) {
.namespace => |node| return decl_index.get().file.get().field_count(node),
else => return 0,
}
}
fn decl_error_set_fallible(decl_index: Decl.Index) Oom![]ErrorIdentifier {
error_set_result.clearRetainingCapacity();
try addErrorsFromDecl(decl_index, &error_set_result);
sort_error_set_result();
return error_set_result.values();
}
fn sort_error_set_result() void {
const sort_context: struct {
pub fn lessThan(sc: @This(), a_index: usize, b_index: usize) bool {
_ = sc;
const a_name = error_set_result.keys()[a_index];
const b_name = error_set_result.keys()[b_index];
return std.mem.lessThan(u8, a_name, b_name);
}
} = .{};
error_set_result.sortUnstable(sort_context);
}
fn addErrorsFromDecl(
decl_index: Decl.Index,
out: *std.StringArrayHashMapUnmanaged(ErrorIdentifier),
) Oom!void {
switch (decl_index.get().categorize()) {
.error_set => |node| try addErrorsFromExpr(decl_index, out, node),
.alias => |aliasee| try addErrorsFromDecl(aliasee, out),
else => |cat| log.debug("unable to addErrorsFromDecl: {any}", .{cat}),
}
}
fn addErrorsFromExpr(
decl_index: Decl.Index,
out: *std.StringArrayHashMapUnmanaged(ErrorIdentifier),
node: Ast.Node.Index,
) Oom!void {
const decl = decl_index.get();
const ast = decl.file.get_ast();
const node_tags = ast.nodes.items(.tag);
const node_datas = ast.nodes.items(.data);
switch (decl.file.categorize_expr(node)) {
.error_set => |n| switch (node_tags[n]) {
.error_set_decl => {
try addErrorsFromNode(decl_index, out, node);
},
.merge_error_sets => {
try addErrorsFromExpr(decl_index, out, node_datas[node].lhs);
try addErrorsFromExpr(decl_index, out, node_datas[node].rhs);
},
else => unreachable,
},
.alias => |aliasee| {
try addErrorsFromDecl(aliasee, out);
},
else => return,
}
}
fn addErrorsFromNode(
decl_index: Decl.Index,
out: *std.StringArrayHashMapUnmanaged(ErrorIdentifier),
node: Ast.Node.Index,
) Oom!void {
const decl = decl_index.get();
const ast = decl.file.get_ast();
const main_tokens = ast.nodes.items(.main_token);
const token_tags = ast.tokens.items(.tag);
const error_token = main_tokens[node];
var tok_i = error_token + 2;
while (true) : (tok_i += 1) switch (token_tags[tok_i]) {
.doc_comment, .comma => {},
.identifier => {
const name = ast.tokenSlice(tok_i);
const gop = try out.getOrPut(gpa, name);
// If there are more than one, take the one with doc comments.
// If they both have doc comments, prefer the existing one.
const new: ErrorIdentifier = .{
.token_index = tok_i,
.decl_index = decl_index,
};
if (!gop.found_existing or
(!gop.value_ptr.hasDocs() and new.hasDocs()))
{
gop.value_ptr.* = new;
}
},
.r_brace => break,
else => unreachable,
};
}
export fn type_fn_fields(decl_index: Decl.Index) Slice(Ast.Node.Index) {
return decl_fields(decl_index);
}
export fn decl_fields(decl_index: Decl.Index) Slice(Ast.Node.Index) {
return Slice(Ast.Node.Index).init(decl_fields_fallible(decl_index) catch @panic("OOM"));
}
export fn decl_params(decl_index: Decl.Index) Slice(Ast.Node.Index) {
return Slice(Ast.Node.Index).init(decl_params_fallible(decl_index) catch @panic("OOM"));
}
fn decl_fields_fallible(decl_index: Decl.Index) ![]Ast.Node.Index {
const g = struct {
var result: std.ArrayListUnmanaged(Ast.Node.Index) = .{};
@@ -237,12 +406,38 @@ fn decl_fields_fallible(decl_index: Decl.Index) ![]Ast.Node.Index {
return g.result.items;
}
fn decl_params_fallible(decl_index: Decl.Index) ![]Ast.Node.Index {
const g = struct {
var result: std.ArrayListUnmanaged(Ast.Node.Index) = .{};
};
g.result.clearRetainingCapacity();
const decl = decl_index.get();
const ast = decl.file.get_ast();
const value_node = decl.value_node() orelse return &.{};
var buf: [1]Ast.Node.Index = undefined;
const fn_proto = ast.fullFnProto(&buf, value_node) orelse return &.{};
try g.result.appendSlice(gpa, fn_proto.ast.params);
return g.result.items;
}
export fn error_html(base_decl: Decl.Index, error_identifier: ErrorIdentifier) String {
string_result.clearRetainingCapacity();
error_identifier.html(base_decl, &string_result) catch @panic("OOM");
return String.init(string_result.items);
}
export fn decl_field_html(decl_index: Decl.Index, field_node: Ast.Node.Index) String {
string_result.clearRetainingCapacity();
decl_field_html_fallible(&string_result, decl_index, field_node) catch @panic("OOM");
return String.init(string_result.items);
}
export fn decl_param_html(decl_index: Decl.Index, param_node: Ast.Node.Index) String {
string_result.clearRetainingCapacity();
decl_param_html_fallible(&string_result, decl_index, param_node) catch @panic("OOM");
return String.init(string_result.items);
}
fn decl_field_html_fallible(
out: *std.ArrayListUnmanaged(u8),
decl_index: Decl.Index,
@@ -251,7 +446,7 @@ fn decl_field_html_fallible(
const decl = decl_index.get();
const ast = decl.file.get_ast();
try out.appendSlice(gpa, "<pre><code>");
try file_source_html(decl.file, out, field_node);
try file_source_html(decl.file, out, field_node, .{});
try out.appendSlice(gpa, "</code></pre>");
const field = ast.fullContainerField(field_node).?;
@@ -259,12 +454,48 @@ fn decl_field_html_fallible(
if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
try out.appendSlice(gpa, "<div class=\"fieldDocs\">");
try render_docs(out, ast, first_doc_comment, false);
try render_docs(out, decl_index, first_doc_comment, false);
try out.appendSlice(gpa, "</div>");
}
}
export fn decl_fn_proto_html(decl_index: Decl.Index) String {
fn decl_param_html_fallible(
out: *std.ArrayListUnmanaged(u8),
decl_index: Decl.Index,
param_node: Ast.Node.Index,
) !void {
const decl = decl_index.get();
const ast = decl.file.get_ast();
const token_tags = ast.tokens.items(.tag);
const colon = ast.firstToken(param_node) - 1;
const name_token = colon - 1;
const first_doc_comment = f: {
var it = ast.firstToken(param_node);
while (it > 0) {
it -= 1;
switch (token_tags[it]) {
.doc_comment, .colon, .identifier, .keyword_comptime, .keyword_noalias => {},
else => break,
}
}
break :f it + 1;
};
const name = ast.tokenSlice(name_token);
try out.appendSlice(gpa, "<pre><code>");
try appendEscaped(out, name);
try out.appendSlice(gpa, ": ");
try file_source_html(decl.file, out, param_node, .{});
try out.appendSlice(gpa, "</code></pre>");
if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
try out.appendSlice(gpa, "<div class=\"fieldDocs\">");
try render_docs(out, decl_index, first_doc_comment, false);
try out.appendSlice(gpa, "</div>");
}
}
export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) String {
const decl = decl_index.get();
const ast = decl.file.get_ast();
const node_tags = ast.nodes.items(.tag);
@@ -282,7 +513,12 @@ export fn decl_fn_proto_html(decl_index: Decl.Index) String {
};
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, proto_node) catch |err| {
file_source_html(decl.file, &string_result, proto_node, .{
.skip_doc_comments = true,
.skip_comments = true,
.collapse_whitespace = true,
.fn_link = if (linkify_fn_name) decl_index else .none,
}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
@@ -292,7 +528,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {
const decl = decl_index.get();
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, decl.ast_node) catch |err| {
file_source_html(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
@@ -304,7 +540,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
return String.init("");
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, doctest_ast_node) catch |err| {
file_source_html(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
@@ -312,6 +548,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
export fn decl_fqn(decl_index: Decl.Index) String {
const decl = decl_index.get();
string_result.clearRetainingCapacity();
decl.fqn(&string_result) catch @panic("OOM");
return String.init(string_result.items);
}
@@ -321,6 +558,20 @@ export fn decl_parent(decl_index: Decl.Index) Decl.Index {
return decl.parent;
}
export fn fn_error_set(decl_index: Decl.Index) Ast.Node.Index {
const decl = decl_index.get();
const ast = decl.file.get_ast();
var buf: [1]Ast.Node.Index = undefined;
const full = ast.fullFnProto(&buf, decl.ast_node).?;
const node_tags = ast.nodes.items(.tag);
const node_datas = ast.nodes.items(.data);
return switch (node_tags[full.ast.return_type]) {
.error_set_decl => full.ast.return_type,
.error_union => node_datas[full.ast.return_type].lhs,
else => 0,
};
}
export fn decl_file_path(decl_index: Decl.Index) String {
string_result.clearRetainingCapacity();
string_result.appendSlice(gpa, decl_index.get().file.path()) catch @panic("OOM");
@@ -350,7 +601,8 @@ export fn decl_category_name(decl_index: Decl.Index) String {
},
.global_variable => "Global Variable",
.function => "Function",
.type => "Type",
.type_function => "Type Function",
.type, .type_type => "Type",
.error_set => "Error Set",
.global_const => "Constant",
.primitive => "Primitive Value",
@@ -375,9 +627,8 @@ export fn decl_name(decl_index: Decl.Index) String {
export fn decl_docs_html(decl_index: Decl.Index, short: bool) String {
const decl = decl_index.get();
const ast = decl.file.get_ast();
string_result.clearRetainingCapacity();
render_docs(&string_result, ast, decl.extra_info().first_doc_comment, short) catch @panic("OOM");
render_docs(&string_result, decl_index, decl.extra_info().first_doc_comment, short) catch @panic("OOM");
return String.init(string_result.items);
}
@@ -402,10 +653,12 @@ fn collect_docs(
fn render_docs(
out: *std.ArrayListUnmanaged(u8),
ast: *const Ast,
decl_index: Decl.Index,
first_doc_comment: Ast.TokenIndex,
short: bool,
) Oom!void {
const decl = decl_index.get();
const ast = decl.file.get_ast();
const token_tags = ast.tokens.items(.tag);
var parser = try markdown.Parser.init(gpa);
@@ -423,10 +676,14 @@ fn render_docs(
var parsed_doc = try parser.endInput();
defer parsed_doc.deinit(gpa);
const g = struct {
var link_buffer: std.ArrayListUnmanaged(u8) = .{};
};
const Writer = std.ArrayListUnmanaged(u8).Writer;
const Renderer = markdown.Renderer(Writer, void);
const Renderer = markdown.Renderer(Writer, Decl.Index);
const renderer: Renderer = .{
.context = {},
.context = decl_index,
.renderFn = struct {
fn render(
r: Renderer,
@@ -436,23 +693,22 @@ fn render_docs(
) !void {
const data = doc.nodes.items(.data)[@intFromEnum(node)];
switch (doc.nodes.items(.tag)[@intFromEnum(node)]) {
// TODO: detect identifier references (dotted paths) in
// these three node types and render them appropriately.
// Also, syntax highlighting can be applied in code blocks
// unless the tag says otherwise.
.code_block => {
const tag = doc.string(data.code_block.tag);
_ = tag;
const content = doc.string(data.code_block.content);
try writer.print("<pre><code>{}</code></pre>\n", .{markdown.fmtHtml(content)});
},
.code_span => {
try writer.writeAll("<code>");
const content = doc.string(data.text.content);
try writer.print("<code>{}</code>", .{markdown.fmtHtml(content)});
},
.text => {
const content = doc.string(data.text.content);
try writer.print("{}", .{markdown.fmtHtml(content)});
if (resolve_decl_path(r.context, content)) |resolved_decl_index| {
g.link_buffer.clearRetainingCapacity();
try resolve_decl_link(resolved_decl_index, &g.link_buffer);
try writer.writeAll("<a href=\"#");
_ = missing_feature_url_escape;
try writer.writeAll(g.link_buffer.items);
try writer.print("\">{}</a>", .{markdown.fmtHtml(content)});
} else {
try writer.print("{}", .{markdown.fmtHtml(content)});
}
try writer.writeAll("</code>");
},
else => try Renderer.renderDefault(r, doc, node, writer),
@@ -463,12 +719,39 @@ fn render_docs(
try renderer.render(parsed_doc, out.writer(gpa));
}
fn resolve_decl_path(decl_index: Decl.Index, path: []const u8) ?Decl.Index {
var path_components = std.mem.splitScalar(u8, path, '.');
var current_decl_index = decl_index.get().lookup(path_components.first()) orelse return null;
while (path_components.next()) |component| {
switch (current_decl_index.get().categorize()) {
.alias => |aliasee| current_decl_index = aliasee,
else => {},
}
current_decl_index = current_decl_index.get().get_child(component) orelse return null;
}
return current_decl_index;
}
export fn decl_type_html(decl_index: Decl.Index) String {
const decl = decl_index.get();
const ast = decl.file.get_ast();
string_result.clearRetainingCapacity();
_ = ast; // TODO
string_result.appendSlice(gpa, "TODO_type_here") catch @panic("OOM");
t: {
// If there is an explicit type, use it.
if (ast.fullVarDecl(decl.ast_node)) |var_decl| {
if (var_decl.ast.type_node != 0) {
string_result.appendSlice(gpa, "<code>") catch @panic("OOM");
file_source_html(decl.file, &string_result, var_decl.ast.type_node, .{
.skip_comments = true,
.collapse_whitespace = true,
}) catch |e| {
fatal("unable to render html: {s}", .{@errorName(e)});
};
string_result.appendSlice(gpa, "</code>") catch @panic("OOM");
break :t;
}
}
}
return String.init(string_result.items);
}
@@ -491,7 +774,7 @@ fn unpack_inner(tar_bytes: []u8) !void {
const file_name = try gpa.dupe(u8, tar_file.name);
if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| {
const pkg_name = file_name[0..pkg_name_end];
const gop = try Walk.packages.getOrPut(gpa, pkg_name);
const gop = try Walk.modules.getOrPut(gpa, pkg_name);
const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len);
if (!gop.found_existing or
std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or
@@ -527,13 +810,13 @@ fn ascii_lower(bytes: []u8) void {
for (bytes) |*b| b.* = std.ascii.toLower(b.*);
}
export fn package_name(index: u32) String {
const names = Walk.packages.keys();
export fn module_name(index: u32) String {
const names = Walk.modules.keys();
return String.init(if (index >= names.len) "" else names[index]);
}
export fn find_package_root(pkg: Walk.PackageIndex) Decl.Index {
const root_file = Walk.packages.values()[@intFromEnum(pkg)];
export fn find_module_root(pkg: Walk.ModuleIndex) Decl.Index {
const root_file = Walk.modules.values()[@intFromEnum(pkg)];
const result = root_file.findRootDecl();
assert(result != .none);
return result;
@@ -555,11 +838,17 @@ export fn find_file_root() Decl.Index {
}
/// Uses `input_string`.
/// Tries to look up the Decl component-wise but then falls back to a file path
/// based scan.
export fn find_decl() Decl.Index {
const result = Decl.find(input_string.items);
if (result != .none) return result;
const g = struct {
var match_fqn: std.ArrayListUnmanaged(u8) = .{};
};
for (Walk.decls.items, 0..) |*decl, decl_index| {
g.match_fqn.clearRetainingCapacity();
decl.fqn(&g.match_fqn) catch @panic("OOM");
if (std.mem.eql(u8, g.match_fqn.items, input_string.items)) {
//const path = @as(Decl.Index, @enumFromInt(decl_index)).get().file.path();
@@ -583,7 +872,7 @@ export fn categorize_decl(decl_index: Decl.Index, resolve_alias_count: usize) Wa
var decl = decl_index.get();
while (true) {
const result = decl.categorize();
switch (decl.categorize()) {
switch (result) {
.alias => |new_index| {
assert(new_index != .none);
global_aliasee = new_index;
@@ -599,6 +888,10 @@ export fn categorize_decl(decl_index: Decl.Index, resolve_alias_count: usize) Wa
}
}
export fn type_fn_members(parent: Decl.Index, include_private: bool) Slice(Decl.Index) {
return namespace_members(parent, include_private);
}
export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Decl.Index) {
const g = struct {
var members: std.ArrayListUnmanaged(Decl.Index) = .{};
@@ -617,10 +910,18 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec
return Slice(Decl.Index).init(g.members.items);
}
const RenderSourceOptions = struct {
skip_doc_comments: bool = false,
skip_comments: bool = false,
collapse_whitespace: bool = false,
fn_link: Decl.Index = .none,
};
fn file_source_html(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
root_node: Ast.Node.Index,
options: RenderSourceOptions,
) !void {
const ast = file_index.get_ast();
const file = file_index.get();
@@ -631,6 +932,7 @@ fn file_source_html(
const token_tags = ast.tokens.items(.tag);
const token_starts = ast.tokens.items(.start);
const main_tokens = ast.nodes.items(.main_token);
const start_token = ast.firstToken(root_node);
const end_token = ast.lastToken(root_node) + 1;
@@ -643,7 +945,20 @@ fn file_source_html(
start_token..,
) |tag, start, token_index| {
const between = ast.source[cursor..start];
try appendEscaped(out, between);
if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
if (!options.skip_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendEscaped(out, between);
try out.appendSlice(gpa, "</span>");
}
} else if (between.len > 0) {
if (options.collapse_whitespace) {
if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
try out.append(gpa, ' ');
} else {
try out.appendSlice(gpa, between);
}
}
if (tag == .eof) break;
const slice = ast.tokenSlice(token_index);
cursor = start + slice.len;
@@ -723,17 +1038,36 @@ fn file_source_html(
.doc_comment,
.container_doc_comment,
=> {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
if (!options.skip_doc_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
}
},
.identifier => i: {
if (std.mem.eql(u8, slice, "undefined") or
std.mem.eql(u8, slice, "null") or
std.mem.eql(u8, slice, "true") or
std.mem.eql(u8, slice, "false"))
{
if (options.fn_link != .none) {
const fn_link = options.fn_link.get();
const fn_token = main_tokens[fn_link.ast_node];
if (token_index == fn_token + 1) {
try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
_ = missing_feature_url_escape;
try fn_link.fqn(out);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
break :i;
}
}
if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
try out.appendSlice(gpa, "<span class=\"tok-fn\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (Walk.isPrimitiveNonType(slice)) {
try out.appendSlice(gpa, "<span class=\"tok-null\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
@@ -752,7 +1086,8 @@ fn file_source_html(
try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
try out.appendSlice(gpa, g.field_access_buffer.items); // TODO url escape
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
@@ -767,7 +1102,8 @@ fn file_source_html(
try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
try out.appendSlice(gpa, g.field_access_buffer.items); // TODO url escape
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
@@ -860,7 +1196,10 @@ fn resolve_ident_link(
) Oom!void {
const decl_index = file_index.get().lookup_token(ident_token);
if (decl_index == .none) return;
try resolve_decl_link(decl_index, out);
}
fn resolve_decl_link(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
const decl = decl_index.get();
switch (decl.categorize()) {
.alias => |alias_decl| try alias_decl.get().fqn(out),
+8 -1
View File
@@ -517,6 +517,10 @@ test "tables require leading and trailing pipes" {
\\
\\| But | this | is |
\\
\\Also not a table:
\\|
\\ |
\\
,
\\<p>Not | a | table</p>
\\<table>
@@ -526,6 +530,9 @@ test "tables require leading and trailing pipes" {
\\<td>is</td>
\\</tr>
\\</table>
\\<p>Also not a table:
\\|
\\|</p>
\\
);
}
@@ -584,7 +591,7 @@ test "code blocks" {
\\<pre><code>Hello, world!
\\This is some code.
\\</code></pre>
\\<pre><code class="zig test">const std = @import(&quot;std&quot;);
\\<pre><code>const std = @import(&quot;std&quot;);
\\
\\test {
\\ try std.testing.expect(2 + 2 == 4);
+2 -1
View File
@@ -610,7 +610,8 @@ const TableRowStart = struct {
};
fn startTableRow(unindented_line: []const u8) ?TableRowStart {
if (!mem.startsWith(u8, unindented_line, "|") or
if (unindented_line.len < 2 or
!mem.startsWith(u8, unindented_line, "|") or
mem.endsWith(u8, unindented_line, "\\|") or
!mem.endsWith(u8, unindented_line, "|")) return null;
+1 -6
View File
@@ -112,13 +112,8 @@ pub fn Renderer(comptime Writer: type, comptime Context: type) type {
try writer.print("</h{}>\n", .{data.heading.level});
},
.code_block => {
const tag = doc.string(data.code_block.tag);
const content = doc.string(data.code_block.content);
if (tag.len > 0) {
try writer.print("<pre><code class=\"{}\">{}</code></pre>\n", .{ fmtHtml(tag), fmtHtml(content) });
} else {
try writer.print("<pre><code>{}</code></pre>\n", .{fmtHtml(content)});
}
try writer.print("<pre><code>{}</code></pre>\n", .{fmtHtml(content)});
},
.blockquote => {
try writer.writeAll("<blockquote>\n");