Merge pull request #4871 from boothby/issue4769

Address bugs when //, /// or //! are immediately followed by EOF

src/tokenizer.cpp

@@ -1494,9 +1494,17 @@ void tokenize(Buf *buf, Tokenization *out) {
             tokenize_error(&t, "unexpected EOF");
             break;
         case TokenizeStateLineComment:
+            break;
         case TokenizeStateSawSlash2:
+            cancel_token(&t);
+            break;
         case TokenizeStateSawSlash3:
+            set_token_id(&t, t.cur_tok, TokenIdDocComment);
+            end_token(&t);
+            break;
         case TokenizeStateSawSlashBang:
+            set_token_id(&t, t.cur_tok, TokenIdContainerDocComment);
+            end_token(&t);
             break;
     }
     if (t.state != TokenizeStateError) {

test/stage1/behavior.zig

@@ -41,6 +41,9 @@ comptime {
     _ = @import("behavior/bugs/3586.zig");
     _ = @import("behavior/bugs/3742.zig");
     _ = @import("behavior/bugs/4560.zig");
+    _ = @import("behavior/bugs/4769_a.zig");
+    _ = @import("behavior/bugs/4769_b.zig");
+    _ = @import("behavior/bugs/4769_c.zig");
     _ = @import("behavior/bugs/394.zig");
     _ = @import("behavior/bugs/421.zig");
     _ = @import("behavior/bugs/529.zig");

test/stage1/behavior/bugs/4769_a.zig

@@ -0,0 +1 @@
+//

test/stage1/behavior/bugs/4769_b.zig

@@ -0,0 +1 @@
+//!

test/stage1/behavior/bugs/4769_c.zig

@@ -0,0 +1 @@
+///