diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index c71e6ffe34d9b..51515976e4ee9 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -57,29 +57,42 @@ pub enum TokenKind { // Multi-char tokens: /// "// comment" LineComment { doc_style: Option }, + /// `/* block comment */` /// - /// Block comments can be recursive, so the sequence like `/* /* */` + /// Block comments can be recursive, so a sequence like `/* /* */` /// will not be considered terminated and will result in a parsing error. BlockComment { doc_style: Option, terminated: bool }, - /// Any whitespace characters sequence. + + /// Any whitespace character sequence. Whitespace, + /// "ident" or "continue" - /// At this step keywords are also considered identifiers. + /// + /// At this step, keywords are also considered identifiers. Ident, + /// Like the above, but containing invalid unicode codepoints. InvalidIdent, + /// "r#ident" RawIdent, - /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the + + /// An unknown prefix, like `foo#`, `foo'`, `foo"`. + /// + /// Note that only the /// prefix (`foo`) is included in the token, not the separator (which is /// lexed as its own distinct token). In Rust 2021 and later, reserved /// prefixes are reported as errors; in earlier editions, they result in a /// (allowed by default) lint, and are treated as regular identifier /// tokens. UnknownPrefix, - /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details. + + /// Examples: `"12_u8"`, `"1.0e-40"`, `b"123`. + /// + /// See [LiteralKind] for more details. Literal { kind: LiteralKind, suffix_start: u32 }, + /// "'a" Lifetime { starts_with_number: bool },