diff --git a/logos-codegen/src/generator/context.rs b/logos-codegen/src/generator/context.rs index 515785ee..82f76f39 100644 --- a/logos-codegen/src/generator/context.rs +++ b/logos-codegen/src/generator/context.rs @@ -59,21 +59,12 @@ impl Context { self.available.saturating_sub(self.at) } - pub fn read_unchecked(&mut self, len: usize) -> TokenStream { + pub fn read_byte_unchecked(&mut self) -> TokenStream { let at = self.at; - match len { - 0 => { - self.advance(1); + self.advance(1); - quote!(lex.read_unchecked::(#at)) - } - l => { - self.advance(l); - - quote!(lex.read_unchecked::<&[u8; #l]>(#at)) - } - } + quote!(lex.read_byte_unchecked(#at)) } pub fn read(&mut self, len: usize) -> TokenStream { diff --git a/logos-codegen/src/generator/fork.rs b/logos-codegen/src/generator/fork.rs index 0a96056b..a0631f93 100644 --- a/logos-codegen/src/generator/fork.rs +++ b/logos-codegen/src/generator/fork.rs @@ -127,7 +127,7 @@ impl<'a> Generator<'a> { let min_read = self.meta[this].min_read; if ctx.remainder() >= max(min_read, 1) { - let read = ctx.read_unchecked(0); + let read = ctx.read_byte_unchecked(); return (quote!(byte), quote!(let byte = unsafe { #read };)); } diff --git a/src/internal.rs b/src/internal.rs index 0dae92e8..2957fede 100644 --- a/src/internal.rs +++ b/src/internal.rs @@ -16,15 +16,12 @@ pub trait LexerInternal<'source> { /// Read a chunk at current position, offset by `n`. fn read_at>(&self, n: usize) -> Option; - /// Unchecked read a chunk at current position, offset by `n`. - unsafe fn read_unchecked>(&self, n: usize) -> T; + /// Unchecked read a byte at current position, offset by `n`. + unsafe fn read_byte_unchecked(&self, n: usize) -> u8; /// Test a chunk at current position with a closure. fn test, F: FnOnce(T) -> bool>(&self, test: F) -> bool; - /// Test a chunk at current position offset by `n` with a closure. - fn test_at, F: FnOnce(T) -> bool>(&self, n: usize, test: F) -> bool; - /// Bump the position by `size`. fn bump_unchecked(&mut self, size: usize); diff --git a/src/lexer.rs b/src/lexer.rs index ec93bcd9..f5b12b2d 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -302,11 +302,8 @@ where } #[inline] - unsafe fn read_unchecked(&self, n: usize) -> Chunk - where - Chunk: source::Chunk<'source>, - { - self.source.read_unchecked(self.token_end + n) + unsafe fn read_byte_unchecked(&self, n: usize) -> u8 { + self.source.read_byte_unchecked(self.token_end + n) } /// Test a chunk at current position with a closure. @@ -322,19 +319,6 @@ where } } - /// Test a chunk at current position offset by `n` with a closure. - #[inline] - fn test_at(&self, n: usize, test: F) -> bool - where - T: source::Chunk<'source>, - F: FnOnce(T) -> bool, - { - match self.source.read::(self.token_end + n) { - Some(chunk) => test(chunk), - None => false, - } - } - /// Bump the position `Lexer` is reading from by `size`. #[inline] fn bump_unchecked(&mut self, size: usize) { diff --git a/src/source.rs b/src/source.rs index 7ba5a16b..5ada04d1 100644 --- a/src/source.rs +++ b/src/source.rs @@ -43,14 +43,12 @@ pub trait Source { where Chunk: self::Chunk<'a>; - /// Read a chunk of bytes into an array without doing bounds checks. + /// Read a byte without doing bounds checks. /// /// # Safety /// /// Offset should not exceed bounds. - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>; + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8; /// Get a slice of the source at given range. This is analogous to /// `slice::get(range)`. @@ -119,10 +117,7 @@ impl Source for str { } #[inline] - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>, - { + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { Chunk::from_ptr(self.as_ptr().add(offset)) } @@ -179,10 +174,7 @@ impl Source for [u8] { } #[inline] - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>, - { + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { Chunk::from_ptr(self.as_ptr().add(offset)) } @@ -232,11 +224,8 @@ where self.deref().read(offset) } - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: self::Chunk<'a>, - { - self.deref().read_unchecked(offset) + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { + self.deref().read_byte_unchecked(offset) } fn slice(&self, range: Range) -> Option> { diff --git a/tests/tests/source.rs b/tests/tests/source.rs index a0024a53..c747a354 100644 --- a/tests/tests/source.rs +++ b/tests/tests/source.rs @@ -19,11 +19,8 @@ impl<'s, S: ?Sized + Source> Source for RefSource<'s, S> { self.0.read(offset) } - unsafe fn read_unchecked<'a, Chunk>(&'a self, offset: usize) -> Chunk - where - Chunk: logos::source::Chunk<'a>, - { - self.0.read_unchecked(offset) + unsafe fn read_byte_unchecked(&self, offset: usize) -> u8 { + self.0.read_byte_unchecked(offset) } fn slice(&self, range: Range) -> Option> {