fallback.rs

   1// SPDX-License-Identifier: Apache-2.0 OR MIT
   2
   3#[cfg(wrap_proc_macro)]
   4use crate::imp;
   5#[cfg(span_locations)]
   6use crate::location::LineColumn;
   7use crate::parse::{self, Cursor};
   8use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
   9use crate::{Delimiter, Spacing, TokenTree};
  10#[cfg(all(span_locations, not(fuzzing)))]
  11use alloc::collections::BTreeMap;
  12#[cfg(all(span_locations, not(fuzzing)))]
  13use core::cell::RefCell;
  14#[cfg(span_locations)]
  15use core::cmp;
  16#[cfg(all(span_locations, not(fuzzing)))]
  17use core::cmp::Ordering;
  18use core::fmt::{self, Debug, Display, Write};
  19use core::mem::ManuallyDrop;
  20#[cfg(span_locations)]
  21use core::ops::Range;
  22use core::ops::RangeBounds;
  23use core::ptr;
  24use core::str;
  25#[cfg(feature = "proc-macro")]
  26use core::str::FromStr;
  27use std::ffi::CStr;
  28#[cfg(wrap_proc_macro)]
  29use std::panic;
  30#[cfg(span_locations)]
  31use std::path::PathBuf;
  32
  33/// Force use of proc-macro2's fallback implementation of the API for now, even
  34/// if the compiler's implementation is available.
  35pub fn force() {
  36    #[cfg(wrap_proc_macro)]
  37    crate::detection::force_fallback();
  38}
  39
  40/// Resume using the compiler's implementation of the proc macro API if it is
  41/// available.
  42pub fn unforce() {
  43    #[cfg(wrap_proc_macro)]
  44    crate::detection::unforce_fallback();
  45}
  46
  47#[derive(Clone)]
  48pub(crate) struct TokenStream {
  49    inner: RcVec<TokenTree>,
  50}
  51
  52#[derive(Debug)]
  53pub(crate) struct LexError {
  54    pub(crate) span: Span,
  55}
  56
  57impl LexError {
  58    pub(crate) fn span(&self) -> Span {
  59        self.span
  60    }
  61
  62    pub(crate) fn call_site() -> Self {
  63        LexError {
  64            span: Span::call_site(),
  65        }
  66    }
  67}
  68
  69impl TokenStream {
  70    pub(crate) fn new() -> Self {
  71        TokenStream {
  72            inner: RcVecBuilder::new().build(),
  73        }
  74    }
  75
  76    pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
  77        // Create a dummy file & add it to the source map
  78        let mut cursor = get_cursor(src);
  79
  80        // Strip a byte order mark if present
  81        const BYTE_ORDER_MARK: &str = "\u{feff}";
  82        if cursor.starts_with(BYTE_ORDER_MARK) {
  83            cursor = cursor.advance(BYTE_ORDER_MARK.len());
  84        }
  85
  86        parse::token_stream(cursor)
  87    }
  88
  89    #[cfg(feature = "proc-macro")]
  90    pub(crate) fn from_str_unchecked(src: &str) -> Self {
  91        Self::from_str_checked(src).unwrap()
  92    }
  93
  94    pub(crate) fn is_empty(&self) -> bool {
  95        self.inner.len() == 0
  96    }
  97
  98    fn take_inner(self) -> RcVecBuilder<TokenTree> {
  99        let nodrop = ManuallyDrop::new(self);
 100        unsafe { ptr::read(&nodrop.inner) }.make_owned()
 101    }
 102}
 103
 104fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
 105    // https://github.com/dtolnay/proc-macro2/issues/235
 106    match token {
 107        TokenTree::Literal(crate::Literal {
 108            #[cfg(wrap_proc_macro)]
 109                inner: crate::imp::Literal::Fallback(literal),
 110            #[cfg(not(wrap_proc_macro))]
 111                inner: literal,
 112            ..
 113        }) if literal.repr.starts_with('-') => {
 114            push_negative_literal(vec, literal);
 115        }
 116        _ => vec.push(token),
 117    }
 118
 119    #[cold]
 120    fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
 121        literal.repr.remove(0);
 122        let mut punct = crate::Punct::new('-', Spacing::Alone);
 123        punct.set_span(crate::Span::_new_fallback(literal.span));
 124        vec.push(TokenTree::Punct(punct));
 125        vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal)));
 126    }
 127}
 128
 129// Nonrecursive to prevent stack overflow.
 130impl Drop for TokenStream {
 131    fn drop(&mut self) {
 132        let mut stack = Vec::new();
 133        let mut current = match self.inner.get_mut() {
 134            Some(inner) => inner.take().into_iter(),
 135            None => return,
 136        };
 137        loop {
 138            while let Some(token) = current.next() {
 139                let group = match token {
 140                    TokenTree::Group(group) => group.inner,
 141                    _ => continue,
 142                };
 143                #[cfg(wrap_proc_macro)]
 144                let group = match group {
 145                    crate::imp::Group::Fallback(group) => group,
 146                    crate::imp::Group::Compiler(_) => continue,
 147                };
 148                let mut group = group;
 149                if let Some(inner) = group.stream.inner.get_mut() {
 150                    stack.push(current);
 151                    current = inner.take().into_iter();
 152                }
 153            }
 154            match stack.pop() {
 155                Some(next) => current = next,
 156                None => return,
 157            }
 158        }
 159    }
 160}
 161
 162pub(crate) struct TokenStreamBuilder {
 163    inner: RcVecBuilder<TokenTree>,
 164}
 165
 166impl TokenStreamBuilder {
 167    pub(crate) fn new() -> Self {
 168        TokenStreamBuilder {
 169            inner: RcVecBuilder::new(),
 170        }
 171    }
 172
 173    pub(crate) fn with_capacity(cap: usize) -> Self {
 174        TokenStreamBuilder {
 175            inner: RcVecBuilder::with_capacity(cap),
 176        }
 177    }
 178
 179    pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
 180        self.inner.push(tt);
 181    }
 182
 183    pub(crate) fn build(self) -> TokenStream {
 184        TokenStream {
 185            inner: self.inner.build(),
 186        }
 187    }
 188}
 189
 190#[cfg(span_locations)]
 191fn get_cursor(src: &str) -> Cursor {
 192    #[cfg(fuzzing)]
 193    return Cursor { rest: src, off: 1 };
 194
 195    // Create a dummy file & add it to the source map
 196    #[cfg(not(fuzzing))]
 197    SOURCE_MAP.with(|sm| {
 198        let mut sm = sm.borrow_mut();
 199        let span = sm.add_file(src);
 200        Cursor {
 201            rest: src,
 202            off: span.lo,
 203        }
 204    })
 205}
 206
 207#[cfg(not(span_locations))]
 208fn get_cursor(src: &str) -> Cursor {
 209    Cursor { rest: src }
 210}
 211
 212impl Display for LexError {
 213    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 214        f.write_str("cannot parse string into token stream")
 215    }
 216}
 217
 218impl Display for TokenStream {
 219    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 220        let mut joint = false;
 221        for (i, tt) in self.inner.iter().enumerate() {
 222            if i != 0 && !joint {
 223                write!(f, " ")?;
 224            }
 225            joint = false;
 226            match tt {
 227                TokenTree::Group(tt) => Display::fmt(tt, f),
 228                TokenTree::Ident(tt) => Display::fmt(tt, f),
 229                TokenTree::Punct(tt) => {
 230                    joint = tt.spacing() == Spacing::Joint;
 231                    Display::fmt(tt, f)
 232                }
 233                TokenTree::Literal(tt) => Display::fmt(tt, f),
 234            }?;
 235        }
 236
 237        Ok(())
 238    }
 239}
 240
 241impl Debug for TokenStream {
 242    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 243        f.write_str("TokenStream ")?;
 244        f.debug_list().entries(self.clone()).finish()
 245    }
 246}
 247
 248#[cfg(feature = "proc-macro")]
 249impl From<proc_macro::TokenStream> for TokenStream {
 250    fn from(inner: proc_macro::TokenStream) -> Self {
 251        TokenStream::from_str_unchecked(&inner.to_string())
 252    }
 253}
 254
 255#[cfg(feature = "proc-macro")]
 256impl From<TokenStream> for proc_macro::TokenStream {
 257    fn from(inner: TokenStream) -> Self {
 258        proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
 259    }
 260}
 261
 262impl From<TokenTree> for TokenStream {
 263    fn from(tree: TokenTree) -> Self {
 264        let mut stream = RcVecBuilder::new();
 265        push_token_from_proc_macro(stream.as_mut(), tree);
 266        TokenStream {
 267            inner: stream.build(),
 268        }
 269    }
 270}
 271
 272impl FromIterator<TokenTree> for TokenStream {
 273    fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
 274        let mut stream = TokenStream::new();
 275        stream.extend(tokens);
 276        stream
 277    }
 278}
 279
 280impl FromIterator<TokenStream> for TokenStream {
 281    fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
 282        let mut v = RcVecBuilder::new();
 283
 284        for stream in streams {
 285            v.extend(stream.take_inner());
 286        }
 287
 288        TokenStream { inner: v.build() }
 289    }
 290}
 291
 292impl Extend<TokenTree> for TokenStream {
 293    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
 294        let mut vec = self.inner.make_mut();
 295        tokens
 296            .into_iter()
 297            .for_each(|token| push_token_from_proc_macro(vec.as_mut(), token));
 298    }
 299}
 300
 301impl Extend<TokenStream> for TokenStream {
 302    fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
 303        self.inner.make_mut().extend(streams.into_iter().flatten());
 304    }
 305}
 306
 307pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
 308
 309impl IntoIterator for TokenStream {
 310    type Item = TokenTree;
 311    type IntoIter = TokenTreeIter;
 312
 313    fn into_iter(self) -> TokenTreeIter {
 314        self.take_inner().into_iter()
 315    }
 316}
 317
 318#[cfg(all(span_locations, not(fuzzing)))]
 319thread_local! {
 320    static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
 321        // Start with a single dummy file which all call_site() and def_site()
 322        // spans reference.
 323        files: vec![FileInfo {
 324            source_text: String::new(),
 325            span: Span { lo: 0, hi: 0 },
 326            lines: vec![0],
 327            char_index_to_byte_offset: BTreeMap::new(),
 328        }],
 329    });
 330}
 331
 332#[cfg(span_locations)]
 333pub(crate) fn invalidate_current_thread_spans() {
 334    #[cfg(not(fuzzing))]
 335    SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
 336}
 337
 338#[cfg(all(span_locations, not(fuzzing)))]
 339struct FileInfo {
 340    source_text: String,
 341    span: Span,
 342    lines: Vec<usize>,
 343    char_index_to_byte_offset: BTreeMap<usize, usize>,
 344}
 345
 346#[cfg(all(span_locations, not(fuzzing)))]
 347impl FileInfo {
 348    fn offset_line_column(&self, offset: usize) -> LineColumn {
 349        assert!(self.span_within(Span {
 350            lo: offset as u32,
 351            hi: offset as u32,
 352        }));
 353        let offset = offset - self.span.lo as usize;
 354        match self.lines.binary_search(&offset) {
 355            Ok(found) => LineColumn {
 356                line: found + 1,
 357                column: 0,
 358            },
 359            Err(idx) => LineColumn {
 360                line: idx,
 361                column: offset - self.lines[idx - 1],
 362            },
 363        }
 364    }
 365
 366    fn span_within(&self, span: Span) -> bool {
 367        span.lo >= self.span.lo && span.hi <= self.span.hi
 368    }
 369
 370    fn byte_range(&mut self, span: Span) -> Range<usize> {
 371        let lo_char = (span.lo - self.span.lo) as usize;
 372
 373        // Look up offset of the largest already-computed char index that is
 374        // less than or equal to the current requested one. We resume counting
 375        // chars from that point.
 376        let (&last_char_index, &last_byte_offset) = self
 377            .char_index_to_byte_offset
 378            .range(..=lo_char)
 379            .next_back()
 380            .unwrap_or((&0, &0));
 381
 382        let lo_byte = if last_char_index == lo_char {
 383            last_byte_offset
 384        } else {
 385            let total_byte_offset = match self.source_text[last_byte_offset..]
 386                .char_indices()
 387                .nth(lo_char - last_char_index)
 388            {
 389                Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
 390                None => self.source_text.len(),
 391            };
 392            self.char_index_to_byte_offset
 393                .insert(lo_char, total_byte_offset);
 394            total_byte_offset
 395        };
 396
 397        let trunc_lo = &self.source_text[lo_byte..];
 398        let char_len = (span.hi - span.lo) as usize;
 399        lo_byte..match trunc_lo.char_indices().nth(char_len) {
 400            Some((offset, _ch)) => lo_byte + offset,
 401            None => self.source_text.len(),
 402        }
 403    }
 404
 405    fn source_text(&mut self, span: Span) -> String {
 406        let byte_range = self.byte_range(span);
 407        self.source_text[byte_range].to_owned()
 408    }
 409}
 410
 411/// Computes the offsets of each line in the given source string
 412/// and the total number of characters
 413#[cfg(all(span_locations, not(fuzzing)))]
 414fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
 415    let mut lines = vec![0];
 416    let mut total = 0;
 417
 418    for ch in s.chars() {
 419        total += 1;
 420        if ch == '\n' {
 421            lines.push(total);
 422        }
 423    }
 424
 425    (total, lines)
 426}
 427
 428#[cfg(all(span_locations, not(fuzzing)))]
 429struct SourceMap {
 430    files: Vec<FileInfo>,
 431}
 432
 433#[cfg(all(span_locations, not(fuzzing)))]
 434impl SourceMap {
 435    fn next_start_pos(&self) -> u32 {
 436        // Add 1 so there's always space between files.
 437        //
 438        // We'll always have at least 1 file, as we initialize our files list
 439        // with a dummy file.
 440        self.files.last().unwrap().span.hi + 1
 441    }
 442
 443    fn add_file(&mut self, src: &str) -> Span {
 444        let (len, lines) = lines_offsets(src);
 445        let lo = self.next_start_pos();
 446        let span = Span {
 447            lo,
 448            hi: lo + (len as u32),
 449        };
 450
 451        self.files.push(FileInfo {
 452            source_text: src.to_owned(),
 453            span,
 454            lines,
 455            // Populated lazily by source_text().
 456            char_index_to_byte_offset: BTreeMap::new(),
 457        });
 458
 459        span
 460    }
 461
 462    fn find(&self, span: Span) -> usize {
 463        match self.files.binary_search_by(|file| {
 464            if file.span.hi < span.lo {
 465                Ordering::Less
 466            } else if file.span.lo > span.hi {
 467                Ordering::Greater
 468            } else {
 469                assert!(file.span_within(span));
 470                Ordering::Equal
 471            }
 472        }) {
 473            Ok(i) => i,
 474            Err(_) => unreachable!("Invalid span with no related FileInfo!"),
 475        }
 476    }
 477
 478    fn filepath(&self, span: Span) -> String {
 479        let i = self.find(span);
 480        if i == 0 {
 481            "<unspecified>".to_owned()
 482        } else {
 483            format!("<parsed string {}>", i)
 484        }
 485    }
 486
 487    fn fileinfo(&self, span: Span) -> &FileInfo {
 488        let i = self.find(span);
 489        &self.files[i]
 490    }
 491
 492    fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
 493        let i = self.find(span);
 494        &mut self.files[i]
 495    }
 496}
 497
 498#[derive(Clone, Copy, PartialEq, Eq)]
 499pub(crate) struct Span {
 500    #[cfg(span_locations)]
 501    pub(crate) lo: u32,
 502    #[cfg(span_locations)]
 503    pub(crate) hi: u32,
 504}
 505
 506impl Span {
 507    #[cfg(not(span_locations))]
 508    pub(crate) fn call_site() -> Self {
 509        Span {}
 510    }
 511
 512    #[cfg(span_locations)]
 513    pub(crate) fn call_site() -> Self {
 514        Span { lo: 0, hi: 0 }
 515    }
 516
 517    pub(crate) fn mixed_site() -> Self {
 518        Span::call_site()
 519    }
 520
 521    #[cfg(procmacro2_semver_exempt)]
 522    pub(crate) fn def_site() -> Self {
 523        Span::call_site()
 524    }
 525
 526    pub(crate) fn resolved_at(&self, _other: Span) -> Span {
 527        // Stable spans consist only of line/column information, so
 528        // `resolved_at` and `located_at` only select which span the
 529        // caller wants line/column information from.
 530        *self
 531    }
 532
 533    pub(crate) fn located_at(&self, other: Span) -> Span {
 534        other
 535    }
 536
 537    #[cfg(span_locations)]
 538    pub(crate) fn byte_range(&self) -> Range<usize> {
 539        #[cfg(fuzzing)]
 540        return 0..0;
 541
 542        #[cfg(not(fuzzing))]
 543        {
 544            if self.is_call_site() {
 545                0..0
 546            } else {
 547                SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
 548            }
 549        }
 550    }
 551
 552    #[cfg(span_locations)]
 553    pub(crate) fn start(&self) -> LineColumn {
 554        #[cfg(fuzzing)]
 555        return LineColumn { line: 0, column: 0 };
 556
 557        #[cfg(not(fuzzing))]
 558        SOURCE_MAP.with(|sm| {
 559            let sm = sm.borrow();
 560            let fi = sm.fileinfo(*self);
 561            fi.offset_line_column(self.lo as usize)
 562        })
 563    }
 564
 565    #[cfg(span_locations)]
 566    pub(crate) fn end(&self) -> LineColumn {
 567        #[cfg(fuzzing)]
 568        return LineColumn { line: 0, column: 0 };
 569
 570        #[cfg(not(fuzzing))]
 571        SOURCE_MAP.with(|sm| {
 572            let sm = sm.borrow();
 573            let fi = sm.fileinfo(*self);
 574            fi.offset_line_column(self.hi as usize)
 575        })
 576    }
 577
 578    #[cfg(span_locations)]
 579    pub(crate) fn file(&self) -> String {
 580        #[cfg(fuzzing)]
 581        return "<unspecified>".to_owned();
 582
 583        #[cfg(not(fuzzing))]
 584        SOURCE_MAP.with(|sm| {
 585            let sm = sm.borrow();
 586            sm.filepath(*self)
 587        })
 588    }
 589
 590    #[cfg(span_locations)]
 591    pub(crate) fn local_file(&self) -> Option<PathBuf> {
 592        None
 593    }
 594
 595    #[cfg(not(span_locations))]
 596    pub(crate) fn join(&self, _other: Span) -> Option<Span> {
 597        Some(Span {})
 598    }
 599
 600    #[cfg(span_locations)]
 601    pub(crate) fn join(&self, other: Span) -> Option<Span> {
 602        #[cfg(fuzzing)]
 603        return {
 604            let _ = other;
 605            None
 606        };
 607
 608        #[cfg(not(fuzzing))]
 609        SOURCE_MAP.with(|sm| {
 610            let sm = sm.borrow();
 611            // If `other` is not within the same FileInfo as us, return None.
 612            if !sm.fileinfo(*self).span_within(other) {
 613                return None;
 614            }
 615            Some(Span {
 616                lo: cmp::min(self.lo, other.lo),
 617                hi: cmp::max(self.hi, other.hi),
 618            })
 619        })
 620    }
 621
 622    #[cfg(not(span_locations))]
 623    pub(crate) fn source_text(&self) -> Option<String> {
 624        None
 625    }
 626
 627    #[cfg(span_locations)]
 628    pub(crate) fn source_text(&self) -> Option<String> {
 629        #[cfg(fuzzing)]
 630        return None;
 631
 632        #[cfg(not(fuzzing))]
 633        {
 634            if self.is_call_site() {
 635                None
 636            } else {
 637                Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
 638            }
 639        }
 640    }
 641
 642    #[cfg(not(span_locations))]
 643    pub(crate) fn first_byte(self) -> Self {
 644        self
 645    }
 646
 647    #[cfg(span_locations)]
 648    pub(crate) fn first_byte(self) -> Self {
 649        Span {
 650            lo: self.lo,
 651            hi: cmp::min(self.lo.saturating_add(1), self.hi),
 652        }
 653    }
 654
 655    #[cfg(not(span_locations))]
 656    pub(crate) fn last_byte(self) -> Self {
 657        self
 658    }
 659
 660    #[cfg(span_locations)]
 661    pub(crate) fn last_byte(self) -> Self {
 662        Span {
 663            lo: cmp::max(self.hi.saturating_sub(1), self.lo),
 664            hi: self.hi,
 665        }
 666    }
 667
 668    #[cfg(span_locations)]
 669    fn is_call_site(&self) -> bool {
 670        self.lo == 0 && self.hi == 0
 671    }
 672}
 673
 674impl Debug for Span {
 675    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 676        #[cfg(span_locations)]
 677        return write!(f, "bytes({}..{})", self.lo, self.hi);
 678
 679        #[cfg(not(span_locations))]
 680        write!(f, "Span")
 681    }
 682}
 683
 684pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
 685    #[cfg(span_locations)]
 686    {
 687        if span.is_call_site() {
 688            return;
 689        }
 690    }
 691
 692    if cfg!(span_locations) {
 693        debug.field("span", &span);
 694    }
 695}
 696
 697#[derive(Clone)]
 698pub(crate) struct Group {
 699    delimiter: Delimiter,
 700    stream: TokenStream,
 701    span: Span,
 702}
 703
 704impl Group {
 705    pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
 706        Group {
 707            delimiter,
 708            stream,
 709            span: Span::call_site(),
 710        }
 711    }
 712
 713    pub(crate) fn delimiter(&self) -> Delimiter {
 714        self.delimiter
 715    }
 716
 717    pub(crate) fn stream(&self) -> TokenStream {
 718        self.stream.clone()
 719    }
 720
 721    pub(crate) fn span(&self) -> Span {
 722        self.span
 723    }
 724
 725    pub(crate) fn span_open(&self) -> Span {
 726        self.span.first_byte()
 727    }
 728
 729    pub(crate) fn span_close(&self) -> Span {
 730        self.span.last_byte()
 731    }
 732
 733    pub(crate) fn set_span(&mut self, span: Span) {
 734        self.span = span;
 735    }
 736}
 737
 738impl Display for Group {
 739    // We attempt to match libproc_macro's formatting.
 740    // Empty parens: ()
 741    // Nonempty parens: (...)
 742    // Empty brackets: []
 743    // Nonempty brackets: [...]
 744    // Empty braces: { }
 745    // Nonempty braces: { ... }
 746    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 747        let (open, close) = match self.delimiter {
 748            Delimiter::Parenthesis => ("(", ")"),
 749            Delimiter::Brace => ("{ ", "}"),
 750            Delimiter::Bracket => ("[", "]"),
 751            Delimiter::None => ("", ""),
 752        };
 753
 754        f.write_str(open)?;
 755        Display::fmt(&self.stream, f)?;
 756        if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
 757            f.write_str(" ")?;
 758        }
 759        f.write_str(close)?;
 760
 761        Ok(())
 762    }
 763}
 764
 765impl Debug for Group {
 766    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
 767        let mut debug = fmt.debug_struct("Group");
 768        debug.field("delimiter", &self.delimiter);
 769        debug.field("stream", &self.stream);
 770        debug_span_field_if_nontrivial(&mut debug, self.span);
 771        debug.finish()
 772    }
 773}
 774
 775#[derive(Clone)]
 776pub(crate) struct Ident {
 777    sym: Box<str>,
 778    span: Span,
 779    raw: bool,
 780}
 781
 782impl Ident {
 783    #[track_caller]
 784    pub(crate) fn new_checked(string: &str, span: Span) -> Self {
 785        validate_ident(string);
 786        Ident::new_unchecked(string, span)
 787    }
 788
 789    pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
 790        Ident {
 791            sym: Box::from(string),
 792            span,
 793            raw: false,
 794        }
 795    }
 796
 797    #[track_caller]
 798    pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
 799        validate_ident_raw(string);
 800        Ident::new_raw_unchecked(string, span)
 801    }
 802
 803    pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
 804        Ident {
 805            sym: Box::from(string),
 806            span,
 807            raw: true,
 808        }
 809    }
 810
 811    pub(crate) fn span(&self) -> Span {
 812        self.span
 813    }
 814
 815    pub(crate) fn set_span(&mut self, span: Span) {
 816        self.span = span;
 817    }
 818}
 819
 820pub(crate) fn is_ident_start(c: char) -> bool {
 821    c == '_' || c.is_ascii_alphabetic()
 822}
 823
 824pub(crate) fn is_ident_continue(c: char) -> bool {
 825    c == '_' || c.is_ascii_alphanumeric()
 826}
 827
 828#[track_caller]
 829fn validate_ident(string: &str) {
 830    if string.is_empty() {
 831        panic!("Ident is not allowed to be empty; use Option<Ident>");
 832    }
 833
 834    if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
 835        panic!("Ident cannot be a number; use Literal instead");
 836    }
 837
 838    fn ident_ok(string: &str) -> bool {
 839        let mut chars = string.chars();
 840        let first = chars.next().unwrap();
 841        if !is_ident_start(first) {
 842            return false;
 843        }
 844        for ch in chars {
 845            if !is_ident_continue(ch) {
 846                return false;
 847            }
 848        }
 849        true
 850    }
 851
 852    if !ident_ok(string) {
 853        panic!("{:?} is not a valid Ident", string);
 854    }
 855}
 856
 857#[track_caller]
 858fn validate_ident_raw(string: &str) {
 859    validate_ident(string);
 860
 861    match string {
 862        "_" | "super" | "self" | "Self" | "crate" => {
 863            panic!("`r#{}` cannot be a raw identifier", string);
 864        }
 865        _ => {}
 866    }
 867}
 868
 869impl PartialEq for Ident {
 870    fn eq(&self, other: &Ident) -> bool {
 871        self.sym == other.sym && self.raw == other.raw
 872    }
 873}
 874
 875impl<T> PartialEq<T> for Ident
 876where
 877    T: ?Sized + AsRef<str>,
 878{
 879    fn eq(&self, other: &T) -> bool {
 880        let other = other.as_ref();
 881        if self.raw {
 882            other.starts_with("r#") && *self.sym == other[2..]
 883        } else {
 884            *self.sym == *other
 885        }
 886    }
 887}
 888
 889impl Display for Ident {
 890    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 891        if self.raw {
 892            f.write_str("r#")?;
 893        }
 894        Display::fmt(&self.sym, f)
 895    }
 896}
 897
 898#[allow(clippy::missing_fields_in_debug)]
 899impl Debug for Ident {
 900    // Ident(proc_macro), Ident(r#union)
 901    #[cfg(not(span_locations))]
 902    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 903        let mut debug = f.debug_tuple("Ident");
 904        debug.field(&format_args!("{}", self));
 905        debug.finish()
 906    }
 907
 908    // Ident {
 909    //     sym: proc_macro,
 910    //     span: bytes(128..138)
 911    // }
 912    #[cfg(span_locations)]
 913    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 914        let mut debug = f.debug_struct("Ident");
 915        debug.field("sym", &format_args!("{}", self));
 916        debug_span_field_if_nontrivial(&mut debug, self.span);
 917        debug.finish()
 918    }
 919}
 920
 921#[derive(Clone)]
 922pub(crate) struct Literal {
 923    pub(crate) repr: String,
 924    span: Span,
 925}
 926
 927macro_rules! suffixed_numbers {
 928    ($($name:ident => $kind:ident,)*) => ($(
 929        pub(crate) fn $name(n: $kind) -> Literal {
 930            Literal::_new(format!(concat!("{}", stringify!($kind)), n))
 931        }
 932    )*)
 933}
 934
 935macro_rules! unsuffixed_numbers {
 936    ($($name:ident => $kind:ident,)*) => ($(
 937        pub(crate) fn $name(n: $kind) -> Literal {
 938            Literal::_new(n.to_string())
 939        }
 940    )*)
 941}
 942
 943impl Literal {
 944    pub(crate) fn _new(repr: String) -> Self {
 945        Literal {
 946            repr,
 947            span: Span::call_site(),
 948        }
 949    }
 950
 951    pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
 952        let mut cursor = get_cursor(repr);
 953        #[cfg(span_locations)]
 954        let lo = cursor.off;
 955
 956        let negative = cursor.starts_with_char('-');
 957        if negative {
 958            cursor = cursor.advance(1);
 959            if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
 960                return Err(LexError::call_site());
 961            }
 962        }
 963
 964        if let Ok((rest, mut literal)) = parse::literal(cursor) {
 965            if rest.is_empty() {
 966                if negative {
 967                    literal.repr.insert(0, '-');
 968                }
 969                literal.span = Span {
 970                    #[cfg(span_locations)]
 971                    lo,
 972                    #[cfg(span_locations)]
 973                    hi: rest.off,
 974                };
 975                return Ok(literal);
 976            }
 977        }
 978        Err(LexError::call_site())
 979    }
 980
 981    pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
 982        Literal::_new(repr.to_owned())
 983    }
 984
 985    suffixed_numbers! {
 986        u8_suffixed => u8,
 987        u16_suffixed => u16,
 988        u32_suffixed => u32,
 989        u64_suffixed => u64,
 990        u128_suffixed => u128,
 991        usize_suffixed => usize,
 992        i8_suffixed => i8,
 993        i16_suffixed => i16,
 994        i32_suffixed => i32,
 995        i64_suffixed => i64,
 996        i128_suffixed => i128,
 997        isize_suffixed => isize,
 998
 999        f32_suffixed => f32,
1000        f64_suffixed => f64,
1001    }
1002
1003    unsuffixed_numbers! {
1004        u8_unsuffixed => u8,
1005        u16_unsuffixed => u16,
1006        u32_unsuffixed => u32,
1007        u64_unsuffixed => u64,
1008        u128_unsuffixed => u128,
1009        usize_unsuffixed => usize,
1010        i8_unsuffixed => i8,
1011        i16_unsuffixed => i16,
1012        i32_unsuffixed => i32,
1013        i64_unsuffixed => i64,
1014        i128_unsuffixed => i128,
1015        isize_unsuffixed => isize,
1016    }
1017
1018    pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1019        let mut s = f.to_string();
1020        if !s.contains('.') {
1021            s.push_str(".0");
1022        }
1023        Literal::_new(s)
1024    }
1025
1026    pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1027        let mut s = f.to_string();
1028        if !s.contains('.') {
1029            s.push_str(".0");
1030        }
1031        Literal::_new(s)
1032    }
1033
1034    pub(crate) fn string(string: &str) -> Literal {
1035        let mut repr = String::with_capacity(string.len() + 2);
1036        repr.push('"');
1037        escape_utf8(string, &mut repr);
1038        repr.push('"');
1039        Literal::_new(repr)
1040    }
1041
1042    pub(crate) fn character(ch: char) -> Literal {
1043        let mut repr = String::new();
1044        repr.push('\'');
1045        if ch == '"' {
1046            // escape_debug turns this into '\"' which is unnecessary.
1047            repr.push(ch);
1048        } else {
1049            repr.extend(ch.escape_debug());
1050        }
1051        repr.push('\'');
1052        Literal::_new(repr)
1053    }
1054
1055    pub(crate) fn byte_character(byte: u8) -> Literal {
1056        let mut repr = "b'".to_string();
1057        #[allow(clippy::match_overlapping_arm)]
1058        match byte {
1059            b'\0' => repr.push_str(r"\0"),
1060            b'\t' => repr.push_str(r"\t"),
1061            b'\n' => repr.push_str(r"\n"),
1062            b'\r' => repr.push_str(r"\r"),
1063            b'\'' => repr.push_str(r"\'"),
1064            b'\\' => repr.push_str(r"\\"),
1065            b'\x20'..=b'\x7E' => repr.push(byte as char),
1066            _ => {
1067                let _ = write!(repr, r"\x{:02X}", byte);
1068            }
1069        }
1070        repr.push('\'');
1071        Literal::_new(repr)
1072    }
1073
1074    pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1075        let mut repr = "b\"".to_string();
1076        let mut bytes = bytes.iter();
1077        while let Some(&b) = bytes.next() {
1078            #[allow(clippy::match_overlapping_arm)]
1079            match b {
1080                b'\0' => repr.push_str(match bytes.as_slice().first() {
1081                    // circumvent clippy::octal_escapes lint
1082                    Some(b'0'..=b'7') => r"\x00",
1083                    _ => r"\0",
1084                }),
1085                b'\t' => repr.push_str(r"\t"),
1086                b'\n' => repr.push_str(r"\n"),
1087                b'\r' => repr.push_str(r"\r"),
1088                b'"' => repr.push_str("\\\""),
1089                b'\\' => repr.push_str(r"\\"),
1090                b'\x20'..=b'\x7E' => repr.push(b as char),
1091                _ => {
1092                    let _ = write!(repr, r"\x{:02X}", b);
1093                }
1094            }
1095        }
1096        repr.push('"');
1097        Literal::_new(repr)
1098    }
1099
1100    pub(crate) fn c_string(string: &CStr) -> Literal {
1101        let mut repr = "c\"".to_string();
1102        let mut bytes = string.to_bytes();
1103        while !bytes.is_empty() {
1104            let (valid, invalid) = match str::from_utf8(bytes) {
1105                Ok(all_valid) => {
1106                    bytes = b"";
1107                    (all_valid, bytes)
1108                }
1109                Err(utf8_error) => {
1110                    let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1111                    let valid = str::from_utf8(valid).unwrap();
1112                    let invalid = utf8_error
1113                        .error_len()
1114                        .map_or(rest, |error_len| &rest[..error_len]);
1115                    bytes = &bytes[valid.len() + invalid.len()..];
1116                    (valid, invalid)
1117                }
1118            };
1119            escape_utf8(valid, &mut repr);
1120            for &byte in invalid {
1121                let _ = write!(repr, r"\x{:02X}", byte);
1122            }
1123        }
1124        repr.push('"');
1125        Literal::_new(repr)
1126    }
1127
1128    pub(crate) fn span(&self) -> Span {
1129        self.span
1130    }
1131
1132    pub(crate) fn set_span(&mut self, span: Span) {
1133        self.span = span;
1134    }
1135
1136    pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1137        #[cfg(not(span_locations))]
1138        {
1139            let _ = range;
1140            None
1141        }
1142
1143        #[cfg(span_locations)]
1144        {
1145            use core::ops::Bound;
1146
1147            let lo = match range.start_bound() {
1148                Bound::Included(start) => {
1149                    let start = u32::try_from(*start).ok()?;
1150                    self.span.lo.checked_add(start)?
1151                }
1152                Bound::Excluded(start) => {
1153                    let start = u32::try_from(*start).ok()?;
1154                    self.span.lo.checked_add(start)?.checked_add(1)?
1155                }
1156                Bound::Unbounded => self.span.lo,
1157            };
1158            let hi = match range.end_bound() {
1159                Bound::Included(end) => {
1160                    let end = u32::try_from(*end).ok()?;
1161                    self.span.lo.checked_add(end)?.checked_add(1)?
1162                }
1163                Bound::Excluded(end) => {
1164                    let end = u32::try_from(*end).ok()?;
1165                    self.span.lo.checked_add(end)?
1166                }
1167                Bound::Unbounded => self.span.hi,
1168            };
1169            if lo <= hi && hi <= self.span.hi {
1170                Some(Span { lo, hi })
1171            } else {
1172                None
1173            }
1174        }
1175    }
1176}
1177
1178impl Display for Literal {
1179    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1180        Display::fmt(&self.repr, f)
1181    }
1182}
1183
1184impl Debug for Literal {
1185    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1186        let mut debug = fmt.debug_struct("Literal");
1187        debug.field("lit", &format_args!("{}", self.repr));
1188        debug_span_field_if_nontrivial(&mut debug, self.span);
1189        debug.finish()
1190    }
1191}
1192
1193fn escape_utf8(string: &str, repr: &mut String) {
1194    let mut chars = string.chars();
1195    while let Some(ch) = chars.next() {
1196        if ch == '\0' {
1197            repr.push_str(
1198                if chars
1199                    .as_str()
1200                    .starts_with(|next| '0' <= next && next <= '7')
1201                {
1202                    // circumvent clippy::octal_escapes lint
1203                    r"\x00"
1204                } else {
1205                    r"\0"
1206                },
1207            );
1208        } else if ch == '\'' {
1209            // escape_debug turns this into "\'" which is unnecessary.
1210            repr.push(ch);
1211        } else {
1212            repr.extend(ch.escape_debug());
1213        }
1214    }
1215}
1216
1217#[cfg(feature = "proc-macro")]
1218pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1219    #[cfg(wrap_proc_macro)]
1220    fn valid(src: &str) -> bool;
1221
1222    #[cfg(wrap_proc_macro)]
1223    fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1224        // Validate using fallback parser, because rustc is incapable of
1225        // returning a recoverable Err for certain invalid token streams, and
1226        // will instead permanently poison the compilation.
1227        if !Self::valid(src) {
1228            return Err(imp::LexError::CompilerPanic);
1229        }
1230
1231        // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1232        match panic::catch_unwind(|| Self::from_str(src)) {
1233            Ok(Ok(ok)) => Ok(ok),
1234            Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1235            Err(_panic) => Err(imp::LexError::CompilerPanic),
1236        }
1237    }
1238
1239    fn from_str_unchecked(src: &str) -> Self {
1240        Self::from_str(src).unwrap()
1241    }
1242}
1243
1244#[cfg(feature = "proc-macro")]
1245impl FromStr2 for proc_macro::TokenStream {
1246    #[cfg(wrap_proc_macro)]
1247    fn valid(src: &str) -> bool {
1248        TokenStream::from_str_checked(src).is_ok()
1249    }
1250}
1251
1252#[cfg(feature = "proc-macro")]
1253impl FromStr2 for proc_macro::Literal {
1254    #[cfg(wrap_proc_macro)]
1255    fn valid(src: &str) -> bool {
1256        Literal::from_str_checked(src).is_ok()
1257    }
1258}