Skip to content

Commit 7e77f06

Browse files
SpriteOvOSpriteOvO
committed
Append TokenTree with ToTokens in proc_macro::quote! 
1 parent 2a2e87b commit 7e77f06

14 files changed

+108
-64
lines changed

‎library/proc_macro/src/quote.rs

+49-33
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
//! This quasiquoter uses macros 2.0 hygiene to reliably access
55
//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
66
7-
use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
7+
use crate::{
8+
Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
9+
};
810

911
macro_rules! minimal_quote_tt {
1012
(($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
@@ -50,7 +52,7 @@ macro_rules! minimal_quote {
5052
() => { TokenStream::new() };
5153
($($t:tt)*) => {
5254
[
53-
$(TokenStream::from(minimal_quote_ts!($t)),)*
55+
$(ToTokens::into_token_stream(minimal_quote_ts!($t)),)*
5456
].iter().cloned().collect::<TokenStream>()
5557
};
5658
}
@@ -66,48 +68,56 @@ pub fn quote(stream: TokenStream) -> TokenStream {
6668
}
6769
let proc_macro_crate = minimal_quote!(crate);
6870
let mut after_dollar = false;
69-
let tokens = stream
70-
.into_iter()
71-
.filter_map(|tree| {
72-
if after_dollar {
73-
after_dollar = false;
74-
match tree {
75-
TokenTree::Ident(_) => {
76-
return Some(minimal_quote!(Into::<crate::TokenStream>::into(
77-
Clone::clone(&(@ tree))),));
78-
}
79-
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
80-
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
81-
}
82-
} else if let TokenTree::Punct(ref tt) = tree {
83-
if tt.as_char() == '$' {
84-
after_dollar = true;
85-
return None;
71+
72+
let mut tokens = crate::TokenStream::new();
73+
for tree in stream {
74+
if after_dollar {
75+
after_dollar = false;
76+
match tree {
77+
TokenTree::Ident(_) => {
78+
minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
79+
.to_tokens(&mut tokens);
80+
continue;
8681
}
82+
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
83+
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
8784
}
85+
} else if let TokenTree::Punct(ref tt) = tree {
86+
if tt.as_char() == '$' {
87+
after_dollar = true;
88+
continue;
89+
}
90+
}
8891

89-
Some(minimal_quote!(crate::TokenStream::from((@ match tree {
90-
TokenTree::Punct(tt) => minimal_quote!(crate::TokenTree::Punct(crate::Punct::new(
92+
match tree {
93+
TokenTree::Punct(tt) => {
94+
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
9195
(@ TokenTree::from(Literal::character(tt.as_char()))),
9296
(@ match tt.spacing() {
9397
Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
9498
Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
9599
}),
96-
))),
97-
TokenTree::Group(tt) => minimal_quote!(crate::TokenTree::Group(crate::Group::new(
100+
)), &mut ts);)
101+
}
102+
TokenTree::Group(tt) => {
103+
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
98104
(@ match tt.delimiter() {
99105
Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
100106
Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
101107
Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
102108
Delimiter::None => minimal_quote!(crate::Delimiter::None),
103109
}),
104110
(@ quote(tt.stream())),
105-
))),
106-
TokenTree::Ident(tt) => minimal_quote!(crate::TokenTree::Ident(crate::Ident::new(
111+
)), &mut ts);)
112+
}
113+
TokenTree::Ident(tt) => {
114+
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new(
107115
(@ TokenTree::from(Literal::string(&tt.to_string()))),
108116
(@ quote_span(proc_macro_crate.clone(), tt.span())),
109-
))),
110-
TokenTree::Literal(tt) => minimal_quote!(crate::TokenTree::Literal({
117+
)), &mut ts);)
118+
}
119+
TokenTree::Literal(tt) => {
120+
minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
111121
let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
112122
.parse::<crate::TokenStream>()
113123
.unwrap()
@@ -120,16 +130,22 @@ pub fn quote(stream: TokenStream) -> TokenStream {
120130
} else {
121131
unreachable!()
122132
}
123-
}))
124-
})),))
125-
})
126-
.collect::<TokenStream>();
127-
133+
}), &mut ts);)
134+
}
135+
}
136+
.to_tokens(&mut tokens);
137+
}
128138
if after_dollar {
129139
panic!("unexpected trailing `$` in `quote!`");
130140
}
131141

132-
minimal_quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
142+
minimal_quote! {
143+
{
144+
let mut ts = crate::TokenStream::new();
145+
(@ tokens)
146+
ts
147+
}
148+
}
133149
}
134150

135151
/// Quote a `Span` into a `TokenStream`.

‎tests/ui/deriving/auxiliary/another-proc-macro.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45

‎tests/ui/proc-macro/auxiliary/cond_plugin.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45

‎tests/ui/proc-macro/auxiliary/count_compound_ops.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45

5-
use proc_macro::{TokenStream, TokenTree, Spacing, Literal, quote};
6+
use proc_macro::{Literal, Spacing, TokenStream, TokenTree, quote};
67

78
#[proc_macro]
89
pub fn count_compound_ops(input: TokenStream) -> TokenStream {

‎tests/ui/proc-macro/auxiliary/double.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45
use proc_macro::*;

‎tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45
use proc_macro::*;

‎tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro as proc_macro_renamed; // This does not break `quote!`
45

‎tests/ui/proc-macro/auxiliary/mixed-site-span.rs

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45
use proc_macro::*;
@@ -13,10 +14,8 @@ pub fn proc_macro_rules(input: TokenStream) -> TokenStream {
1314
let local_use = id("local_use");
1415
let mut single_quote = Punct::new('\'', Spacing::Joint);
1516
single_quote.set_span(Span::mixed_site());
16-
let label_use: TokenStream = [
17-
TokenTree::from(single_quote),
18-
id("label_use"),
19-
].iter().cloned().collect();
17+
let label_use: TokenStream =
18+
[TokenTree::from(single_quote), id("label_use")].iter().cloned().collect();
2019
quote!(
2120
struct $item_def;
2221
let $local_def = 0;

‎tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23

34
extern crate proc_macro;
45
use proc_macro::{TokenStream, quote};

‎tests/ui/proc-macro/auxiliary/resolved-located-at.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#![feature(proc_macro_def_site)]
22
#![feature(proc_macro_diagnostic)]
33
#![feature(proc_macro_quote)]
4+
#![feature(proc_macro_totokens)]
45

56
extern crate proc_macro;
67
use proc_macro::*;

‎tests/ui/proc-macro/auxiliary/span-from-proc-macro.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
#![feature(proc_macro_quote)]
2+
#![feature(proc_macro_totokens)]
23
#![feature(proc_macro_internals)] // FIXME - this shouldn't be necessary
34

4-
extern crate proc_macro;
55
extern crate custom_quote;
6+
extern crate proc_macro;
67

7-
use proc_macro::{quote, TokenStream};
8+
use proc_macro::{TokenStream, quote};
89

910
macro_rules! expand_to_quote {
1011
() => {
1112
quote! {
1213
let bang_error: bool = 25;
1314
}
14-
}
15+
};
1516
}
1617

1718
#[proc_macro]

‎tests/ui/proc-macro/quote-debug.stdout

+23-19
Original file line numberDiff line numberDiff line change
@@ -19,25 +19,29 @@ extern crate std;
1919
extern crate proc_macro;
2020

2121
fn main() {
22-
[crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("let",
23-
crate::Span::recover_proc_macro_span(0)))),
24-
crate::TokenStream::from(crate::TokenTree::Ident(crate::Ident::new("hello",
25-
crate::Span::recover_proc_macro_span(1)))),
26-
crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new('=',
27-
crate::Spacing::Alone))),
28-
crate::TokenStream::from(crate::TokenTree::Literal({
29-
let mut iter =
30-
"\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
31-
if let (Some(crate::TokenTree::Literal(mut lit)), None) =
32-
(iter.next(), iter.next()) {
33-
lit.set_span(crate::Span::recover_proc_macro_span(2));
34-
lit
35-
} else {
36-
::core::panicking::panic("internal error: entered unreachable code")
37-
}
38-
})),
39-
crate::TokenStream::from(crate::TokenTree::Punct(crate::Punct::new(';',
40-
crate::Spacing::Alone)))].iter().cloned().collect::<crate::TokenStream>()
22+
{
23+
let mut ts = crate::TokenStream::new();
24+
crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("let",
25+
crate::Span::recover_proc_macro_span(0))), &mut ts);
26+
crate::ToTokens::to_tokens(&crate::TokenTree::Ident(crate::Ident::new("hello",
27+
crate::Span::recover_proc_macro_span(1))), &mut ts);
28+
crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new('=',
29+
crate::Spacing::Alone)), &mut ts);
30+
crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
31+
let mut iter =
32+
"\"world\"".parse::<crate::TokenStream>().unwrap().into_iter();
33+
if let (Some(crate::TokenTree::Literal(mut lit)), None) =
34+
(iter.next(), iter.next()) {
35+
lit.set_span(crate::Span::recover_proc_macro_span(2));
36+
lit
37+
} else {
38+
::core::panicking::panic("internal error: entered unreachable code")
39+
}
40+
}), &mut ts);
41+
crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(';',
42+
crate::Spacing::Alone)), &mut ts);
43+
ts
44+
}
4145
}
4246
const _: () =
4347
{

‎tests/ui/proc-macro/quote-interpolation.rs

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
//@ check-pass
2+
3+
#![feature(proc_macro_quote)]
4+
#![feature(proc_macro_totokens)]
5+
#![crate_type = "proc-macro"]
6+
7+
extern crate proc_macro;
8+
9+
use proc_macro::*;
10+
11+
fn main() {
12+
let x = Ident::new("foo", Span::call_site());
13+
let _ = quote! {
14+
let $x = 199;
15+
};
16+
}

‎tests/ui/proc-macro/span-from-proc-macro.stderr

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
error[E0412]: cannot find type `MissingType` in this scope
2-
--> $DIR/auxiliary/span-from-proc-macro.rs:33:20
2+
--> $DIR/auxiliary/span-from-proc-macro.rs:34:20
33
|
44
LL | pub fn error_from_attribute(_args: TokenStream, _input: TokenStream) -> TokenStream {
55
| ----------------------------------------------------------------------------------- in this expansion of `#[error_from_attribute]`
@@ -13,7 +13,7 @@ LL | #[error_from_attribute]
1313
| ----------------------- in this procedural macro expansion
1414

1515
error[E0412]: cannot find type `OtherMissingType` in this scope
16-
--> $DIR/auxiliary/span-from-proc-macro.rs:42:21
16+
--> $DIR/auxiliary/span-from-proc-macro.rs:43:21
1717
|
1818
LL | pub fn error_from_derive(_input: TokenStream) -> TokenStream {
1919
| ------------------------------------------------------------ in this expansion of `#[derive(ErrorFromDerive)]`
@@ -27,7 +27,7 @@ LL | #[derive(ErrorFromDerive)]
2727
| --------------- in this derive macro expansion
2828

2929
error[E0425]: cannot find value `my_ident` in this scope
30-
--> $DIR/auxiliary/span-from-proc-macro.rs:25:9
30+
--> $DIR/auxiliary/span-from-proc-macro.rs:26:9
3131
|
3232
LL | pub fn other_error_from_bang(_input: TokenStream) -> TokenStream {
3333
| ---------------------------------------------------------------- in this expansion of `other_error_from_bang!`
@@ -41,7 +41,7 @@ LL | other_error_from_bang!();
4141
| ------------------------ in this macro invocation
4242

4343
error[E0308]: mismatched types
44-
--> $DIR/auxiliary/span-from-proc-macro.rs:12:36
44+
--> $DIR/auxiliary/span-from-proc-macro.rs:13:36
4545
|
4646
LL | let bang_error: bool = 25;
4747
| ---- ^^ expected `bool`, found integer

0 commit comments

Comments
 (0)