feat: Make macros operate on token streams instead of AST nodes (#5301)

# Description

## Problem\*

Resolves #5284

## Summary\*

Adds a new `Quoted` type which is the new default output of a `quote {
... }` expression. In addition, a `quote` expression now quotes an
unparsed, raw vector of tokens terminated by the matching closing brace.
This has a number of advantages:

- Quoting code fragments: Since these token streams are not parsed,
there is almost no restriction on their grammar. Code fragments such as
`quote { ) ( }` are valid as long as they're grammatically valid by the
time the macro is unquoted at its call site. The only restriction on the
grammar of these is that they cannot have an unmatched `}` since that
would close the `quote` expression itself. Matched `{` and `}` paris
however, are fine: `quote { look, a block: { foo } }`.
- Since we don't need to parse to a specific grammar rule, we don't need
a new quoted type for each grammar rule we quote (see the existing
`Expr`, `Type`, and `TopLevelItem` types).
- The implementation details are simpler (e.g. this PR net removes
lines).

The current plan is to keep the other macro types (`Expr`, `Type`,
`TopLevelItem`, etc) as optional outputs of `quote` only if they're
specified after the keyword, e.g. `quote Type { Foo<A, B> }`. Since the
new approach of using token streams by default is much more flexible,
there's less of a need for these however.

## Additional Context

The formatting of quote expressions has also been fixed. This was a
necessary change since these expressions hold token streams now instead
of ASTs and thus can't be formatted the old way.

Edit: This does not fix the existing issue with formatting comptime
blocks, only `quote` blocks which were similarly affected.

## Documentation\*

Check one:
- [ ] No documentation needed.
- [ ] Documentation included in this PR.
- [x] **[For Experimental Features]** Documentation to be submitted in a
separate PR.

# PR Checklist\*

- [x] I have tested the changes locally.
- [x] I have formatted the changes with [Prettier](https://prettier.io/)
and/or `cargo fmt` on default settings.

---------

Co-authored-by: Tom French <[email protected]>

compiler/noirc_frontend/src/ast/expression.rs

@@ -7,7 +7,7 @@ use crate::ast::{
 };
 use crate::macros_api::StructId;
 use crate::node_interner::ExprId;
-use crate::token::{Attributes, Token};
+use crate::token::{Attributes, Token, Tokens};
 use acvm::{acir::AcirField, FieldElement};
 use iter_extended::vecmap;
 use noirc_errors::{Span, Spanned};
@@ -33,18 +33,10 @@ pub enum ExpressionKind {
     Tuple(Vec<Expression>),
     Lambda(Box<Lambda>),
     Parenthesized(Box<Expression>),
-    Quote(BlockExpression, Span),
+    Quote(Tokens),
     Unquote(Box<Expression>),
     Comptime(BlockExpression, Span),
 
-    /// Unquote expressions are replaced with UnquoteMarkers when Quoted
-    /// expressions are resolved. Since the expression being quoted remains an
-    /// ExpressionKind (rather than a resolved ExprId), the UnquoteMarker must be
-    /// here in the AST even though it is technically HIR-only.
-    /// Each usize in an UnquoteMarker is an index which corresponds to the index of the
-    /// expression in the Hir::Quote expression list to replace it with.
-    UnquoteMarker(usize),
-
     // This variant is only emitted when inlining the result of comptime
     // code. It is used to translate function values back into the AST while
     // guaranteeing they have the same instantiated type and definition id without resolving again.
@@ -557,12 +549,14 @@ impl Display for ExpressionKind {
             }
             Lambda(lambda) => lambda.fmt(f),
             Parenthesized(sub_expr) => write!(f, "({sub_expr})"),
-            Quote(block, _) => write!(f, "quote {block}"),
             Comptime(block, _) => write!(f, "comptime {block}"),
             Error => write!(f, "Error"),
             Resolved(_) => write!(f, "?Resolved"),
             Unquote(expr) => write!(f, "$({expr})"),
-            UnquoteMarker(index) => write!(f, "${index}"),
+            Quote(tokens) => {
+                let tokens = vecmap(&tokens.0, ToString::to_string);
+                write!(f, "quote {{ {} }}", tokens.join(" "))
+            }
         }
     }
 }

compiler/noirc_frontend/src/elaborator/expressions.rs

@@ -18,7 +18,7 @@ use crate::{
             HirArrayLiteral, HirBinaryOp, HirBlockExpression, HirCallExpression, HirCastExpression,
             HirConstructorExpression, HirIfExpression, HirIndexExpression, HirInfixExpression,
             HirLambda, HirMemberAccess, HirMethodCallExpression, HirMethodReference,
-            HirPrefixExpression, HirQuoted,
+            HirPrefixExpression,
         },
         traits::TraitConstraint,
     },
@@ -28,6 +28,7 @@ use crate::{
         MethodCallExpression, PrefixExpression,
     },
     node_interner::{DefinitionKind, ExprId, FuncId},
+    token::Tokens,
     QuotedType, Shared, StructType, Type,
 };
 
@@ -58,7 +59,7 @@ impl<'context> Elaborator<'context> {
             ExpressionKind::Tuple(tuple) => self.elaborate_tuple(tuple),
             ExpressionKind::Lambda(lambda) => self.elaborate_lambda(*lambda),
             ExpressionKind::Parenthesized(expr) => return self.elaborate_expression(*expr),
-            ExpressionKind::Quote(quote, _) => self.elaborate_quote(quote),
+            ExpressionKind::Quote(quote) => self.elaborate_quote(quote),
             ExpressionKind::Comptime(comptime, _) => {
                 return self.elaborate_comptime_block(comptime, expr.span)
             }
@@ -68,9 +69,6 @@ impl<'context> Elaborator<'context> {
                 self.push_err(ResolverError::UnquoteUsedOutsideQuote { span: expr.span });
                 (HirExpression::Error, Type::Error)
             }
-            ExpressionKind::UnquoteMarker(index) => {
-                unreachable!("UnquoteMarker({index}) remaining in runtime code")
-            }
         };
         let id = self.interner.push_expr(hir_expr);
         self.interner.push_expr_location(id, expr.span, self.file);
@@ -646,11 +644,9 @@ impl<'context> Elaborator<'context> {
         (expr, Type::Function(arg_types, Box::new(body_type), Box::new(env_type)))
     }
 
-    fn elaborate_quote(&mut self, mut block: BlockExpression) -> (HirExpression, Type) {
-        let mut unquoted_exprs = Vec::new();
-        self.find_unquoted_exprs_in_block(&mut block, &mut unquoted_exprs);
-        let quoted = HirQuoted { quoted_block: block, unquoted_exprs };
-        (HirExpression::Quote(quoted), Type::Quoted(QuotedType::Expr))
+    fn elaborate_quote(&mut self, mut tokens: Tokens) -> (HirExpression, Type) {
+        tokens = self.find_unquoted_exprs_tokens(tokens);
+        (HirExpression::Quote(tokens), Type::Quoted(QuotedType::Quoted))
     }
 
     fn elaborate_comptime_block(&mut self, block: BlockExpression, span: Span) -> (ExprId, Type) {
@@ -716,7 +712,7 @@ impl<'context> Elaborator<'context> {
         location: Location,
         return_type: Type,
     ) -> Option<(HirExpression, Type)> {
-        self.unify(&return_type, &Type::Quoted(QuotedType::Expr), || {
+        self.unify(&return_type, &Type::Quoted(QuotedType::Quoted), || {
             TypeCheckError::MacroReturningNonExpr { typ: return_type.clone(), span: location.span }
         });