From 3a5183e04805c5f8909e77c23cffc02a81d2b831 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Mon, 9 Dec 2024 17:14:33 -0500 Subject: [PATCH 1/2] Implements if_none and friends --- src/lazy/expanded/compiler.rs | 83 +++++++--- src/lazy/expanded/e_expression.rs | 18 +- src/lazy/expanded/macro_evaluator.rs | 239 +++++++++++++++++++++++++++ src/lazy/expanded/macro_table.rs | 11 +- src/lazy/expanded/template.rs | 7 +- 5 files changed, 330 insertions(+), 28 deletions(-) diff --git a/src/lazy/expanded/compiler.rs b/src/lazy/expanded/compiler.rs index 04f5ab9e..c8ecd71a 100644 --- a/src/lazy/expanded/compiler.rs +++ b/src/lazy/expanded/compiler.rs @@ -17,13 +17,14 @@ use crate::lazy::value::LazyValue; use crate::lazy::value_ref::ValueRef; use crate::result::IonFailure; use crate::{ - AnyEncoding, IonError, IonInput, IonResult, IonType, Macro, MacroTable, Reader, Symbol, - SymbolRef, + AnyEncoding, EncodingContext, IonError, IonInput, IonResult, IonType, Macro, MacroKind, + MacroTable, Reader, Symbol, SymbolRef, }; +use phf::phf_set; use rustc_hash::FxHashMap; use smallvec::SmallVec; use std::ops::Range; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; /// Information inferred about a template's expansion at compile time. #[derive(Copy, Clone, Debug, PartialEq)] @@ -1411,30 +1412,74 @@ impl<'top, D: Decoder> TdlSExpKind<'top, D> { } }; - // See if it's the special form `literal`. - if let ValueRef::Symbol(s) = operation.read()? { - if s == "literal" { - // If it's `$ion::literal`, it's the special form. - if operation.annotations().are(["$ion"])? - // Otherwise, if it has no annotations... - || (!first_expr.has_annotations() - // ...and has not been shadowed by a user-defined macro name... - && tdl_context.pending_macros.macro_with_name("literal").is_none() - && tdl_context.context.macro_table.macro_with_name("literal").is_none()) - { - // ...then it's the special form. - return Ok(TdlSExpKind::Literal(expressions)); - } - } + let operation_name = TemplateCompiler::expect_symbol_text("operation name", operation)?; + + // TDL-only operations that are not in the system macro table. + static SPECIAL_FORM_NAMES: phf::Set<&'static str> = + phf_set!("literal", "if_none", "if_some", "if_single", "if_multi"); + + let is_special_form = SPECIAL_FORM_NAMES.contains(operation_name) + // If it's qualified to the system namespace, it's a special form. + && (operation.annotations().are(["$ion"])? + // Otherwise, if it has no annotations... + || (!first_expr.has_annotations() + // ...and has not been shadowed by a user-defined macro name, it's a special form. + && tdl_context.pending_macros.macro_with_name(operation_name).is_none() + && tdl_context.context.macro_table.macro_with_name(operation_name).is_none())); + + if is_special_form { + let special_form_macro: &Arc = match operation_name { + // The 'literal' operation exists only at compile time... + "literal" => return Ok(TdlSExpKind::Literal(expressions)), + // ...while the cardinality tests are implemented as different flavors of + // the `ConditionalExpansion` macro. + "if_none" => &IF_NONE_MACRO, + "if_some" => &IF_SOME_MACRO, + "if_single" => &IF_SINGLE_MACRO, + "if_multi" => &IF_MULTI_MACRO, + other => unreachable!("unknown name '{}' found in special forms set", other), + }; + + return Ok(TdlSExpKind::MacroInvocation( + Arc::clone(special_form_macro), + expressions, + )); } - // At this point, we know the sexp must be a macro invocation. + // At this point, we know the sexp must be a normal macro invocation. // Resolve the macro name or address to the macro it represents. let macro_ref = TemplateCompiler::resolve_macro_id_expr(tdl_context, operation)?; Ok(TdlSExpKind::MacroInvocation(macro_ref, expressions)) } } +pub static IF_NONE_MACRO: LazyLock> = + LazyLock::new(|| initialize_cardinality_test_macro("if_none", MacroKind::IfNone)); + +pub static IF_SOME_MACRO: LazyLock> = + LazyLock::new(|| initialize_cardinality_test_macro("if_some", MacroKind::IfSome)); + +pub static IF_SINGLE_MACRO: LazyLock> = + LazyLock::new(|| initialize_cardinality_test_macro("if_single", MacroKind::IfSingle)); + +pub static IF_MULTI_MACRO: LazyLock> = + LazyLock::new(|| initialize_cardinality_test_macro("if_multi", MacroKind::IfMulti)); + +fn initialize_cardinality_test_macro(name: &str, kind: MacroKind) -> Arc { + let context = EncodingContext::empty(); + let definition = Macro::new( + Some(name.into()), + TemplateCompiler::compile_signature( + context.get_ref(), + "(test_expr* true_expr* false_expr*)", + ) + .unwrap(), + kind, + ExpansionAnalysis::no_assertions_made(), + ); + Arc::new(definition) +} + #[derive(Copy, Clone)] struct TdlContext<'top> { // The encoding context that was active when compilation began. The body of the macro we're diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index 44eafdb5..0aadd319 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -9,9 +9,9 @@ use crate::lazy::decoder::{Decoder, RawValueExpr}; use crate::lazy::encoding::TextEncoding_1_1; use crate::lazy::expanded::compiler::{ExpansionAnalysis, ExpansionSingleton}; use crate::lazy::expanded::macro_evaluator::{ - AnnotateExpansion, EExpressionArgGroup, ExprGroupExpansion, FlattenExpansion, - IsExhaustedIterator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, - MakeTextExpansion, RawEExpression, TemplateExpansion, ValueExpr, + AnnotateExpansion, ConditionalExpansion, EExpressionArgGroup, ExprGroupExpansion, + FlattenExpansion, IsExhaustedIterator, MacroExpansion, MacroExpansionKind, MacroExpr, + MacroExprArgsIterator, MakeTextExpansion, RawEExpression, TemplateExpansion, ValueExpr, }; use crate::lazy::expanded::macro_table::{MacroKind, MacroRef}; use crate::lazy::expanded::template::TemplateMacroRef; @@ -135,6 +135,18 @@ impl<'top, D: Decoder> EExpression<'top, D> { MacroExpansionKind::Template(TemplateExpansion::new(template_ref)) } MacroKind::ToDo => todo!("system macro {}", invoked_macro.name().unwrap()), + MacroKind::IfNone => { + MacroExpansionKind::Conditional(ConditionalExpansion::if_none(arguments)) + } + MacroKind::IfSome => { + MacroExpansionKind::Conditional(ConditionalExpansion::if_some(arguments)) + } + MacroKind::IfSingle => { + MacroExpansionKind::Conditional(ConditionalExpansion::if_single(arguments)) + } + MacroKind::IfMulti => { + MacroExpansionKind::Conditional(ConditionalExpansion::if_multi(arguments)) + } }; Ok(MacroExpansion::new( self.context(), diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index f67bea27..c64f355b 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -222,6 +222,39 @@ impl<'top, D: Decoder> MacroExpr<'top, D> { } } + /// Returns an [`ExpansionCardinality`] indicating whether this macro invocation will expand + /// to a stream that is empty, a singleton, or multiple values. + pub(crate) fn expansion_cardinality( + &self, + environment: Environment<'top, D>, + ) -> IonResult { + // If we've statically determined this to produce exactly one value, + // the expansion cardinality must be `Single`. + if self.expansion_analysis().must_produce_exactly_one_value() { + return Ok(ExpansionCardinality::Single); + } + // If it's an empty arg group, the expansion cardinality is `None`. + let is_empty = match self.kind() { + MacroExprKind::EExpArgGroup(group) => group.expressions().next().is_none(), + MacroExprKind::TemplateArgGroup(group) => group.arg_expressions().is_empty(), + _ => false, + }; + if is_empty { + return Ok(ExpansionCardinality::None); + } + + // Otherwise, we need to begin expanding the invocation to see how many values it will produce. + let mut evaluator = MacroEvaluator::new_with_environment(environment); + evaluator.push(self.expand()?); + if evaluator.next()?.is_none() { + return Ok(ExpansionCardinality::None); + } + if evaluator.next()?.is_none() { + return Ok(ExpansionCardinality::Single); + } + Ok(ExpansionCardinality::Multi) + } + pub(crate) fn context(&self) -> EncodingContextRef<'top> { use MacroExprKind::*; match self.kind { @@ -429,6 +462,14 @@ pub enum MacroExpansionKind<'top, D: Decoder> { Annotate(AnnotateExpansion<'top, D>), Flatten(FlattenExpansion<'top, D>), Template(TemplateExpansion<'top>), + // `if_none`, `if_single`, `if_multi` + Conditional(ConditionalExpansion<'top, D>), +} + +pub enum ExpansionCardinality { + None, + Single, + Multi, } /// A macro in the process of being evaluated. Stores both the state of the evaluation and the @@ -501,6 +542,7 @@ impl<'top, D: Decoder> MacroExpansion<'top, D> { MakeSymbol(make_symbol_expansion) => make_symbol_expansion.make_text_value(context), Annotate(annotate_expansion) => annotate_expansion.next(context, environment), Flatten(flatten_expansion) => flatten_expansion.next(), + Conditional(cardiality_test_expansion) => cardiality_test_expansion.next(environment), // `none` is trivial and requires no delegation None => Ok(MacroExpansionStep::FinalStep(Option::None)), } @@ -516,6 +558,7 @@ impl Debug for MacroExpansion<'_, D> { MacroExpansionKind::MakeSymbol(_) => "make_symbol", MacroExpansionKind::Annotate(_) => "annotate", MacroExpansionKind::Flatten(_) => "flatten", + MacroExpansionKind::Conditional(test) => test.name(), MacroExpansionKind::Template(t) => { return if let Some(name) = t.template.name() { write!(f, "", name) @@ -974,6 +1017,101 @@ impl<'top, D: Decoder> ExprGroupExpansion<'top, D> { } } +// ===== Implementation of `if_none`, `if_some`, `if_single`, `if_multi` ===== +#[derive(Debug)] +pub struct ConditionalExpansion<'top, D: Decoder> { + test_kind: CardinalityTestKind, + arguments: MacroExprArgsIterator<'top, D>, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +// The compiler objects to each variant having the same prefix (`If`) because this is often a sign +// that the enum name is being repeated. In this case, removing the prefix causes `None` and `Some` +// to collide with the `Option` variants that are already in the prelude. +#[allow(clippy::enum_variant_names)] +pub enum CardinalityTestKind { + IfNone, + IfSome, + IfSingle, + IfMulti, +} + +impl<'top, D: Decoder> ConditionalExpansion<'top, D> { + pub fn new(kind: CardinalityTestKind, arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self { + test_kind: kind, + arguments, + } + } + + pub fn if_none(arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self::new(CardinalityTestKind::IfNone, arguments) + } + + pub fn if_some(arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self::new(CardinalityTestKind::IfSome, arguments) + } + + pub fn if_single(arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self::new(CardinalityTestKind::IfSingle, arguments) + } + + pub fn if_multi(arguments: MacroExprArgsIterator<'top, D>) -> Self { + Self::new(CardinalityTestKind::IfMulti, arguments) + } + + pub fn name(&self) -> &str { + use CardinalityTestKind::*; + match self.test_kind { + IfNone => "if_none", + IfSome => "if_some", + IfSingle => "if_single", + IfMulti => "if_multi", + } + } + + pub fn next( + &mut self, + environment: Environment<'top, D>, + ) -> IonResult> { + // These are errors are `unreachable!` because all three arguments are zero-or-more. + // In text, `(.if_none)` would be passing three empty streams due to rest syntax. + // In binary, not passing arguments at all would be invalid data. Setting the empty stream + // bits in the bitmap would be close, but then you'd still have three empty streams. + let expr_to_test = self.arguments.next().transpose()?.unwrap_or_else(|| { + unreachable!( + "macro `{}` was not given an expression to test", + self.name() + ) + }); + let true_expr = self.arguments.next().transpose()?.unwrap_or_else(|| { + unreachable!("macro `{}` was not given a `true` expression", self.name()) + }); + let true_result = Ok(MacroExpansionStep::FinalStep(Some(true_expr))); + + let cardinality = match expr_to_test { + ValueExpr::ValueLiteral(_) => ExpansionCardinality::Single, + ValueExpr::MacroInvocation(invocation) => { + invocation.expansion_cardinality(environment)? + } + }; + use CardinalityTestKind::*; + use ExpansionCardinality as EC; + match (self.test_kind, cardinality) { + (IfNone, EC::None) + | (IfSome, EC::Single | EC::Multi) + | (IfSingle, EC::Single) + | (IfMulti, EC::Multi) => true_result, + _ => { + let false_expr = self.arguments.next().transpose()?.unwrap_or_else(|| { + unreachable!("macro `{}` was not given a `false` expression", self.name()) + }); + Ok(MacroExpansionStep::FinalStep(Some(false_expr))) + } + } + } +} + // ===== Implementation of the `make_string` macro ===== #[derive(Copy, Clone, Debug)] @@ -2549,6 +2687,107 @@ mod tests { stream_eq(e_expression, r#" [1, 2, 3, 4, 5, 6, 7] "#) } + #[test] + fn default_eexp() -> IonResult<()> { + stream_eq( + r#" + (:add_macros + (macro foo (x?) + (.make_string "Hello, " (.default (%x) "World!")))) + (:foo "Gary") + (:foo) + "#, + r#" + "Hello, Gary" + "Hello, World!" + "#, + ) + } + + #[test] + fn special_form_if_none() -> IonResult<()> { + stream_eq( + r#" + (:add_macros + (macro foo (x*) + (.make_string "Hello, " (.if_none (%x) "world!" (%x))))) + (:foo "Gary") + (:foo "Gary" " and " "Susan") + (:foo (:flatten ["Tina", " and ", "Lisa"])) + (:foo) + "#, + r#" + "Hello, Gary" + "Hello, Gary and Susan" + "Hello, Tina and Lisa" + "Hello, world!" + "#, + ) + } + + #[test] + fn special_form_if_some() -> IonResult<()> { + stream_eq( + r#" + (:add_macros + (macro foo (x*) + (.make_string "Hello, " (.if_some (%x) (%x) "world!" )))) + (:foo "Gary") + (:foo "Gary" " and " "Susan") + (:foo (:flatten ["Tina", " and ", "Lisa"])) + (:foo) + "#, + r#" + "Hello, Gary" + "Hello, Gary and Susan" + "Hello, Tina and Lisa" + "Hello, world!" + "#, + ) + } + + #[test] + fn special_form_if_single() -> IonResult<()> { + stream_eq( + r#" + (:add_macros + (macro snack (x*) + { + fruit: (.if_single (%x) (%x) [(%x)] ) + })) + (:snack) + (:snack "apple") + (:snack "apple" "banana" "cherry") + "#, + r#" + {fruit: []} + {fruit: "apple"} + {fruit: ["apple", "banana", "cherry"]} + "#, + ) + } + + #[test] + fn special_form_if_multi() -> IonResult<()> { + stream_eq( + r#" + (:add_macros + (macro snack (x*) + { + fruit: (.if_multi (%x) [(%x)] (%x) ) + })) + (:snack) + (:snack "apple") + (:snack "apple" "banana" "cherry") + "#, + r#" + {} + {fruit: "apple"} + {fruit: ["apple", "banana", "cherry"]} + "#, + ) + } + #[test] fn make_string_tdl_macro_invocation() -> IonResult<()> { let invocation = r#" diff --git a/src/lazy/expanded/macro_table.rs b/src/lazy/expanded/macro_table.rs index 25f9ddae..27d4c43e 100644 --- a/src/lazy/expanded/macro_table.rs +++ b/src/lazy/expanded/macro_table.rs @@ -117,6 +117,10 @@ pub enum MacroKind { Annotate, Flatten, Template(TemplateBody), + IfNone, + IfSome, + IfSingle, + IfMulti, // A placeholder for not-yet-implemented macros ToDo, } @@ -406,11 +410,8 @@ impl MacroTable { MacroKind::ToDo, ExpansionAnalysis::single_application_value(IonType::Struct), ), - builtin( - "default", - "(expr* default_expr*)", - MacroKind::ToDo, - ExpansionAnalysis::no_assertions_made(), + template( + "(macro default (expr* default_expr*) (.if_none (%expr) (%default_expr) (%expr) ))", ), ] } diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index fae796de..d4233159 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -8,7 +8,7 @@ use compact_str::CompactString; use crate::lazy::binary::raw::v1_1::immutable_buffer::ArgGroupingBitmap; use crate::lazy::decoder::Decoder; use crate::lazy::expanded::compiler::ExpansionAnalysis; -use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion, FlattenExpansion}; +use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion, FlattenExpansion, ConditionalExpansion}; use crate::lazy::expanded::macro_table::{Macro, MacroKind}; use crate::lazy::expanded::r#struct::UnexpandedField; use crate::lazy::expanded::sequence::Environment; @@ -1230,6 +1230,11 @@ impl<'top, D: Decoder> TemplateMacroInvocation<'top, D> { return Ok(MacroExpansion::new(self.context(), new_environment, kind)); } MacroKind::ToDo => todo!("system macro {}", macro_ref.name().unwrap()), + + MacroKind::IfNone => MacroExpansionKind::Conditional(ConditionalExpansion::if_none(arguments)), + MacroKind::IfSome => MacroExpansionKind::Conditional(ConditionalExpansion::if_some(arguments)), + MacroKind::IfSingle => MacroExpansionKind::Conditional(ConditionalExpansion::if_single(arguments)), + MacroKind::IfMulti => MacroExpansionKind::Conditional(ConditionalExpansion::if_multi(arguments)), }; Ok(MacroExpansion::new( self.context(), From c576f2e5c4f800feac3f6bc7b8e8645307c8c3a1 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 11 Dec 2024 07:41:38 -0500 Subject: [PATCH 2/2] Capitalization fix Co-authored-by: Joshua Barr <70981087+jobarr-amzn@users.noreply.github.com> --- src/lazy/expanded/macro_evaluator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index c64f355b..27e6f5ca 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -2693,13 +2693,13 @@ mod tests { r#" (:add_macros (macro foo (x?) - (.make_string "Hello, " (.default (%x) "World!")))) + (.make_string "Hello, " (.default (%x) "world!")))) (:foo "Gary") (:foo) "#, r#" "Hello, Gary" - "Hello, World!" + "Hello, world!" "#, ) }