[compiler-v2] Enum types in the VM

In #13725 support for enums was added to the stackless bytecode IR. This PR extends the Move VM's representation of bytecode ('file format') by struct variants. It also implements

- representation in file format
- serialization/deserialization of file format
- bytecode verifier
- code generation in compiler v2
- intepreter and paranoid mode

The runtime semantics (intepreter, runtime types, compatibility checking), as well as certain other features (as marked by #13806 in the code), are not yet implemented by this PR.

On Move bytecode level, there are `5 * 2` new instructions (each instruction has a dual generic version):

```
TestVariant(StructVariantHandleIndex)
	and TestVariantGeneric(StructVariantInstantiationIndex)
PackVariant(StructVariantHandleIndex)
	and PackVariantGeneric(StructVariantInstantiationIndex)
UnpackVariant(StructVariantHandleIndex)
	and UnpackVariantGeneric(StructVariantInstantiationIndex)
ImmBorrowVariantField(VariantFieldHandleIndex)
	and ImmBorrowVariantFieldGeneric(VariantFieldInstantiationIndex)
MutBorrowVariantField(VariantFieldHandleIndex)
	and MutBorrowVariantFieldGeneric(VariantFieldInstantiationIndex)
```

For the indices used in those instructions, 4 new tables have been added to the file format holding the associated data.

There is a lot of boilerplate code to support the new instructions and tables. Some refactoring of existing code has been done to avoid too much copy and paste, specifically in the serializers and in the bytecode verifier.

Apart of passing existing tests, there is a new test in move-compiler-v2/tests/file-format-generator/struct_variants.move which shows the disassembeled output of various match expressions.

There are also new e2e transactional tests in move-compiler-v2/transactional-tests/tests/enun. To add negative tests for the bytecode verifier and serializers, we first need a better way to build code with injected faults. See also #14074 and #13812.

api/types/src/bytecode.rs

@@ -109,6 +109,10 @@ pub trait Bytecode {
                     .map(|f| self.new_move_struct_field(f))
                     .collect(),
             ),
+            StructFieldInformation::DeclaredVariants(..) => {
+                // TODO(#13806): implement for enums. Currently we pretend they don't have fields
+                (false, vec![])
+            },
         };
         let name = self.identifier_at(handle.name).to_owned();
         let abilities = handle

aptos-move/aptos-gas-meter/src/meter.rs

@@ -101,6 +101,13 @@ where
             MutBorrowField => MUT_BORROW_FIELD,
             ImmBorrowFieldGeneric => IMM_BORROW_FIELD_GENERIC,
             MutBorrowFieldGeneric => MUT_BORROW_FIELD_GENERIC,
+            ImmBorrowVariantField => IMM_BORROW_VARIANT_FIELD,
+            MutBorrowVariantField => MUT_BORROW_VARIANT_FIELD,
+            ImmBorrowVariantFieldGeneric => IMM_BORROW_VARIANT_FIELD_GENERIC,
+            MutBorrowVariantFieldGeneric => MUT_BORROW_VARIANT_FIELD_GENERIC,
+            TestVariant => TEST_VARIANT,
+            TestVariantGeneric => TEST_VARIANT_GENERIC,
+
             FreezeRef => FREEZE_REF,
 
             CastU8 => CAST_U8,

aptos-move/aptos-gas-schedule/src/gas_schedule/instr.rs

@@ -45,6 +45,15 @@ crate::gas_schedule::macros::define_gas_parameters!(
         [mut_borrow_field: InternalGas, "mut_borrow_field", 735],
         [imm_borrow_field_generic: InternalGas, "imm_borrow_field_generic", 735],
         [mut_borrow_field_generic: InternalGas, "mut_borrow_field_generic", 735],
+        [imm_borrow_variant_field: InternalGas, "imm_borrow_variant_field", 835],
+        [mut_borrow_variant_field: InternalGas, "mut_borrow_variant_field", 835],
+        [imm_borrow_variant_field_generic: InternalGas, "imm_borrow_variant_field_generic", 835],
+        [mut_borrow_variant_field_generic: InternalGas, "mut_borrow_variant_field_generic", 835],
+
+        // variant testing
+        [test_variant: InternalGas, "test_variant", 835],
+        [test_variant_generic: InternalGas, "test_variant_generic", 835],
+
         // locals
         [copy_loc_base: InternalGas, "copy_loc.base", 294],
         [copy_loc_per_abs_val_unit: InternalGasPerAbstractValueUnit, "copy_loc.per_abs_val_unit", 14],

aptos-move/e2e-move-tests/src/tests/access_path_test.rs

@@ -90,6 +90,10 @@ fn access_path_panic() {
                 ],
             }),
         }],
+        struct_variant_handles: vec![],
+        struct_variant_instantiations: vec![],
+        variant_field_handles: vec![],
+        variant_field_instantiations: vec![],
     };
 
     let mut module_bytes = vec![];

aptos-move/framework/src/module_metadata.rs

@@ -431,7 +431,7 @@ pub fn is_valid_resource_group(
     if let Ok(ident_struct) = Identifier::new(struct_) {
         if let Some((struct_handle, struct_def)) = structs.get(ident_struct.as_ident_str()) {
             let num_fields = match &struct_def.field_information {
-                StructFieldInformation::Native => 0,
+                StructFieldInformation::Native | StructFieldInformation::DeclaredVariants(_) => 0,
                 StructFieldInformation::Declared(fields) => fields.len(),
             };
             if struct_handle.abilities == AbilitySet::EMPTY

aptos-move/framework/src/natives/string_utils.rs

@@ -308,6 +308,31 @@ fn native_format_impl(
             )?;
             out.push('}');
         },
+        MoveTypeLayout::Struct(MoveStructLayout::RuntimeVariants(variants)) => {
+            let strct = val.value_as::<Struct>()?;
+            let mut elems = strct.unpack()?.collect::<Vec<_>>();
+            if elems.is_empty() {
+                return Err(SafeNativeError::Abort {
+                    abort_code: EARGS_MISMATCH,
+                });
+            }
+            let tag = elems.pop().unwrap().value_as::<u32>()? as usize;
+            if tag >= variants.len() {
+                return Err(SafeNativeError::Abort {
+                    abort_code: EINVALID_FORMAT,
+                });
+            }
+            out.push_str(&format!("#{}{{", tag));
+            format_vector(
+                context,
+                variants[tag].iter(),
+                elems,
+                depth,
+                !context.single_line,
+                out,
+            )?;
+            out.push('}');
+        },
 
         // This is unreachable because we check layout at the start. Still, return
         // an error to be safe.

third_party/move/evm/move-to-yul/src/functions.rs

@@ -551,7 +551,7 @@ impl<'a> FunctionGenerator<'a> {
                     TestVariant(..)
                     | PackVariant(..)
                     | UnpackVariant(..)
-                    | BorrowFieldVariant(..) => {
+                    | BorrowVariantField(..) => {
                         unimplemented!("variants")
                     },
 

third_party/move/move-binary-format/serializer-tests/tests/serializer_tests.proptest-regressions

@@ -7,3 +7,4 @@
 cc 3d00be1cbcb9f344e7bce080015d7a755f6e69acd37dbde62e449732af226fe4 # shrinks to module = CompiledModule: { module_handles: [ 	ModuleHandle { address: AddressPoolIndex(0), name: IdentifierIndex(0) },] struct_handles: [ 	StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(0), is_resource: false }, 	StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(0), is_resource: false },] function_handles: [ 	FunctionHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(0), signature: FunctionSignatureIndex(0) }, 	FunctionHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(0), signature: FunctionSignatureIndex(1) },] struct_defs: [ 	StructDefinition { struct_handle: 1, field_count: 0, fields: 0 },] field_defs: [] function_defs: [ 	FunctionDefinition { function: 1, flags: 0x0, 		code: CodeUnit { 			max_stack_size: 0, 			locals: 0 			code: [] } },] type_signatures: [ 	TypeSignature(Unit), 	TypeSignature(Unit),] function_signatures: [ 	FunctionSignature { return_type: Unit, arg_types: [] }, 	FunctionSignature { return_type: Unit, arg_types: [] },] locals_signatures: [ 	LocalsSignature([]),] string_pool: [ 	"",] address_pool: [ 	Address([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),] }
 cc 7b1bb969b87bfcdbb0f635eb46212f8437d21bcd1ba754de84d66bb552e6aec2 # shrinks to module = CompiledModule { module_handles: [], struct_handles: [], function_handles: [], type_signatures: [], function_signatures: [], locals_signatures: [], string_pool: [], byte_array_pool: [], address_pool: [0000000000000000000000000000000000000000000000000000000000000000, 0000000000000000000000000000000000000000000000000000000000000000], struct_defs: [], field_defs: [], function_defs: [] }
 cc 4118fc247fb7d48382876de931d47a8999a6e42658bbecc93afff9245ade141b # shrinks to module = CompiledModule { module_handles: [], struct_handles: [], function_handles: [], type_signatures: [], function_signatures: [], locals_signatures: [], string_pool: [], byte_array_pool: [], address_pool: [], struct_defs: [], field_defs: [FieldDefinition { struct_: StructHandleIndex(0), name: IdentifierIndex(0), signature: TypeSignatureIndex(0) }], function_defs: [] }
+cc 42cd5d033842c708fc3a704a573a77455676f33b4cb987516cee12efeaa33655 # shrinks to module = CompiledModule { version: 7, self_module_handle_idx: ModuleHandleIndex(0), module_handles: [ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(0) }], struct_handles: [StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(7), abilities: [Copy, Store, ], type_parameters: [StructTypeParameter { constraints: [Copy, ], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(16), abilities: [Copy, Drop, Store, ], type_parameters: [StructTypeParameter { constraints: [Drop, ], is_phantom: false }, StructTypeParameter { constraints: [], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(10), abilities: [Copy, ], type_parameters: [] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(15), abilities: [], type_parameters: [StructTypeParameter { constraints: [], is_phantom: true }, StructTypeParameter { constraints: [Copy, Store, ], is_phantom: false }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(12), abilities: [Copy, ], type_parameters: [] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(0), abilities: [Drop, ], type_parameters: [StructTypeParameter { constraints: [Copy, Drop, Store, ], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(5), abilities: [Copy, ], type_parameters: [StructTypeParameter { constraints: [Copy, Store, ], is_phantom: false }, StructTypeParameter { constraints: [], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(13), abilities: [Drop, ], type_parameters: [StructTypeParameter { constraints: [Copy, Drop, ], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(3), abilities: [Drop, ], type_parameters: [StructTypeParameter { constraints: [Drop, Store, ], is_phantom: false }, StructTypeParameter { constraints: [Copy, Drop, Store, ], is_phantom: false }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(4), abilities: [Drop, ], type_parameters: [StructTypeParameter { constraints: [Copy, Drop, Store, ], is_phantom: true }] }, StructHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(6), abilities: [Copy, ], type_parameters: [StructTypeParameter { constraints: [], is_phantom: false }, StructTypeParameter { constraints: [Drop, Store, ], is_phantom: true }] }], function_handles: [FunctionHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(9), parameters: SignatureIndex(2), return_: SignatureIndex(3), type_parameters: [], access_specifiers: None }, FunctionHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(16), parameters: SignatureIndex(5), return_: SignatureIndex(6), type_parameters: [], access_specifiers: None }, FunctionHandle { module: ModuleHandleIndex(0), name: IdentifierIndex(4), parameters: SignatureIndex(9), return_: SignatureIndex(10), type_parameters: [], access_specifiers: None }], field_handles: [], friend_decls: [ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(0) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(2) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(4) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(5) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(10) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(12) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(13) }, ModuleHandle { address: AddressIdentifierIndex(0), name: IdentifierIndex(14) }], struct_def_instantiations: [StructDefInstantiation { def: StructDefinitionIndex(10), type_parameters: SignatureIndex(4) }, StructDefInstantiation { def: StructDefinitionIndex(5), type_parameters: SignatureIndex(7) }, StructDefInstantiation { def: StructDefinitionIndex(8), type_parameters: SignatureIndex(4) }], function_instantiations: [], field_instantiations: [], signatures: [Signature([U32, Bool]), Signature([U16, Address, U32]), Signature([]), Signature([Reference(Bool)]), Signature([U64, U64]), Signature([StructInstantiation(StructHandleIndex(7), [U64]), Reference(U32)]), Signature([U16, Reference(U256)]), Signature([U64]), Signature([StructInstantiation(StructHandleIndex(3), [U64, U64])]), Signature([StructInstantiation(StructHandleIndex(8), [U64, U64]), Vector(Signer), U8]), Signature([U128, MutableReference(Address)]), Signature([Bool])], identifiers: [Identifier("A"), Identifier("A0"), Identifier("AA"), Identifier("AB"), Identifier("AC"), Identifier("AD"), Identifier("AE"), Identifier("A_"), Identifier("Aa"), Identifier("a"), Identifier("a0"), Identifier("a1"), Identifier("a2"), Identifier("a3"), Identifier("aA"), Identifier("a_"), Identifier("aa")], address_identifiers: [0000000000000000000000000000000000000000000000000000000000000000], constant_pool: [], metadata: [], struct_defs: [StructDefinition { struct_handle: StructHandleIndex(0), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(1), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(2), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(3), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(4), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(5), field_information: Declared([FieldDefinition { name: IdentifierIndex(16), signature: TypeSignature(U64) }, FieldDefinition { name: IdentifierIndex(7), signature: TypeSignature(Address) }, FieldDefinition { name: IdentifierIndex(12), signature: TypeSignature(U64) }]) }, StructDefinition { struct_handle: StructHandleIndex(6), field_information: DeclaredVariants([VariantDefinition { name: IdentifierIndex(2), fields: [FieldDefinition { name: IdentifierIndex(5), signature: TypeSignature(U256) }] }]) }, StructDefinition { struct_handle: StructHandleIndex(7), field_information: Native }, StructDefinition { struct_handle: StructHandleIndex(8), field_information: DeclaredVariants([VariantDefinition { name: IdentifierIndex(14), fields: [FieldDefinition { name: IdentifierIndex(11), signature: TypeSignature(StructInstantiation(StructHandleIndex(1), [Signer, U64])) }] }, VariantDefinition { name: IdentifierIndex(10), fields: [] }]) }, StructDefinition { struct_handle: StructHandleIndex(9), field_information: DeclaredVariants([VariantDefinition { name: IdentifierIndex(14), fields: [FieldDefinition { name: IdentifierIndex(13), signature: TypeSignature(Bool) }, FieldDefinition { name: IdentifierIndex(9), signature: TypeSignature(U16) }, FieldDefinition { name: IdentifierIndex(14), signature: TypeSignature(Address) }, FieldDefinition { name: IdentifierIndex(7), signature: TypeSignature(Signer) }] }]) }, StructDefinition { struct_handle: StructHandleIndex(10), field_information: Native }], function_defs: [FunctionDefinition { function: FunctionHandleIndex(0), visibility: Private, is_entry: true, acquires_global_resources: [StructDefinitionIndex(5)], code: Some(CodeUnit { locals: SignatureIndex(2), code: [MutBorrowGlobalGeneric(StructDefInstantiationIndex(0)), Branch(0)] }) }, FunctionDefinition { function: FunctionHandleIndex(1), visibility: Private, is_entry: false, acquires_global_resources: [StructDefinitionIndex(10)], code: Some(CodeUnit { locals: SignatureIndex(8), code: [Call(0), Xor, ImmBorrowVariantFieldGeneric(VariantFieldInstantiationIndex(0)), ExistsGeneric(StructDefInstantiationIndex(1)), ImmBorrowGlobal(StructDefinitionIndex(4)), MoveFromGeneric(StructDefInstantiationIndex(2)), BrFalse(1), ImmBorrowLoc(0), MutBorrowGlobalGeneric(StructDefInstantiationIndex(1)), ImmBorrowLoc(0), ExistsGeneric(StructDefInstantiationIndex(0)), ImmBorrowLoc(0), Exists(StructDefinitionIndex(2)), MutBorrowLoc(0), MoveToGeneric(StructDefInstantiationIndex(1))] }) }, FunctionDefinition { function: FunctionHandleIndex(2), visibility: Friend, is_entry: true, acquires_global_resources: [], code: Some(CodeUnit { locals: SignatureIndex(11), code: [StLoc(0), ImmBorrowVariantFieldGeneric(VariantFieldInstantiationIndex(1))] }) }], struct_variant_handles: [], struct_variant_instantiations: [], variant_field_handles: [VariantFieldHandle { owner: StructDefinitionIndex(9), variants: [0], field: 3 }, VariantFieldHandle { owner: StructDefinitionIndex(6), variants: [0], field: 0 }], variant_field_instantiations: [VariantFieldInstantiation { handle: VariantFieldHandleIndex(0), type_parameters: SignatureIndex(7) }, VariantFieldInstantiation { handle: VariantFieldHandleIndex(1), type_parameters: SignatureIndex(4) }] }

third_party/move/move-binary-format/src/access.rs

@@ -66,6 +66,18 @@ pub trait ModuleAccess: Sync {
         handle
     }
 
+    fn variant_field_handle_at(&self, idx: VariantFieldHandleIndex) -> &VariantFieldHandle {
+        let handle = &self.as_module().variant_field_handles[idx.into_index()];
+        debug_assert!(handle.owner.into_index() < self.as_module().struct_variant_handles.len()); // invariant
+        handle
+    }
+
+    fn struct_variant_handle_at(&self, idx: StructVariantHandleIndex) -> &StructVariantHandle {
+        let handle = &self.as_module().struct_variant_handles[idx.into_index()];
+        debug_assert!(handle.struct_index.into_index() < self.as_module().struct_defs.len()); // invariant
+        handle
+    }
+
     fn struct_instantiation_at(&self, idx: StructDefInstantiationIndex) -> &StructDefInstantiation {
         &self.as_module().struct_def_instantiations[idx.into_index()]
     }
@@ -78,6 +90,20 @@ pub trait ModuleAccess: Sync {
         &self.as_module().field_instantiations[idx.into_index()]
     }
 
+    fn variant_field_instantiation_at(
+        &self,
+        idx: VariantFieldInstantiationIndex,
+    ) -> &VariantFieldInstantiation {
+        &self.as_module().variant_field_instantiations[idx.into_index()]
+    }
+
+    fn struct_variant_instantiation_at(
+        &self,
+        idx: StructVariantInstantiationIndex,
+    ) -> &StructVariantInstantiation {
+        &self.as_module().struct_variant_instantiations[idx.into_index()]
+    }
+
     fn signature_at(&self, idx: SignatureIndex) -> &Signature {
         &self.as_module().signatures[idx.into_index()]
     }
@@ -124,6 +150,14 @@ pub trait ModuleAccess: Sync {
         &self.as_module().field_handles
     }
 
+    fn variant_field_handles(&self) -> &[VariantFieldHandle] {
+        &self.as_module().variant_field_handles
+    }
+
+    fn struct_variant_handles(&self) -> &[StructVariantHandle] {
+        &self.as_module().struct_variant_handles
+    }
+
     fn struct_instantiations(&self) -> &[StructDefInstantiation] {
         &self.as_module().struct_def_instantiations
     }
@@ -136,6 +170,14 @@ pub trait ModuleAccess: Sync {
         &self.as_module().field_instantiations
     }
 
+    fn variant_field_instantiations(&self) -> &[VariantFieldInstantiation] {
+        &self.as_module().variant_field_instantiations
+    }
+
+    fn struct_variant_instantiations(&self) -> &[StructVariantInstantiation] {
+        &self.as_module().struct_variant_instantiations
+    }
+
     fn signatures(&self) -> &[Signature] {
         &self.as_module().signatures
     }