Memory relocation

.github/workflows/ci.yml

@@ -12,6 +12,9 @@ jobs:
       - uses: dtolnay/[email protected]
       - uses: actions/checkout@v3
       - run: make check_fmt
+      - run: sudo apt install cpputest -y
+      - run: make Collections-C
+      - run: sudo ldconfig
       - run: make test
       - run: sudo apt install valgrind -y
       - run: make SANITIZER_FLAGS=-fno-omit-frame-pointer valgrind

.gitignore

@@ -54,3 +54,5 @@ dkms.conf
 build/
 .vscode/
 .gdb_history
+
+Collections-C

Dockerfile

@@ -1,6 +1,8 @@
 FROM rust:latest
 
 RUN apt update && \
+    apt install cmake -y && \
+    apt install cpputest -y && \
     apt install valgrind -y
 
 WORKDIR /usr/cairo-vm_in_C

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean fmt check_fmt valgrind compile_rust deps_macos docker_build docker_run docker_test_and_format docker_clean
+.PHONY: clean fmt check_fmt valgrind compile_rust deps_macos build_collections_lib docker_build docker_run docker_test_and_format docker_clean docker_build_collections_lib docker_valgrind
 
 TARGET=cairo_vm
 TEST_TARGET=cairo_vm_test
@@ -9,7 +9,7 @@ SANITIZER_FLAGS=-fsanitize=address -fno-omit-frame-pointer
 CFLAGS=-std=c11 -Wall -Wextra -Wimplicit-fallthrough -Werror -pedantic -g -O0
 CXX_FLAGS=-std=c++14 -Wall -Wextra -Wimplicit-fallthrough -Werror -pedantic -g -O0
 CFLAGS_TEST=-I./src
-LN_FLAGS=-L./lambdaworks/lib/lambdaworks/target/release/ -Bstatic -llambdaworks -ldl -lpthread -lm
+LN_FLAGS=-L./lambdaworks/lib/lambdaworks/target/release/ -Bstatic -llambdaworks -lcollectc -ldl -lpthread -lm
 
 BUILD_DIR=./build
 SRC_DIR=./src
@@ -30,6 +30,16 @@ OBJECTS_CPP = $(patsubst $(SRC_DIR)/%.cpp, $(BUILD_DIR)/%.o, $(SOURCE_CPP))
 TEST_OBJECTS = $(patsubst $(TEST_DIR)/%.c, $(BUILD_DIR)/%.o, $(TEST_SOURCE))
 LIB_OBJECTS_CPP = $(patsubst $(LIB_DIR)/%.cpp, $(BUILD_DIR)/%.o, $(LIB_SOURCE_CPP))
 
+COLLECTIONS_LIB_DIR= Collections-C
+$(COLLECTIONS_LIB_DIR):
+	git clone https://github.com/srdja/Collections-C.git
+	cd Collections-C && \
+	mkdir build && \
+	cd build && \
+	cmake .. && \
+	make && \
+	sudo make install
+
 # Gcc/Clang will create these .d files containing dependencies.
 DEP = $(OBJECTS:%.o=%.d)
 
@@ -77,11 +87,12 @@ deps_macos:
 run: default
 	$(BUILD_DIR)/$(TARGET)
 
-test: compile_rust $(TEST_TARGET)
+test: compile_rust build_collections_lib $(TEST_TARGET)
 	$(BUILD_DIR)/$(TEST_TARGET)
 
 clean:
 	rm -rf $(BUILD_DIR) && \
+	rm -rf $(COLLECTIONS_LIB_DIR) && \
 	cd lambdaworks/lib/lambdaworks && cargo clean
 
 fmt:
@@ -90,12 +101,14 @@ fmt:
 check_fmt:
 	clang-format --style=file -Werror -n $(SOURCE) $(TEST_SOURCE) $(HEADERS) $(TEST_HEADERS)
 
-valgrind: clean compile_rust $(TEST_TARGET)
+valgrind: clean compile_rust build_collections_lib $(TEST_TARGET)
 	valgrind --leak-check=full --show-reachable=yes --show-leak-kinds=all --track-origins=yes --error-exitcode=1 ./build/cairo_vm_test
 
 compile_rust: 
 	cd lambdaworks/lib/lambdaworks && cargo build --release
 
+build_collections_lib: | $(COLLECTIONS_LIB_DIR)
+
 docker_build:
 	docker build . -t cairo-vm_in_c
 
@@ -105,9 +118,23 @@ docker_run:
 docker_test_and_format:
 	docker create --name test -t -v `pwd`:/usr/cairo-vm_in_C cairo-vm_in_c
 	docker start test
-	docker exec -t test bash -c "make && make test && make SANITIZER_FLAGS=-fno-omit-frame-pointer valgrind"
+	docker exec -t test bash -c "make clean && make docker_build_collections_lib && make test && make SANITIZER_FLAGS=-fno-omit-frame-pointer docker_valgrind"
 	docker stop test
 	docker rm test
 
 docker_clean:
 	docker rmi cairo-vm_in_c
+
+docker_build_collections_lib:
+	git clone https://github.com/srdja/Collections-C.git
+	cd Collections-C && \
+	mkdir build && \
+	cd build && \
+	cmake .. && \
+	make && \
+	make install && \
+	ldconfig
+
+docker_valgrind: clean compile_rust docker_build_collections_lib $(TEST_TARGET)
+	valgrind --leak-check=full --show-reachable=yes --show-leak-kinds=all --track-origins=yes --error-exitcode=1 ./build/cairo_vm_test
+

README.md

@@ -10,6 +10,9 @@ This is a work in progress implementation of the [Cairo VM](https://github.com/l
 - A working C compiler with `C11` support.
 - `clang-format` installed in the system.
 - [Rust](https://www.rust-lang.org/tools/install)
+- [cmake](https://cmake.org/install/)
+- pkg-config
+- [cpputest](http://cpputest.github.io)
 - Docker (for valgrind on MacOS)
 
 ## Local development
@@ -38,6 +41,12 @@ To remove all compilation objects:
 make clean
 ```
 
+Note: When building outside of macos, you may have to run this command after building the collections lib to make the system's runtime aware of the location of the new library:
+
+```
+sudo ldconfig
+```
+
 ### Valgrind/Asan on MacOS
 
 To run `valgrind` on MacOS, first run:
@@ -57,7 +66,7 @@ This will run a new container from the built image and will execute bash by usin
 Finally, run:
 
 ```
-make SANITIZER_FLAGS=-fno-omit-frame-pointer valgrind
+make SANITIZER_FLAGS=-fno-omit-frame-pointer docker_valgrind
 ```
 
 ## Tests

src/cairo_runner.c

@@ -1,20 +1,35 @@
 #include "cairo_runner.h"
+#include "collectc/cc_common.h"
+#include "collectc/cc_hashtable.h"
+#include "memory.h"
 #include "program.h"
 #include "relocatable.h"
 #include "vm.h"
+#include <assert.h>
+#include <collectc/cc_array.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 cairo_runner runner_new(struct program program) {
 	virtual_machine vm = vm_new();
-	cairo_runner runner = {program, vm, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
+	CC_Array *relocated_memory;
+	cc_array_new(&relocated_memory);
+	CC_Array *relocation_table;
+	cc_array_new(&relocation_table);
+	cairo_runner runner = {program, vm, relocated_memory, relocation_table, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
 	return runner;
 }
 
 void runner_free(cairo_runner *runner) {
 	memory_free(&runner->vm.memory);
 	program_free(&runner->program);
+	cc_array_remove_all_free(runner->relocated_memory);
+	cc_array_destroy(runner->relocated_memory);
+	cc_array_remove_all_free(runner->relocation_table);
+	cc_array_destroy(runner->relocation_table);
 }
 
-// Creates program, execution and builtin segments
+// Creates program, execution and builtin segments.
 void runner_initialize_segments(cairo_runner *runner) {
 	// Program Segment
 	runner->program_base = memory_add_segment(&runner->vm.memory);
@@ -25,7 +40,7 @@ void runner_initialize_segments(cairo_runner *runner) {
 
 // Initializes the program segment & initial pc
 // Warning: Can fail if used outside of runner_initialize
-void runner_initialize_state(cairo_runner *runner, unsigned int entrypoint, CList *stack) {
+void runner_initialize_state(cairo_runner *runner, unsigned int entrypoint, CC_Array *stack) {
 	runner->initial_pc = runner->program_base;
 	runner->initial_pc.offset += entrypoint;
 	// We ignore the error cases when loading data as these cases are unreachable when called after
@@ -39,15 +54,15 @@ void runner_initialize_state(cairo_runner *runner, unsigned int entrypoint, CLis
 // Initializes memory, initial register values & returns the end pointer (final pc) to run from a given pc offset
 // (entrypoint)
 // Warning: Can fail if runner_initialize_segments is not called before
-relocatable runner_initialize_function_entrypoint(cairo_runner *runner, unsigned int entrypoint, CList *stack,
+relocatable runner_initialize_function_entrypoint(cairo_runner *runner, unsigned int entrypoint, CC_Array *stack,
                                                   relocatable return_fp) {
 	relocatable end = memory_add_segment(&runner->vm.memory);
 	maybe_relocatable return_fp_mr = {.is_felt = false, .value = {.relocatable = return_fp}};
 	maybe_relocatable end_mr = {.is_felt = false, .value = {.relocatable = end}};
-	stack->add(stack, &return_fp_mr);
-	stack->add(stack, &end_mr);
+	cc_array_add(stack, &return_fp_mr);
+	cc_array_add(stack, &end_mr);
 	runner->initial_fp.segment_index = runner->execution_base.segment_index;
-	runner->initial_fp.offset = stack->count(stack);
+	runner->initial_fp.offset = cc_array_size(stack);
 	runner_initialize_state(runner, entrypoint, stack);
 	runner->final_pc = end;
 	return end;
@@ -56,12 +71,13 @@ relocatable runner_initialize_function_entrypoint(cairo_runner *runner, unsigned
 // Initializes memory, initial register values & returns the end pointer (final pc) to run from the main entrypoint
 // Warning: Can fail if runner_initialize_segments is not called before
 relocatable runner_initialize_main_entrypoint(cairo_runner *runner) {
-	CList *stack = CList_init(sizeof(maybe_relocatable));
+	CC_Array *stack;
+	cc_array_new(&stack);
 	// Handle builtin initial stack
 	// Handle proof-mode specific behaviour
 	relocatable return_fp = memory_add_segment(&runner->vm.memory);
 	relocatable end = runner_initialize_function_entrypoint(runner, runner->program.main, stack, return_fp);
-	stack->free(stack);
+	cc_array_destroy(stack);
 	return end;
 }
 
@@ -74,10 +90,110 @@ void runner_initialize_vm(cairo_runner *runner) {
 	// Apply validation rules to memory
 }
 
+void runner_get_segment_sizes(cairo_runner *runner, CC_Array *segment_sizes) {
+	int num_segments = runner->vm.memory.num_segments;
+	unsigned int *zero;
+	for (int i = 0; i < num_segments; i++) {
+		zero = malloc(sizeof(unsigned int));
+		*zero = 0;
+		cc_array_add(segment_sizes, zero);
+	}
+
+	unsigned int *new_size;
+	CC_HashTableIter memory_iter;
+	cc_hashtable_iter_init(&memory_iter, runner->vm.memory.data);
+	TableEntry *entry;
+	while (cc_hashtable_iter_next(&memory_iter, &entry) != CC_ITER_END) {
+		relocatable *ptr = entry->key;
+		unsigned int *segment_size;
+		assert(cc_array_get_at(segment_sizes, ptr->segment_index, (void **)&segment_size) == CC_OK);
+
+		if (*segment_size < ptr->offset + 1) {
+			new_size = malloc(sizeof(unsigned int));
+			*new_size = ptr->offset + 1;
+			cc_array_replace_at(segment_sizes, new_size, ptr->segment_index, NULL);
+		}
+	}
+}
+
+// Fills relocation table with the first relocated address of each memory segment
+// Warning: Can fail if runner_initialize_main_entrypoint is not called before
+void runner_fill_relocation_table(cairo_runner *runner, CC_Array *segment_sizes) {
+	int *first_addr = malloc(sizeof(int));
+	*first_addr = 1;
+	cc_array_add(runner->relocation_table, first_addr);
+	int *last_segment_size_added;
+	assert(cc_array_get_at(runner->relocation_table, 0, (void **)&last_segment_size_added) == CC_OK);
+
+	int relocation_table_size = runner->vm.memory.num_segments + 1;
+	for (int i = 1; i < relocation_table_size; i++) {
+		int segment_idx = i - 1;
+		int *segment_size;
+		assert(cc_array_get_at(segment_sizes, segment_idx, (void **)&segment_size) == CC_OK);
+		int *address = malloc(sizeof(int));
+		*address = *last_segment_size_added + *segment_size;
+		cc_array_add(runner->relocation_table, address);
+		assert(cc_array_get_at(runner->relocation_table, i, (void **)&last_segment_size_added) == CC_OK);
+	}
+}
+
 relocatable runner_initialize(cairo_runner *runner) {
 	// runner_initialize_builtins
 	runner_initialize_segments(runner);
 	relocatable end = runner_initialize_main_entrypoint(runner);
 	runner_initialize_vm(runner);
 	return end;
 }
+
+int runner_get_relocated_address(cairo_runner *runner, relocatable *relocatable) {
+	int *address;
+	assert(cc_array_get_at(runner->relocation_table, relocatable->segment_index, (void **)&address) == CC_OK);
+	return *address + relocatable->offset;
+}
+
+void runner_get_relocated_value(cairo_runner *runner, maybe_relocatable *value, felt_t out) {
+	if (value->is_felt) {
+		zero(out);
+		add(out, value->value.felt, out);
+	} else {
+		int out_number = runner_get_relocated_address(runner, &value->value.relocatable);
+		from(out, out_number);
+	}
+}
+
+void runner_relocate_memory(cairo_runner *runner) {
+	assert(cc_array_size(runner->relocated_memory) == 0);
+	relocated_memory_cell *first_cell = malloc(sizeof(relocated_memory_cell));
+	first_cell->is_some = false;
+	first_cell->memory_value.none = 0;
+	cc_array_add(runner->relocated_memory, first_cell);
+
+	CC_Array *segment_sizes;
+	cc_array_new(&segment_sizes);
+	runner_get_segment_sizes(runner, segment_sizes);
+	runner_fill_relocation_table(runner, segment_sizes);
+
+	int num_segment_sizes = cc_array_size(segment_sizes);
+
+	for (int i = 0; i < num_segment_sizes; i++) {
+		int *segment_size;
+		assert(cc_array_get_at(segment_sizes, i, (void **)&segment_size) == CC_OK);
+
+		for (int j = 0; j < *segment_size; j++) {
+			relocatable ptr = {i, j};
+			maybe_relocatable *cell;
+			if (cc_hashtable_get(runner->vm.memory.data, &ptr, (void **)&cell) == CC_OK) {
+				int relocated_addr = runner_get_relocated_address(runner, &ptr);
+				relocated_memory_cell *relocated_value = malloc(sizeof(relocated_memory_cell));
+				relocated_value->is_some = true;
+				runner_get_relocated_value(runner, cell, relocated_value->memory_value.value);
+				cc_array_add_at(runner->relocated_memory, relocated_value, relocated_addr);
+			} else {
+				relocated_memory_cell *none = malloc(sizeof(relocated_memory_cell));
+				none->is_some = false;
+				none->memory_value.none = 0;
+				cc_array_add(runner->relocated_memory, none);
+			}
+		}
+	}
+}

src/cairo_runner.h

@@ -1,14 +1,26 @@
 #ifndef RUNNER_H
 #define RUNNER_H
 
-#include "clist.h"
+#include "collectc/cc_array.h"
 #include "program.h"
 #include "relocatable.h"
 #include "vm.h"
 
+typedef union relocated_memory_value {
+	felt_t value;
+	int none;
+} relocated_memory_value;
+
+typedef struct relocated_memory_cell {
+	relocated_memory_value memory_value;
+	bool is_some;
+} relocated_memory_cell;
+
 typedef struct cairo_runner {
 	struct program program;
 	virtual_machine vm;
+	CC_Array *relocated_memory;
+	CC_Array *relocation_table;
 	relocatable program_base;
 	relocatable execution_base;
 	relocatable initial_pc;
@@ -26,4 +38,6 @@ void runner_free(cairo_runner *runner);
 // Performs the intialization step, leaving the runner ready to run a loaded cairo program from a main entrypoint
 relocatable runner_initialize(cairo_runner *runner);
 
+void runner_relocate_memory(cairo_runner *runner);
+
 #endif