Move plasma, common, and photon into the Ray repository.

.gitmodules

@@ -1,17 +1,3 @@
-[submodule "thirdparty/grpc"]
-	path = thirdparty/grpc
-	url = https://github.com/grpc/grpc
-	ignore = dirty
-[submodule "thirdparty/numbuf"]
-	path = thirdparty/numbuf
-	url = https://github.com/ray-project/numbuf.git
 [submodule "thirdparty/arrow"]
 	path = thirdparty/arrow
 	url = https://github.com/ray-project/arrow.git
-[submodule "thirdparty/python"]
-	path = thirdparty/python
-	url = https://github.com/austinsc/python.git
-	ignore = dirty
-[submodule "thirdparty/hiredis"]
-	path = thirdparty/hiredis
-	url = https://github.com/redis/hiredis.git

src/common/Makefile

@@ -0,0 +1,46 @@
+CC = gcc
+CFLAGS = -g -Wall --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae -Wno-typedef-redefinition -Werror
+BUILD = build
+
+all: hiredis $(BUILD)/libcommon.a
+
+$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o state/redis.o thirdparty/ae/ae.o
+	ar rcs $@ $^
+
+$(BUILD)/common_tests: test/common_tests.c $(BUILD)/libcommon.a
+	$(CC) -o $@ test/common_tests.c $(BUILD)/libcommon.a $(CFLAGS)
+
+$(BUILD)/db_tests: hiredis test/db_tests.c $(BUILD)/libcommon.a
+	$(CC) -o $@ test/db_tests.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
+
+$(BUILD)/io_tests: test/io_tests.c $(BUILD)/libcommon.a
+	$(CC) -o $@ $^ $(CFLAGS)
+
+$(BUILD)/task_tests: test/task_tests.c $(BUILD)/libcommon.a
+	$(CC) -o $@ $^ $(CFLAGS)
+
+$(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h
+	$(CC) -o $@ test/redis_tests.c logging.c $(BUILD)/libcommon.a thirdparty/hiredis/libhiredis.a $(CFLAGS)
+
+clean:
+	rm -f *.o state/*.o test/*.o thirdparty/ae/*.o
+	rm -rf $(BUILD)/*
+
+redis:
+	cd thirdparty ; bash ./build-redis.sh
+
+hiredis:
+	git submodule update --init --recursive -- "thirdparty/hiredis" ; cd thirdparty/hiredis ; make
+
+test: hiredis redis $(BUILD)/common_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE
+	./thirdparty/redis-3.2.3/src/redis-server &
+	sleep 1s ; ./build/common_tests ; ./build/db_tests ; ./build/io_tests ; ./build/task_tests ; ./build/redis_tests
+
+valgrind: test
+	valgrind --leak-check=full --error-exitcode=1 ./build/common_tests
+	valgrind --leak-check=full --error-exitcode=1 ./build/db_tests
+	valgrind --leak-check=full --error-exitcode=1 ./build/io_tests
+	valgrind --leak-check=full --error-exitcode=1 ./build/task_tests
+	valgrind --leak-check=full --error-exitcode=1 ./build/redis_tests
+
+FORCE:

src/common/common.c

@@ -0,0 +1,36 @@
+#include "common.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+const unique_id NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
+                           255, 255, 255, 255, 255, 255, 255, 255, 255, 255}};
+
+unique_id globally_unique_id(void) {
+  /* Use /dev/urandom for "real" randomness. */
+  int fd;
+  if ((fd = open("/dev/urandom", O_RDONLY)) == -1) {
+    LOG_ERR("Could not generate random number");
+  }
+  unique_id result;
+  read(fd, &result.id[0], UNIQUE_ID_SIZE);
+  close(fd);
+  return result;
+}
+
+char *sha1_to_hex(const unsigned char *sha1, char *buffer) {
+  static const char hex[] = "0123456789abcdef";
+  char *buf = buffer;
+
+  for (int i = 0; i < UNIQUE_ID_SIZE; i++) {
+    unsigned int val = *sha1++;
+    *buf++ = hex[val >> 4];
+    *buf++ = hex[val & 0xf];
+  }
+  *buf = '\0';
+
+  return buffer;
+}

src/common/common.h

@@ -0,0 +1,61 @@
+#ifndef COMMON_H
+#define COMMON_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#ifndef RAY_COMMON_DEBUG
+#define LOG_DEBUG(M, ...)
+#else
+#define LOG_DEBUG(M, ...) \
+  fprintf(stderr, "[DEBUG] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__)
+#endif
+
+#define LOG_ERR(M, ...)                                                     \
+  fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, \
+          errno == 0 ? "None" : strerror(errno), ##__VA_ARGS__)
+
+#define LOG_INFO(M, ...) \
+  fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__)
+
+#define CHECK(COND)                        \
+  do {                                     \
+    if (!(COND)) {                         \
+      LOG_ERR("Check failure: %s", #COND); \
+      exit(-1);                            \
+    }                                      \
+  } while (0);
+
+#define CHECKM(COND, M, ...)                                   \
+  do {                                                         \
+    if (!(COND)) {                                             \
+      LOG_ERR("Check failure: %s \n" M, #COND, ##__VA_ARGS__); \
+      exit(-1);                                                \
+    }                                                          \
+  } while (0);
+
+#define UNIQUE_ID_SIZE 20
+
+/* Cleanup method for running tests with the greatest library.
+ * Runs the test, then clears the Redis database. */
+#define RUN_REDIS_TEST(context, test) \
+  RUN_TEST(test);                     \
+  freeReplyObject(redisCommand(context, "FLUSHALL"));
+
+typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } unique_id;
+
+extern const unique_id NIL_ID;
+
+/* Generate a globally unique ID. */
+unique_id globally_unique_id(void);
+
+/* Convert a 20 byte sha1 hash to a hexdecimal string. This function assumes
+ * that buffer points to an already allocated char array of size 2 *
+ * UNIQUE_ID_SIZE + 1 */
+char *sha1_to_hex(const unsigned char *sha1, char *buffer);
+
+typedef unique_id object_id;
+
+#endif

src/common/doc/tasks.md

@@ -0,0 +1,33 @@
+# Task specifications, task instances and task logs
+
+A *task specification* contains all information that is needed for computing
+the results of a task:
+
+- The function ID of the function that executes the task
+- The arguments (either object IDs for pass by reference
+or values for pass by value)
+- The IDs of the result objects
+
+From these, a task ID can be computed which is also stored in the task
+specification.
+
+A *task instance* represents one execution of a task specification.
+It consists of:
+
+- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE)
+- The target node where the task is scheduled or executed
+- A unique task instance ID that identifies the particular execution
+  of the task.
+
+The task data structures are defined in `common/task.h`.
+
+The *task log* is a mapping from the task instance ID to a sequence of
+updates to the status of the task instance. It is updated by various parts
+of the system:
+
+1. The local scheduler writes it with status WAITING when submits a task to the global scheduler
+2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler
+3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker
+4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution
+
+The task log is defined in `common/state/task_log.h`.

src/common/event_loop.c

@@ -0,0 +1,62 @@
+#include "event_loop.h"
+
+#include "common.h"
+#include <errno.h>
+
+#define INITIAL_EVENT_LOOP_SIZE 1024
+
+event_loop *event_loop_create() {
+  return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE);
+}
+
+void event_loop_destroy(event_loop *loop) {
+  /* Clean up timer events. This is to make valgrind happy. */
+  aeTimeEvent *te = loop->timeEventHead;
+  while (te) {
+    aeTimeEvent *next = te->next;
+    free(te);
+    te = next;
+  }
+  aeDeleteEventLoop(loop);
+}
+
+void event_loop_add_file(event_loop *loop,
+                         int fd,
+                         int events,
+                         event_loop_file_handler handler,
+                         void *context) {
+  /* Try to add the file descriptor. */
+  int err = aeCreateFileEvent(loop, fd, events, handler, context);
+  /* If it cannot be added, increase the size of the event loop. */
+  if (err == AE_ERR && errno == ERANGE) {
+    err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2);
+    CHECK(err == AE_OK);
+    err = aeCreateFileEvent(loop, fd, events, handler, context);
+  }
+  /* In any case, test if there were errors. */
+  CHECK(err == AE_OK);
+}
+
+void event_loop_remove_file(event_loop *loop, int fd) {
+  aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE);
+}
+
+int64_t event_loop_add_timer(event_loop *loop,
+                             int64_t milliseconds,
+                             event_loop_timer_handler handler,
+                             void *context) {
+  return aeCreateTimeEvent(loop, milliseconds, handler, context, NULL);
+}
+
+void event_loop_remove_timer(event_loop *loop, timer_id timer_id) {
+  int err = aeDeleteTimeEvent(loop, timer_id);
+  CHECK(err == AE_OK); /* timer id found? */
+}
+
+void event_loop_run(event_loop *loop) {
+  aeMain(loop);
+}
+
+void event_loop_stop(event_loop *loop) {
+  aeStop(loop);
+}

src/common/event_loop.h

@@ -0,0 +1,77 @@
+#ifndef EVENT_LOOP_H
+#define EVENT_LOOP_H
+
+#include <stdint.h>
+#include "ae/ae.h"
+
+typedef long long timer_id;
+
+typedef aeEventLoop event_loop;
+
+/* File descriptor is readable. */
+#define EVENT_LOOP_READ AE_READABLE
+
+/* File descriptor is writable. */
+#define EVENT_LOOP_WRITE AE_WRITABLE
+
+/* Constant specifying that the timer is done and it will be removed. */
+#define EVENT_LOOP_TIMER_DONE AE_NOMORE
+
+/* Signature of the handler that will be called when there is a new event
+ * on the file descriptor that this handler has been registered for. The
+ * context is the one that was passed into add_file by the user. The
+ * events parameter indicates which event is available on the file,
+ * it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */
+typedef void (*event_loop_file_handler)(event_loop *loop,
+                                        int fd,
+                                        void *context,
+                                        int events);
+
+/* This handler will be called when a timer times out. The id of the timer
+ * as well as the context that was specified when registering this handler
+ * are passed as arguments. The return is the number of milliseconds the
+ * timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall
+ * not be triggered again. */
+typedef int (*event_loop_timer_handler)(event_loop *loop,
+                                        timer_id timer_id,
+                                        void *context);
+
+/* Create and return a new event loop. */
+event_loop *event_loop_create();
+
+/* Deallocate space associated with the event loop that was created
+ * with the "create" function. */
+void event_loop_destroy(event_loop *loop);
+
+/* Register a handler that will be called any time a new event happens on
+ * a file descriptor. Can specify a context that will be passed as an
+ * argument to the handler. Currently there can only be one handler per file.
+ * The events parameter specifies which events we listen to: EVENT_LOOP_READ
+ * or EVENT_LOOP_WRITE. */
+void event_loop_add_file(event_loop *loop,
+                         int fd,
+                         int events,
+                         event_loop_file_handler handler,
+                         void *context);
+
+/* Remove a registered file event handler from the event loop. */
+void event_loop_remove_file(event_loop *loop, int fd);
+
+/* Register a handler that will be called after a time slice of
+ * "milliseconds" milliseconds. Can specify a context that will be passed
+ * as an argument to the handler. Return the id of the time event. */
+int64_t event_loop_add_timer(event_loop *loop,
+                             int64_t milliseconds,
+                             event_loop_timer_handler handler,
+                             void *context);
+
+/* Remove a registered time event handler from the event loop. */
+void event_loop_remove_timer(event_loop *loop, timer_id timer_id);
+
+/* Run the event loop. */
+void event_loop_run(event_loop *loop);
+
+/* Stop the event loop. */
+void event_loop_stop(event_loop *loop);
+
+#endif