diff --git a/CMakeLists.txt b/CMakeLists.txt index 298fccec8..75d4f86a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,18 +47,11 @@ if (COVERALLS) coveralls_turn_on_coverage() endif() -# Old way to do this -# Data Structure for Unprocessed Event List -#SET(QUEUE calendar) #Calendar Queue -#SET(QUEUE heap) #Push/Down Heap -#SET(QUEUE splay) #Splay Tree -#SET(QUEUE kp_splay) #Splay Tree in KPs - -# New way as of CMake 2.8 -# The default value for the QUEUE variable is splay -# The other options are presented at config time by cmake-gui -SET(QUEUE splay CACHE STRING "Queue type chosen by the user at configure time") -SET_PROPERTY(CACHE QUEUE PROPERTY STRINGS splay calendar heap kp_splay) +# Priority Queue Implementation +SET(QUEUE splay) +# Other queue implementations are no longer supported. +# SET(QUEUE splay CACHE STRING "Queue type chosen by the user at configure time") +# SET_PROPERTY(CACHE QUEUE PROPERTY STRINGS splay calendar heap kp_splay) # Random Library SET(RAND clcg4) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index dc63a57bd..1c63b333a 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -53,7 +53,6 @@ tw-timing.c tw-sched.c tw-setup.c -tw-signal.c tw-stats.c tw-util.c @@ -94,7 +93,7 @@ set(VERSION_SHORT "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") # Data Structure for remote Events # If AVL_TREE is OFF, ROSS reverts to hashing -OPTION(AVL_TREE "Use AVL trees for optimistic mode events? (hash tabels otherwise)" ON) +OPTION(AVL_TREE "Use AVL trees for optimistic mode events? (hash tables otherwise)" ON) IF(AVL_TREE) SET(ross_srcs ${ross_srcs} avl_tree.h avl_tree.c) ENDIF(AVL_TREE) diff --git a/core/gvt/mpi_allreduce.c b/core/gvt/mpi_allreduce.c index ccfd974ec..a3cc499d3 100644 --- a/core/gvt/mpi_allreduce.c +++ b/core/gvt/mpi_allreduce.c @@ -13,8 +13,7 @@ static const tw_optdef gvt_opts [] = { TWOPT_GROUP("ROSS MPI GVT"), TWOPT_UINT("gvt-interval", g_tw_gvt_interval, "GVT Interval: Iterations through scheduling loop (synch=1,2,3,4), or ms between GVTs (synch=5)"), - TWOPT_DOUBLE("report-interval", gvt_print_interval, - "percent of runtime to print GVT"), + TWOPT_DOUBLE("report-interval", gvt_print_interval, "percent of runtime to print GVT"), TWOPT_END() }; diff --git a/core/network-mpi.c b/core/network-mpi.c index 880a115d8..a91a9f303 100644 --- a/core/network-mpi.c +++ b/core/network-mpi.c @@ -7,10 +7,15 @@ int custom_communicator = 0; /** * @struct act_q * @brief Keeps track of posted send or recv operations. + * + * This list structure is used *only* by the network mpi layer (this + * file). Within this file, two lists are used, for MPI Irecv and + * Isend requests. The MPI requests and statusus are linked with an + * event buffer through this struct. */ struct act_q { - const char *name; + const char *name; /**< name of the list, used in error printouts */ tw_event **event_list; /**< list of event pointers in this queue */ MPI_Request *req_list; /**< list of MPI request handles */ @@ -27,8 +32,8 @@ static struct act_q posted_sends; static struct act_q posted_recvs; static tw_eventq outq; -static unsigned int read_buffer = 16; -static unsigned int send_buffer = 1024; +static unsigned int read_buffer = 16; /**< Number of Irecv's to buffer, length of posted_recvs queue */ +static unsigned int send_buffer = 1024; /**< Number of Isend's to buffer, length of posted_sends queue */ static int world_size = 1; static const tw_optdef mpi_opts[] = { @@ -85,20 +90,13 @@ tw_net_init(int *argc, char ***argv) * @param[in] name name of the queue */ static void -init_q(struct act_q *q, const char *name) +init_q(struct act_q *q, const char *name, unsigned int size) { - unsigned int n; - - if(q == &posted_sends) - n = send_buffer; - else - n = read_buffer; - q->name = name; - q->event_list = (tw_event **) tw_calloc(TW_LOC, name, sizeof(*q->event_list), n); - q->req_list = (MPI_Request *) tw_calloc(TW_LOC, name, sizeof(*q->req_list), n); - q->idx_list = (int *) tw_calloc(TW_LOC, name, sizeof(*q->idx_list), n); - q->status_list = (MPI_Status *) tw_calloc(TW_LOC, name, sizeof(*q->status_list), n); + q->event_list = (tw_event **) tw_calloc(TW_LOC, name, sizeof(tw_event *), size); + q->req_list = (MPI_Request *) tw_calloc(TW_LOC, name, sizeof(MPI_Request), size); + q->idx_list = (int *) tw_calloc(TW_LOC, name, sizeof(int), size); + q->status_list = (MPI_Status *) tw_calloc(TW_LOC, name, sizeof(MPI_Status), size); } unsigned int @@ -110,6 +108,7 @@ tw_nnodes(void) void tw_net_start(void) { + // sets value of tw_nnodes if (MPI_Comm_size(MPI_COMM_ROSS, &world_size) != MPI_SUCCESS) tw_error(TW_LOC, "Cannot get MPI_Comm_size(MPI_COMM_ROSS)"); @@ -131,22 +130,12 @@ tw_net_start(void) tw_pe_init(); - //If we're in (some variation of) optimistic mode, we need this hash - if (g_tw_synchronization_protocol == OPTIMISTIC || - g_tw_synchronization_protocol == OPTIMISTIC_DEBUG || - g_tw_synchronization_protocol == OPTIMISTIC_REALTIME) { - g_tw_pe->hash_t = tw_hash_create(); - } else { - g_tw_pe->hash_t = NULL; - } - - if (send_buffer < 1) - tw_error(TW_LOC, "network send buffer must be >= 1"); - if (read_buffer < 1) - tw_error(TW_LOC, "network read buffer must be >= 1"); + // these values are command line options + if (send_buffer < 1) tw_error(TW_LOC, "network send buffer must be >= 1"); + if (read_buffer < 1) tw_error(TW_LOC, "network read buffer must be >= 1"); - init_q(&posted_sends, "MPI send queue"); - init_q(&posted_recvs, "MPI recv queue"); + init_q(&posted_sends, "MPI send queue", send_buffer); + init_q(&posted_recvs, "MPI recv queue", read_buffer); g_tw_net_device_size = read_buffer; diff --git a/core/queue/tw-queue.h b/core/queue/tw-queue.h index c14937177..1f547b49c 100644 --- a/core/queue/tw-queue.h +++ b/core/queue/tw-queue.h @@ -13,4 +13,3 @@ unsigned int tw_pq_max_size(tw_pq *); #ifdef ROSS_QUEUE_kp_splay tw_eventpq * tw_eventpq_create(void); #endif - diff --git a/core/ross-extern.h b/core/ross-extern.h index e3d6a478f..7bd7ff7a8 100644 --- a/core/ross-extern.h +++ b/core/ross-extern.h @@ -143,12 +143,6 @@ extern void tw_scheduler_optimistic(tw_pe * me); extern void tw_scheduler_optimistic_debug(tw_pe * me); extern void tw_scheduler_optimistic_realtime(tw_pe * me); -/* - * tw-signal.c - */ -extern void tw_sigsegv(int sig); -extern void tw_sigterm(int sig); - /* * tw-state.c */ diff --git a/core/ross-types.h b/core/ross-types.h index 631dd8f31..eace35df8 100644 --- a/core/ross-types.h +++ b/core/ross-types.h @@ -379,7 +379,6 @@ struct tw_pe { tw_eventq event_q; /**< @brief Linked list of events sent to this PE */ tw_event *cancel_q; /**< @brief List of canceled events */ tw_pq *pq; /**< @brief Priority queue used to sort events */ - tw_kp *kp_list; /**< @brief */ tw_eventq free_q; /**< @brief Linked list of free tw_events */ tw_event *abort_event; /**< @brief Placeholder event for when free_q is empty */ diff --git a/core/tw-event.c b/core/tw-event.c index 27b435e23..303554515 100644 --- a/core/tw-event.c +++ b/core/tw-event.c @@ -31,7 +31,7 @@ void tw_event_send(tw_event * event) { #endif // moved from network-mpi.c in order to give all events a seq_num - event->event_id = (tw_eventid) ++send_pe->seq_num; + event->event_id = (tw_eventid) ++send_pe->seq_num; // call LP remote mapping function to get dest_pe dest_peid = (*src_lp->type->map) ((tw_lpid) event->dest_lp); diff --git a/core/tw-eventq.h b/core/tw-eventq.h index 366ead29c..e3d0c562b 100644 --- a/core/tw-eventq.h +++ b/core/tw-eventq.h @@ -5,6 +5,9 @@ #include +/** + * debug assitant fuction + */ static inline void tw_eventq_debug(tw_eventq * q) { @@ -39,6 +42,9 @@ tw_eventq_debug(tw_eventq * q) #endif } +/** + * push the contents of one list onto another?? + */ static inline void tw_eventq_push_list(tw_eventq * q, tw_event * h, tw_event * t, long cnt) { @@ -115,6 +121,15 @@ tw_eventq_push_list(tw_eventq * q, tw_event * h, tw_event * t, long cnt) tw_eventq_debug(q); } +/** + * Given a list, move the portion of its contents that is older than GVT to + * the free list. + * + * Assumptions: + * - The provided q is not the free_q + * - The head of the list has the maximum time stamp in the list. Therefore, + * if the head is older than GVT, everything in the list is as well. + */ static inline void tw_eventq_fossil_collect(tw_eventq *q, tw_pe *pe) { @@ -164,6 +179,9 @@ tw_eventq_fossil_collect(tw_eventq *q, tw_pe *pe) } } +/** + * allocate a events into a given tw_eventq + */ static inline void tw_eventq_alloc(tw_eventq * q, unsigned int cnt) { @@ -198,7 +216,6 @@ tw_eventq_alloc(tw_eventq * q, unsigned int cnt) g_tw_event_msg_sz = event_len; // compute number of events needed for the network. - g_tw_gvt_threshold = (int) ceil(g_tw_net_device_size / g_tw_event_msg_sz); g_tw_gvt_threshold = g_tw_net_device_size; g_tw_events_per_pe += g_tw_gvt_threshold; cnt += g_tw_gvt_threshold; @@ -233,6 +250,9 @@ tw_eventq_alloc(tw_eventq * q, unsigned int cnt) q->tail = event; } +/** + * push to tail of list + */ static inline void tw_eventq_push(tw_eventq *q, tw_event *e) { @@ -253,12 +273,18 @@ tw_eventq_push(tw_eventq *q, tw_event *e) tw_eventq_debug(q); } +/** + * peek to tail of list + */ static inline tw_event * tw_eventq_peek(tw_eventq *q) { return q->tail; } +/** + * pop to tail of list + */ static inline tw_event * tw_eventq_pop(tw_eventq * q) { @@ -287,6 +313,9 @@ tw_eventq_pop(tw_eventq * q) return t; } +/** + * push to head of list + */ static inline void tw_eventq_unshift(tw_eventq *q, tw_event *e) { @@ -308,12 +337,18 @@ tw_eventq_unshift(tw_eventq *q, tw_event *e) tw_eventq_debug(q); } +/** + * peek at head of list + */ static inline tw_event * tw_eventq_peek_head(tw_eventq *q) { return q->head; } +/** + * pop from head of list + */ static inline tw_event * tw_eventq_shift(tw_eventq *q) { @@ -342,6 +377,9 @@ tw_eventq_shift(tw_eventq *q) return h; } +/** + * delete an event from anywhere in the list + */ static inline void tw_eventq_delete_any(tw_eventq *q, tw_event *e) { @@ -366,6 +404,10 @@ tw_eventq_delete_any(tw_eventq *q, tw_event *e) tw_eventq_debug(q); } +/** + * pop the entire list. + * After this operation, the size of the provided q is 0. + */ static inline tw_event * tw_eventq_pop_list(tw_eventq * q) { @@ -377,7 +419,7 @@ tw_eventq_pop_list(tw_eventq * q) return h; } -/* +/** * The purpose of this function is to be able to remove some * part of a list.. could be all of list, from head to some inner * buffer, or from some inner buffer to tail. I only care about the diff --git a/core/tw-pe.c b/core/tw-pe.c index ab00dd55e..a974debc4 100644 --- a/core/tw-pe.c +++ b/core/tw-pe.c @@ -22,29 +22,39 @@ tw_pe_settype(const tw_petype * type) #undef copy_pef } -/* - * tw_pe_init: initialize individual PE structs +/** + * initialize individual PE structs + * + * must be called after tw_nnodes / MPI world size is set. * */ void tw_pe_init(void) { - if (g_tw_pe) - tw_error(TW_LOC, "PE %u already initialized", g_tw_mynode); + if (g_tw_pe) tw_error(TW_LOC, "PE %u already initialized", g_tw_mynode); g_tw_pe = (tw_pe*)tw_calloc(TW_LOC, "PE Struct", sizeof(*g_tw_pe), 1); - tw_petype no_type; - memset(&no_type, 0, sizeof(no_type)); + g_tw_pe->id = g_tw_mynode; - g_tw_pe->id = g_tw_mynode; - tw_pe_settype(&no_type); + tw_petype no_type; + memset(&no_type, 0, sizeof(no_type)); + tw_pe_settype(&no_type); - g_tw_pe->trans_msg_ts = TW_STIME_MAX; - g_tw_pe->gvt_status = 0; + g_tw_pe->trans_msg_ts = DBL_MAX; + g_tw_pe->gvt_status = 0; // TODO is the PE RNG ever actually used? - g_tw_pe->rng = tw_rand_init(31, 41); + g_tw_pe->rng = tw_rand_init(31, 41); + + //If we're in (some variation of) optimistic mode, we need this hash + if (g_tw_synchronization_protocol == OPTIMISTIC || + g_tw_synchronization_protocol == OPTIMISTIC_DEBUG || + g_tw_synchronization_protocol == OPTIMISTIC_REALTIME) { + g_tw_pe->hash_t = tw_hash_create(); + } else { + g_tw_pe->hash_t = NULL; + } } diff --git a/core/tw-sched.c b/core/tw-sched.c index 3bdf14c72..47ade507e 100644 --- a/core/tw-sched.c +++ b/core/tw-sched.c @@ -13,7 +13,7 @@ static inline void reset_bitfields(tw_event *revent) memset(&revent->cv, 0, sizeof(revent->cv)); } -/* +/** * Get all events out of my event queue and spin them out into * the priority queue so they can be processed in time stamp * order. @@ -66,7 +66,7 @@ static void tw_sched_event_q(tw_pe * me) { } } -/* +/** * OPT: need to link events into canq in reverse order so * that when we rollback the 1st event, we should not * need to do any further rollbacks. diff --git a/core/tw-signal.c b/core/tw-signal.c deleted file mode 100644 index 167ac63b0..000000000 --- a/core/tw-signal.c +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include - -static unsigned int once = 1; - -void -tw_sigsegv(int sig) -{ - if (sig == SIGBUS) - tw_error(TW_LOC, "Caught SIGBUS; terminating."); - else if (sig == SIGSEGV) - tw_error(TW_LOC, "Caught SIGSEGV; terminating."); - else if (sig != SIGINT) - tw_error(TW_LOC, "Caught unknown signal %d; terminating.", sig); - - /* nothing to report, just exit */ - if(0 == g_tw_sim_started) - tw_net_abort(); - - if (once) - { - if (g_tw_pe) - tw_stats(g_tw_pe); - } - - once = 0; - - tw_printf(TW_LOC, "Caught SIGINT; terminating."); - tw_net_abort(); -} - -void -tw_sigterm(int sig) -{ - if (sig != SIGTERM) - tw_error(TW_LOC, "Caught unknown signal %d; terminating.", sig); - tw_net_abort(); -}