add flag to invoke create callback on all ranks

Author: adammoody

libcircle.h

@@ -1,13 +1,13 @@
 #ifndef LIBCIRCLE_H
 #define LIBCIRCLE_H
 
+#include <stdint.h>
+
 /* define a C interface */
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdint.h>
-
 /**
  * The maximum length of a string value which is allowed to be placed on the
  * queue structure.
@@ -22,8 +22,9 @@ extern "C" {
  * Run time flags for the behavior of splitting work.
  */
 #define CIRCLE_SPLIT_RANDOM     (1 << 0)              /* Split work randomly. */
-#define CIRCLE_SPLIT_EQUAL      ~CIRCLE_SPLIT_RANDOM  /* Split work evenly */
-#define CIRCLE_DEFAULT_FLAGS    CIRCLE_SPLIT_RANDOM   /* Default behavior is random work stealing */
+#define CIRCLE_SPLIT_EQUAL      (1 << 1)              /* Split work evenly */
+#define CIRCLE_CREATE_GLOBAL    (1 << 2)              /* Call create callback on all procs */
+#define CIRCLE_DEFAULT_FLAGS    CIRCLE_SPLIT_EQUAL    /* Default behavior is random work stealing */
 
 /**
  * The various logging levels that libcircle will output.

libcircle/worker.c

@@ -347,11 +347,17 @@ int8_t CIRCLE_worker()
         LOG(CIRCLE_LOG_DBG, "Using randomized load splitting.");
     }
 
+    /* start the termination token on rank 0 */
     if(rank == 0) {
-        (*(CIRCLE_INPUT_ST.create_cb))(&queue_handle);
         local_state.have_token = 1;
     }
 
+    /* start by adding work to queue by calling create_cb,
+     * only invoke on master unless CREATE_GLOBAL is set */
+    if(rank == 0 || CIRCLE_INPUT_ST.options & CIRCLE_CREATE_GLOBAL) {
+        (*(CIRCLE_INPUT_ST.create_cb))(&queue_handle);
+    }
+
     CIRCLE_work_loop(sptr, &queue_handle);
     CIRCLE_cleanup_mpi_messages(sptr);