/* -*- c-file-style: "GNU" -*- */
/*
 * Copyright (C) CNRS, INRIA, Universite Bordeaux 1, Telecom SudParis
 * See COPYING in top-level directory.
 */

#include <stdio.h>
#include <GTG.h>
#include <assert.h>
#include <limits.h>
#include "mpi_ev_codes.h"
#include "mpi_eztrace.h"
#include "eztrace_list.h"
#include "eztrace_convert.h"
#include "eztrace_convert_mpi.h"
#include "eztrace_convert_mpi_p2p.h"
#include "eztrace_convert_mpi_coll.h"
#include "eztrace_convert_mpi_pers.h"
#include "eztrace_stats_mpi.h"

#ifdef USE_MPI
/* todo: we should get rid of these lists. Use the hierarchical arrays instead. */

/* list that contains pending MPI_Comm_spawn */
struct ezt_list_t spawn_list;

static int recording_stats = 0;
static unsigned nb_mpi_calls[MPI_ID_SIZE];

int *rank_to_trace_id = NULL;

#define MPI_CHANGE() if(!recording_stats) CHANGE()

static void mpi_synchronize_processes(struct mpi_coll_msg_t* msg, int my_rank);
static struct ezt_list_token_t* __find_matching_spawn(int ppid)
{
  struct ezt_list_token_t *t = NULL;
  struct mpi_spawn_t *r;
  ezt_list_foreach(&spawn_list, t) {
    r = (struct mpi_spawn_t *) t->data;
    if(r->ppid == ppid)
    return t;
  }
  return t;
}

static struct ezt_mpi_comm* ezt_find_communicator(int index, app_ptr comm_id) {
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(index), p_info);

  if(p_info->__MPI_COMM_WORLD.comm_id == comm_id)
    return &p_info->__MPI_COMM_WORLD;

  if(p_info->__MPI_COMM_SELF.comm_id == comm_id)
  return &p_info->__MPI_COMM_SELF;

  struct ezt_list_token_t *token;
  ezt_list_foreach(&p_info->communicators, token) {
    struct ezt_mpi_comm* p_comm = (struct ezt_mpi_comm*) token->data;
    assert(p_comm);
    if(p_comm->comm_id == comm_id) {
      /* we found the communicator ! */
      return p_comm;
    }
  }

  return 0;
}

#if EZTRACE_DEBUG
/* return 1 if comm_id already exists in comm_list
 * for debugging purpose only
 */
static int __find_communicator(app_ptr comm_id, struct ezt_list_t *comm_list)
{
  struct ezt_list_token_t *token;
  ezt_list_foreach(comm_list, token) {
    struct ezt_mpi_comm* p_comm = (struct ezt_mpi_comm*) token->data;
    assert(p_comm);
    if(p_comm->comm_id == comm_id) {
      /* we found the communicator ! */
      return 1;
    }
    return 0;
  }

}
#endif

/* destroy an mpi communicator */
void handle_mpi_delete_comm()
{
  FUNC_NAME;

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  /* get the communicator id (eg. MPI_COMM_WORLD) and the number of processes */
  app_ptr comm_id;
  GET_PARAM_PACKED_1(CUR_EV, comm_id);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  assert(comm);
  ezt_list_remove(&comm->token);
  ezt_list_add(&p_info->deleted_communicators, &comm->token);
}

/* create a new mpi communicator */
void handle_mpi_new_comm()
{
  FUNC_NAME;

  /* get the communicator id (eg. MPI_COMM_WORLD) and the number of processes */
  app_ptr comm_id;
  int comm_size;
  GET_PARAM_PACKED_2(CUR_EV, comm_id, comm_size);

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  /* allocate a new ezt_mpi_comm and fill it */
  struct ezt_mpi_comm* p_comm = malloc(sizeof(struct ezt_mpi_comm));
  p_comm->comm_id = comm_id;
  p_comm->comm_size = comm_size;
  p_comm->ranks = malloc(sizeof(int)*comm_size);
  p_comm->p_process = p_info;
#if EZTRACE_DEBUG
  if(__find_communicator(comm_id, &p_info->communicators)) {
    fprintf(stderr, "[%d] Error: trying to add a communicator (%d) that already exists!\n", comm_id);
    abort();
  }
#endif

  /* for each rank in the new communicator, translate to the global rank
   * (the corresponding rank in MPI_COMM_WORLD) */
  int i;
  for(i=0; i<comm_size; i++) {
    wait_for_an_event(CUR_INDEX, EZTRACE_MPI_NEW_COMM_Info);
    GET_PARAM_PACKED_1(CUR_EV, p_comm->ranks[i]);
    if(CUR_RANK == p_comm->ranks[i])
    p_comm->my_rank = i;
  }

  p_comm->token.data = p_comm;
  ezt_list_add(&p_info->communicators, &p_comm->token);
}


int is_comm_mine(int global_rank,
		 struct ezt_mpi_comm*comm) {
  if(comm && comm->ranks[comm->my_rank] == global_rank)
    return 1;
  return 0;
}

static struct mpi_p2p_msg_t* __mpi_send_generic(const char* thread_id, int src, int dest, int len, uint32_t tag,
    struct mpi_request* mpi_req, app_ptr comm)
{

  int actual_dest = ezt_get_global_rank_generic(rank_to_trace_id[src], comm, dest);
  assert(actual_dest!= -1);
  struct mpi_p2p_msg_t* msg = __start_send_message(CUR_TIME(CUR_INDEX), src, actual_dest, len, tag, thread_id, mpi_req);
  assert(msg);

  if(! IS_TIME_SET(msg->times[stop_recv])) {
#ifndef GTG_OUT_OF_ORDER
    MPI_CHANGE() startLink(CURRENT, "L_MPI_P2P", "C_Prog",
	msg->sender_thread_id, msg->recver_thread_id,
	msg->link_value,
	msg->id);
#endif
  }
  return msg;
}

static struct mpi_p2p_msg_t *
__mpi_start_recv_generic (char*thread_id,
    int src,
    int dest,
    int len,
    int tag,
    struct mpi_request* mpi_req,
    app_ptr comm)
{

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(rank_to_trace_id[dest]), p_info);
  if (src == p_info->__MPI_PROC_NULL || dest == p_info->__MPI_PROC_NULL)
    return NULL;
  int actual_src = src;
  if(src != p_info->__MPI_ANY_SOURCE) {
    actual_src = ezt_get_global_rank_generic(rank_to_trace_id[dest], comm, src);
    assert(actual_src!= -1);
  }
  return __start_recv_message(CUR_TIME(CUR_INDEX),
			      actual_src,
			      dest,
			      len,
			      tag,
			      thread_id, mpi_req);
}

/* return 1 if the matching isend didn't occured yet.
 * In that case, this function has to be called again when the matching isend occurs
 * In case a synchronization problem is detected, timestamp is updated
 */
static int
__mpi_stop_recv_generic (uint64_t *timestamp,
			 char*thread_id,
			 int src,
			 int dest,
			 int len,
			 int tag,
			 struct mpi_request* mpi_req,
			 app_ptr comm)
{
  int delay_added = 0;
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(rank_to_trace_id[dest]), p_info);
  if (src == p_info->__MPI_PROC_NULL || dest == p_info->__MPI_PROC_NULL) {
    return 0;
  }
  int actual_src = src;
  if(src != p_info->__MPI_ANY_SOURCE) {
    actual_src = ezt_get_global_rank_generic(CUR_INDEX, comm, src);
    assert(actual_src!= -1);
  }

  struct mpi_p2p_msg_t *msg = __stop_recv_message(*timestamp, actual_src, dest, len, tag, thread_id, mpi_req);
  if(!msg) {
    /* There is no matching MPI_Isend yet.
     * This means there's a synchronisation problem between the traces.
     * Let's wait until the remote process calls MPI_Isend
     */
    SKIP = 1;
    CUR_TRACE->skip = 1;
    return 1;
  }

  if(IS_TIME_SET(msg->times[start_isend])) {
    if(msg->times[start_isend] > msg->times[stop_recv]) {
      msg->times[stop_recv] += add_delay_to_trace(rank_to_trace_id[dest],
	  msg->times[stop_recv],
	  msg->times[start_isend],
	  msg->recver_thread_id);

      *timestamp = msg->times[stop_recv];
      delay_added = 1;
    }
    assert(msg->id);
    assert(msg->link_value);
    assert(msg->sender_thread_id);
    assert(msg->recver_thread_id);

#ifdef GTG_OUT_OF_ORDER

    MPI_CHANGE() startLink(NS_TO_MS(msg->times[start_isend]),
	"L_MPI_P2P", "C_Prog",
	msg->sender_thread_id, msg->recver_thread_id,
	msg->link_value, msg->id);

    MPI_CHANGE() endLink(NS_TO_MS(msg->times[stop_recv]),
	"L_MPI_P2P", "C_Prog",
	msg->sender_thread_id, thread_id,
	msg->link_value, msg->id);

#if 0
    char* message_str = NULL;
    float throughput = -1;
    /* duration in us */
    float duration = ((NS_TO_MS(*timestamp)) - (NS_TO_MS(msg->times[start_isend])))*1e3;
    int res __attribute__ ((__unused__));
    if(!delay_added) {
      if(len) {
	/* byte / us = MB/s */
	throughput = len/duration;
      }
      res = asprintf(&message_str, "duration = %f us -- len=%d bytes -- throughput=%f MB/s",
		     duration, len, throughput);
    } else {
      /* There was a synchronization problem that was corrected. Thus, the duration of the
       * communication is 0 (which does not make sense). We still need to add an event here since
       * some trace analysis tools (eg. eztrace pattern detection) may expect an event.
       */
      res = asprintf(&message_str, "duration = ? (lost due to a synchronization correction) -- len=%d bytes -- throughput=? MB/s", len);
    }

    MPI_CHANGE() addEvent((NS_TO_MS(*timestamp)), "E_MPI_EndComm", thread_id, message_str);
#endif

#else  /* GTG_OUT_OF_ORDER */
    MPI_CHANGE() endLink((NS_TO_MS(*timestamp)), "L_MPI_P2P", "C_Prog",
	msg->sender_thread_id, msg->recver_thread_id,
	msg->link_value, msg->id);
#endif
  }
  return 0;
}

#define start_coll_link(msg, src, dest)					\
  startLink (NS_TO_MS(msg->times[src][start_coll]),			\
	     "L_MPI_Coll", "C_Prog",					\
	     msg->thread_ids[src], msg->thread_ids[dest],		\
	     msg->link_value[src][dest], msg->link_id[src][dest])

#define end_coll_link(msg, src, dest)				\
  endLink(NS_TO_MS(msg->times[dest][stop_coll]),		\
	  "L_MPI_Coll", "C_Prog",				\
	  msg->thread_ids[src], msg->thread_ids[dest],		\
	  msg->link_value[src][dest], msg->link_id[src][dest])

static void __mpi_barrier_start_generic(struct mpi_coll_msg_t* msg, int my_rank)
{
  int i;

  for(i=0; i<msg->comm[my_rank]->comm_size; i++) {
    if(i != my_rank) {

#ifdef GTG_OUT_OF_ORDER
	assert(msg->thread_ids[my_rank]);
	if(IS_TIME_SET(msg->times[i][stop_coll])) {
	  /* the remote process stop_coll already happened.
	   * This means that there's a synchronisation problem between the traces
	   */

	  msg->times[i][stop_coll] += add_delay_to_trace(i,
							 msg->times[i][stop_coll],
							 msg->times[my_rank][start_coll],
							 msg->thread_ids[i]);
	}
#else
      /* todo: not implemented yet */
#endif	/* GTG_OUT_OF_ORDER */
    }
  }
}

/* return 1 if the matching isend didn't occured yet.
 * In that case, this function has to be called again when the matching isend occurs
 */
static int __mpi_barrier_stop_generic(struct mpi_coll_msg_t* msg, int my_rank)
{
  int i;
  if(!msg) {
    SKIP = 1;
    CUR_TRACE->skip = 1;
    return 1;
  }

  for(i=0; i<msg->comm[my_rank]->comm_size; i++) {
    if(i != my_rank) {
      if(!IS_TIME_SET(msg->times[i][start_coll])) {
	/* ith process has not yet reached the MPI_Barrier.
	 * This means there's a synchronisation problem between the traces.
	 * Let's wait until the remote process starts MPI_Barrier
	 */
	SKIP = 1;
	CUR_TRACE->skip = 1;
	msg->times[my_rank][stop_coll] = TIME_INIT;
	return 1;
      }
    }
  }

  for(i=0; i<msg->comm[my_rank]->comm_size; i++) {
    if(i != my_rank) {
#ifdef GTG_OUT_OF_ORDER
      assert(msg->thread_ids[my_rank]);
      assert(msg->thread_ids[i]);
      MPI_CHANGE() start_coll_link(msg, my_rank, i);
      MPI_CHANGE() end_coll_link(msg, i, my_rank);
#else
	/* todo: not implemented yet */
#endif /* GTG_OUT_OF_ORDER */
    }
  }
  return 0;
}

void
handle_mpi_init ()
{
  int rank;
  int world_size;
  int mpi_any_source;
  int mpi_any_tag;
  app_ptr mpi_request_null;
  int mpi_proc_null;

  GET_PARAM_PACKED_6(CUR_EV, rank, world_size, mpi_any_source, mpi_any_tag, mpi_request_null, mpi_proc_null);
  EZT_CONVERT_PRINTF(1, "MPI_Init(rank=%d, world_size=%d, mpi_any_source=%x, mpi_any_tag=%x, mpi_request_null=%x, mpi_proc_null=%d)\n", rank, world_size, mpi_any_source, mpi_any_tag, mpi_request_null, mpi_proc_null);

  int *nb_traces = get_nb_traces();
  if(*nb_traces < world_size) {
    fprintf(stderr, "Error: the size of MPI_COMM_WORLD is %d, but only %d trace files were provided. Please give me all the trace files !\n", world_size, *nb_traces);
    abort();
  }

  if(!rank_to_trace_id) {
    int *nb = nb_traces;
    rank_to_trace_id = calloc(*nb, sizeof(int));
  }

  rank_to_trace_id[rank] = CUR_INDEX;

  CUR_TRACE->start_time = LITL_READ_GET_TIME(CUR_EV);

  /* Check wether MPI_SPAWNED event has already occur.
   * In that case, the rank is already known.
   */
  if(!CUR_TRACE->start) {
    CUR_RANK = rank;
    int res __attribute__ ((__unused__));
    res = CREATE_TRACE_ID_STR(CUR_ID, rank);
    GET_PROCESS_INFO(CUR_INDEX)->pid = rank;
    eztrace_create_ids(CUR_INDEX);

    CUR_TRACE->start = 1;
    NB_START++;

    /* Create the process and the main thread */
    DECLARE_PROCESS_ID_STR(process_id, CUR_INDEX);
    if(!recording_stats)addContainer (CURRENT, process_id, "CT_Process", "C_Prog", process_id, "0");
    new_thread(CUR_THREAD_ID);

  } else {
    CUR_TRACE->start = 1;
    NB_START++;
  }
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  p_info->__MPI_ANY_SOURCE = mpi_any_source;
  p_info->__MPI_ANY_TAG = mpi_any_tag;
  p_info->__MPI_REQUEST_NULL = mpi_request_null;
  p_info->__MPI_PROC_NULL = mpi_proc_null;

  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_INIT_Info);

  GET_PARAM_PACKED_2(CUR_EV, p_info->__MPI_COMM_WORLD.comm_id, p_info->__MPI_COMM_SELF.comm_id);

  p_info->__MPI_COMM_WORLD.comm_size = world_size;
  p_info->__MPI_COMM_WORLD.ranks = malloc(world_size * sizeof(int));
  int i;
  for(i=0;i<world_size; i++)
  p_info->__MPI_COMM_WORLD.ranks[i] = i;
  p_info->__MPI_COMM_WORLD.my_rank = rank;
  p_info->__MPI_COMM_WORLD.p_process = p_info;

  p_info->__MPI_COMM_SELF.comm_size = 1;
  p_info->__MPI_COMM_SELF.ranks = malloc(1 * sizeof(int));
  p_info->__MPI_COMM_SELF.ranks[0] = rank;
  p_info->__MPI_COMM_SELF.my_rank = 0;
  p_info->__MPI_COMM_SELF.p_process = p_info;

  ezt_list_new(&p_info->communicators);
  ezt_list_new(&p_info->deleted_communicators);

  FUNC_NAME;
}

void handle_mpi_spawned()
{
  /* set the start time */
  if(CUR_TRACE->start == 0)
  CUR_TRACE->start_time = LITL_READ_GET_TIME(CUR_EV);

  CUR_TRACE->start = 1;
  int ppid;
  /* local_rank = rank amoung the spawned processes */
  int local_rank;
  GET_PARAM_PACKED_2(CUR_EV, ppid, local_rank);

  struct ezt_list_token_t *token = __find_matching_spawn(ppid);
  if(!token) {
    /* the MPI_Comm_spawn has not been handled yet. We have to wait for it */
    SKIP = 1;
    CUR_TRACE->skip = 1;
    return;
  }

  struct mpi_spawn_t *spawn;
  spawn = (struct mpi_spawn_t *) token->data;
  /* We have to add a delay to all events on this trace */
  CUR_TRACE->delay = spawn->start_time;
  spawn->nb_children--;
  struct trace_t *parent_trace = spawn->parent_trace;

  if(!spawn->nb_children) {
    /* all the spawned processes have been handled, we can free this structure. */
    ezt_list_remove(token);
    free(spawn);
    free(token);
  }

  CUR_RANK = ppid + local_rank;

  /* initialize the trace_id */
  int res __attribute__ ((__unused__));
  res = asprintf(&CUR_ID, "%s_%d", parent_trace->trace_id, local_rank);
  eztrace_create_ids(CUR_INDEX);
  DECLARE_PROCESS_ID_STR(process_id, CUR_INDEX);
  if(!recording_stats)addContainer (CURRENT, process_id, "CT_Process", "C_Prog", process_id, "0");
  new_thread(CUR_THREAD_ID);

  /* Crappy hack: we know that next event is MPI_Init and
   * we need the trace to be properly started before continuing the function.
   */
  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_INIT);
  /* We should at least check if the code corresponds to MPI_INIT */
  handle_mpi_init ();

  FUNC_NAME;

  /* prepare the link value */
  char*link_id = NULL;
  int __attribute__((unused)) ret = asprintf(&link_id, "%s_%d", parent_trace->trace_id, local_rank);

  MPI_CHANGE() endLink (CURRENT, "L_MPI_SPAWN", "C_Prog", parent_trace->trace_id, process_id, link_id, link_id);

  free(link_id);
}

void __ezt_mpi_enter_function(MPI_id_t id) {
  DECLARE_CUR_THREAD(p_thread);
  INIT_MPI_THREAD_INFO(p_thread, thread_info);

  assert(thread_info->enter_date_mpi_calls[id] < 0);

  thread_info->nb_mpi_calls[id]++;
  thread_info->enter_date_mpi_calls[id] = CURRENT;
}

void __ezt_mpi_leave_function(MPI_id_t id) {
  DECLARE_CUR_THREAD(p_thread);
  INIT_MPI_THREAD_INFO(p_thread, thread_info);

  assert(thread_info->enter_date_mpi_calls[id] >= 0);
  assert(thread_info->leave_date_mpi_calls[id] < 0);

  thread_info->leave_date_mpi_calls[id] = CURRENT;
  thread_info->total_time_mpi_calls[id] += thread_info->leave_date_mpi_calls[id]-thread_info->enter_date_mpi_calls[id];

  thread_info->enter_date_mpi_calls[id] = -1;
  thread_info->leave_date_mpi_calls[id] = -1;
}

void
handle_mpi_start_send ()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_SEND_ID);

  int len;
  int dest;
  uint32_t tag;
  app_ptr comm;
  GET_PARAM_PACKED_4(CUR_EV, len, dest, tag, comm);

  struct mpi_p2p_msg_t* msg = __mpi_send_generic( thread_id, CUR_RANK, dest, len, tag, NULL, comm);

  if(!IS_TIME_SET(msg->times[stop_isend]))
    msg->times[stop_isend] = CUR_TIME(CUR_INDEX);

  if(!IS_TIME_SET(msg->times[start_swait]))
    msg->times[start_swait] = CUR_TIME(CUR_INDEX);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Send");
}

void
handle_mpi_stop_send ()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_leave_function(MPI_SEND_ID);

  int dest;
  int tag;
  app_ptr comm_id;
  GET_PARAM_PACKED_3(CUR_EV, dest, tag, comm_id);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  int actual_dest = ezt_get_global_rank(comm, dest);
  assert(actual_dest != -1);

  __stop_send_message(CUR_TIME(CUR_INDEX), CUR_RANK, actual_dest, -1, tag, thread_id, NULL);

  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void
handle_mpi_start_recv ()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_RECV_ID);

  int len;
  int src;
  int dest = CUR_RANK;
  uint32_t tag;
  app_ptr comm_id;
  GET_PARAM_PACKED_4(CUR_EV, len, src, tag, comm_id);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, (app_ptr) comm_id);

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);
  int actual_src = src;
  if(src != p_info->__MPI_ANY_SOURCE) {
    actual_src = ezt_get_global_rank(comm, src);
    assert(actual_src!= -1);
  }

  struct mpi_p2p_msg_t* msg = __start_recv_message(CUR_TIME(CUR_INDEX),
      actual_src, dest, len, tag, thread_id, NULL);

  if(!IS_TIME_SET(msg->times[stop_irecv]))
    msg->times[stop_irecv] = CUR_TIME(CUR_INDEX);

  if(!IS_TIME_SET(msg->times[start_rwait]))
    msg->times[start_rwait] = CUR_TIME(CUR_INDEX);

#if DEBUG_MSG_MATCHING
  printf("Start recv(src=%x, dest=%x, tag=%x, len=%d, comm=%p\n", src, dest, tag, len, comm_id);
#endif
  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Recv");
}

void
handle_mpi_stop_recv ()
{
  FUNC_NAME;

  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  int len;
  int src;
  int dest = CUR_RANK;
  uint32_t tag;
  app_ptr comm;
  GET_PARAM_PACKED_4(CUR_EV, len, src, tag, comm);
  uint64_t timestamp = CUR_TIME(CUR_INDEX);
#if DEBUG_MSG_MATCHING
  printf("Stop recv(src=%x, dest=%x, tag=%x, len=%d, comm=%p\n", src, dest, tag, len, comm);
#endif
  if(__mpi_stop_recv_generic (&timestamp, thread_id, src, CUR_RANK, len, tag, NULL, comm)) {
    /* waiting for a mpi_send that matches */
    return;
  }

  __ezt_mpi_leave_function(MPI_RECV_ID);

  MPI_CHANGE() popState(NS_TO_MS(timestamp), "ST_Thread", thread_id);
}

void handle_mpi_start_sendrecv ()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_SENDRECV_ID);

  int rsrc = 0;
  int rdest = CUR_RANK;
  uint32_t rtag = 0;
  int rlen = 0;

  int ssrc = CUR_RANK;
  int sdest = 0;
  uint32_t stag = 0;
  int slen = 0;

  app_ptr comm = 0;

  GET_PARAM_PACKED_4(CUR_EV, rlen, rsrc, rtag, comm);

 /* process the recv */
  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
  GET_PARAM_PACKED_3(CUR_EV, slen, sdest, stag);

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  struct mpi_p2p_msg_t* msg;
  /* process the send */
  if (ssrc != p_info->__MPI_PROC_NULL && sdest != p_info->__MPI_PROC_NULL) {
      msg = __mpi_send_generic( thread_id, ssrc, sdest, slen, stag, NULL, comm);
      if(!IS_TIME_SET(msg->times[stop_isend])) {
	msg->times[stop_isend] = CUR_TIME(CUR_INDEX);
      }
      if(!IS_TIME_SET(msg->times[start_swait])) {
	msg->times[start_swait] = CUR_TIME(CUR_INDEX);
      }
  }
  /* process the recv */
  if (rsrc != p_info->__MPI_PROC_NULL && rdest != p_info->__MPI_PROC_NULL) {
    int actual_src = rsrc;
    if(rsrc != p_info->__MPI_ANY_SOURCE) {
      actual_src = ezt_get_global_rank_generic(CUR_INDEX, comm, rsrc);
      assert(actual_src!= -1);
    }
    msg = __start_recv_message(CUR_TIME(CUR_INDEX),
			       actual_src, rdest, rlen, rtag, thread_id, NULL);
    if(!IS_TIME_SET(msg->times[stop_irecv]))
      msg->times[stop_irecv] = CUR_TIME(CUR_INDEX);
    if(!IS_TIME_SET(msg->times[start_rwait]))
      msg->times[start_rwait] = CUR_TIME(CUR_INDEX);
  }
  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Sendrecv");

}

void handle_mpi_stop_sendrecv ()
{

  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  /* we need to process the recv first because we may have to re-synchronize traces
   * and thus, we may have to call handle_mpi_sop_sendrecv several times (until the corresponding
   * send occurs).
   */
  int rsrc = 0;
  int rdest = CUR_RANK;
  uint32_t rtag = 0;
  int rlen = 0;

  int ssrc = CUR_RANK;
  int sdest = 0;
  uint32_t stag = 0;
  int slen = 0;

  app_ptr comm;
  GET_PARAM_PACKED_4(CUR_EV, rlen, rsrc, rtag, comm);

  uint64_t timestamp = CUR_TIME(CUR_INDEX);

  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);
  /* process the recv */
  if (rsrc != p_info->__MPI_PROC_NULL && rdest != p_info->__MPI_PROC_NULL) {
    if(__mpi_stop_recv_generic (&timestamp, thread_id, rsrc, rdest, rlen, rtag, NULL, comm)) {
      /* waiting for a mpi _send that matches */
      return;
    }
  }

/* process the send */
  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
  GET_PARAM_PACKED_3(CUR_EV, slen, sdest, stag);
  if (ssrc != p_info->__MPI_PROC_NULL && sdest != p_info->__MPI_PROC_NULL) {
    int actual_dest = ezt_get_global_rank_generic(CUR_INDEX, comm, sdest);
    __stop_send_message(timestamp, CUR_RANK, actual_dest, -1, stag, thread_id, NULL);
  }

  __ezt_mpi_leave_function(MPI_SENDRECV_ID);

  /* pop the sendrecv state */
  MPI_CHANGE() popState(NS_TO_MS(timestamp), "ST_Thread", thread_id);
}

void handle_mpi_start_sendrecv_replace ()
{
  handle_mpi_start_sendrecv();
  /* todo: implement this */
}

void handle_mpi_stop_sendrecv_replace ()
{
  handle_mpi_stop_sendrecv();
}

/* todo: implement this ! */
void handle_mpi_start_bsend()
{
  /* for now, let's assume that bsend and send are the same */
  handle_mpi_start_send ();
}

void handle_mpi_stop_bsend()
{
  handle_mpi_stop_send ();
}

void handle_mpi_start_ssend()
{
  handle_mpi_start_send ();
}

void handle_mpi_stop_ssend()
{
  handle_mpi_stop_send ();
}

void handle_mpi_start_rsend()
{
  handle_mpi_start_send ();
}

void handle_mpi_stop_rsend()
{
  handle_mpi_stop_send ();
}

void handle_mpi_isend_generic(const char* state)
{
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_ISEND_ID);

  int len;
  __attribute__((unused)) int src = CUR_RANK;
  int dest;
  uint32_t tag;
  app_ptr app_req;
  app_ptr comm;
  GET_PARAM_PACKED_5(CUR_EV, len, dest, tag, app_req, comm);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
      app_req,
      mpi_req_send);
  assert(mpi_req);
  __mpi_send_generic( thread_id, CUR_RANK, dest, len, tag, mpi_req, comm);

  /* todo: add a send event here */
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, state);
}

void handle_mpi_isend()
{
  FUNC_NAME;
  handle_mpi_isend_generic("STV_MPI_Isend");
}

void handle_mpi_ibsend()
{
  FUNC_NAME;
  handle_mpi_isend_generic("STV_MPI_Ibsend");
}

void handle_mpi_issend()
{
  FUNC_NAME;
  handle_mpi_isend_generic("STV_MPI_Irsend");
}

void handle_mpi_irsend()
{
  FUNC_NAME;
  handle_mpi_isend_generic("STV_MPI_Irsend");
}

void handle_mpi_stop_isend_generic() {

  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_leave_function(MPI_ISEND_ID);

  app_ptr app_req;
  GET_PARAM_PACKED_1(CUR_EV, app_req);

  struct mpi_request * mpi_req = __mpi_find_pending_mpi_req(CUR_RANK,
							    app_req,
							    mpi_req_send);
  assert(mpi_req);

  struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK,
								mpi_req);
  assert(msg);
  if(!IS_TIME_SET(msg->times[stop_isend]))
      msg->times[stop_isend] = CUR_TIME(CUR_INDEX);

  /* todo: add a send event here */
  MPI_CHANGE() popState (CURRENT, "ST_Thread", thread_id);
}


void handle_mpi_stop_isend() {
  FUNC_NAME;
  handle_mpi_stop_isend_generic();
}


void handle_mpi_stop_ibsend() {
  FUNC_NAME;
  handle_mpi_stop_isend_generic();
}

void handle_mpi_stop_issend() {
  FUNC_NAME;
  handle_mpi_stop_isend_generic();
}

void handle_mpi_stop_irsend() {
  FUNC_NAME;
  handle_mpi_stop_isend_generic();
}

void handle_mpi_irecv()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_IRECV_ID);

  int len;
  int src;
  int dest = CUR_RANK;
  uint32_t tag;
  app_ptr app_req;
  app_ptr comm;
  GET_PARAM_PACKED_5(CUR_EV, len, src, tag, app_req, comm);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK, app_req, mpi_req_recv);
  assert(mpi_req);
  __mpi_start_recv_generic (thread_id, src, dest, len, tag, mpi_req, comm);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Irecv");
}

void handle_mpi_stop_irecv()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  app_ptr app_req;
  GET_PARAM_PACKED_1(CUR_EV, app_req);

  __ezt_mpi_leave_function(MPI_IRECV_ID);

  struct mpi_request * mpi_req = __mpi_find_pending_mpi_req(CUR_RANK,
							    app_req,
							    mpi_req_recv);
  assert(mpi_req);

  struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK,
								mpi_req);
  assert(msg);
  if(!IS_TIME_SET(msg->times[stop_irecv]))
      msg->times[stop_irecv] = CUR_TIME(CUR_INDEX);

  /* todo: add a recv event here */
  MPI_CHANGE() popState (CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_start_put()
{
  /* todo */
}

void handle_mpi_stop_put()
{
  /* todo */
}

void handle_mpi_start_get()
{
  /* todo */
}

void handle_mpi_stop_get()
{
  /* todo */
}

int __handle_mpi_start_wait_generic(struct mpi_request * req) {
  if (!req || req->status == mpi_req_status_none) {
    return 0;
  }

  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  if(req->req_type == mpi_req_recv) {
    /*  this is a receive request */
    struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK, req);
    assert(msg);

    msg->times[start_rwait] = CUR_TIME(CUR_INDEX);
    return 1;
  } else if(req->req_type == mpi_req_send) {
    /*  this is a send request */

    struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK, req);
    assert(msg);

    msg->times[start_swait] = CUR_TIME(CUR_INDEX);
    return 1;
  } else {
    /* non blocking collective */
#if 0
      struct mpi_coll_msg_t *coll_msg = req->coll_msg;

    switch(coll_msg->type) {
    case mpi_coll_barrier:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Barrier");
      break;
    case mpi_coll_bcast:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_BCast");
      break;
    case mpi_coll_gather:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Gather");
      break;
    case mpi_coll_scatter:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Scatter");
      break;
    case mpi_coll_allgather:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Allgather");
      break;
    case mpi_coll_alltoall:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Alltoall");
      break;
    case mpi_coll_reduce:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Reduce");
      break;
    case mpi_coll_allreduce:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Allreduce");
      break;
    case mpi_coll_reduce_scatter:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_reduce_scatter");
      break;
    case mpi_coll_scan:
      MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Scan");
      break;
    default:
      fprintf(stderr, "%s: Unknown collective type : %d\n", __FUNCTION__, coll_msg->type);
    }
#endif
    return 1;
  }

  return 0;
}


void handle_mpi_start_wait()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_WAIT_ID);

  app_ptr mpi_req;
  GET_PARAM_PACKED_1(CUR_EV, mpi_req);

  struct mpi_request * req = __mpi_find_pending_mpi_req(CUR_RANK, mpi_req, mpi_req_none);

  if(__handle_mpi_start_wait_generic(req)) {
    MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Wait");
  }
}

void handle_mpi_start_waitany()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  __ezt_mpi_enter_function(MPI_WAITANY_ID);

  int nb_reqs;
  GET_PARAM_PACKED_1(CUR_EV, nb_reqs);
  int i;

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Wait");

  for(i=0; i<nb_reqs; i++) {
    wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
    app_ptr mpi_req;
    GET_PARAM_PACKED_1(CUR_EV, mpi_req);
    if(mpi_req == p_info->__MPI_REQUEST_NULL)
    continue;

    struct mpi_request * req = __mpi_find_pending_mpi_req(CUR_RANK, mpi_req, mpi_req_none);

    __handle_mpi_start_wait_generic(req);
  }
}

void handle_mpi_start_waitall()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  __ezt_mpi_enter_function(MPI_WAITALL_ID);

  int nb_reqs;
  GET_PARAM_PACKED_1(CUR_EV, nb_reqs);
  int i;
  double cur_time = CURRENT;

  for(i=0; i<nb_reqs; i++) {
    wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
    app_ptr mpi_req;
    GET_PARAM_PACKED_1(CUR_EV, mpi_req);
    if(mpi_req == p_info->__MPI_REQUEST_NULL) {
      continue;
    }

    struct mpi_request * req = __mpi_find_pending_mpi_req(CUR_RANK, mpi_req, mpi_req_none);
    __handle_mpi_start_wait_generic(req);
  }

  MPI_CHANGE() pushState(cur_time, "ST_Thread", thread_id, "STV_MPI_Wait");
}

static int __handle_mpi_test_success(app_ptr req, uint64_t timestamp)
{
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  struct mpi_request * mpi_req = __mpi_find_pending_mpi_req(CUR_RANK, req, mpi_req_none);
  if(!mpi_req) {
    /* this means the request was already freed, let's skip this one */
    return 1;
  }


  if(mpi_req->req_type == mpi_req_recv) {
    /* this is a recv request */
    struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK, mpi_req);
    assert(msg);

    assert(msg->recver_request == mpi_req);
    if(__mpi_stop_recv_generic(&timestamp, thread_id, msg->src, msg->dest, msg->len, msg->tag, mpi_req,p_info->__MPI_COMM_WORLD.comm_id))
    return 0;
  } else if(mpi_req->req_type == mpi_req_send) {
    struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK, mpi_req);
    assert(msg);

    assert(mpi_req->req_type == mpi_req_send);
    assert(msg->sender_request == mpi_req);

    /* this is a send request */
    __stop_send_message(timestamp, msg->src, msg->dest, msg->len, msg->tag, thread_id, mpi_req);

  } else {
    /* non blocking collective */
    int local_rank = -1;
    int i;
    /* find the rank of the process in this collective */
    for(i=0; i<mpi_req->coll_msg->comm_size; i++) {
      if(mpi_req->coll_msg->requests[i] == mpi_req) {
	local_rank = i;
	break;
      }
    }

    assert(local_rank >= 0);

    struct mpi_coll_msg_t* msg = __leave_coll(timestamp,
	mpi_req->coll_msg->type,
	mpi_req->coll_msg->comm[local_rank],
	CUR_RANK,
	mpi_req,
	mpi_req->coll_msg->thread_ids[local_rank]);
    if(__mpi_barrier_stop_generic(msg, CUR_RANK))
      return 0;

    if(msg->type == mpi_coll_barrier) {
      mpi_synchronize_processes(msg, local_rank);
    }
  }

  return 1;
}

static void handle_mpi_start_probe()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_PROBE_ID);
  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Probe");
}

static void handle_mpi_stop_probe()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_leave_function(MPI_PROBE_ID);

  char* probe_str = NULL;
  int src;
  uint32_t tag;
  int len;
  GET_PARAM_PACKED_3(CUR_EV, src, tag, len);
  int res __attribute__ ((__unused__));
  res = asprintf(&probe_str, "src=%d, tag=%x, length=%d", src, tag, len);

  MPI_CHANGE() addEvent(CURRENT, "E_MPI_Probe_success", thread_id, probe_str);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);

  free(probe_str);
}

void handle_mpi_iprobe_success()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  char* probe_str = NULL;
  int src;
  uint32_t tag;
  int len;
  GET_PARAM_PACKED_3(CUR_EV, src, tag, len);
  int res __attribute__ ((__unused__));
  res = asprintf(&probe_str, "src=%d, tag=%x, length=%d", src, tag, len);
  MPI_CHANGE() addEvent(CURRENT, "E_MPI_Iprobe_success", thread_id, probe_str);

  free(probe_str);
}

void handle_mpi_iprobe_failed()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  char* probe_str = NULL;
  int src;
  uint32_t tag;
  GET_PARAM_PACKED_2(CUR_EV, src, tag);
  int res __attribute__ ((__unused__));
  res = asprintf(&probe_str, "src=%d, tag=%x", src, tag);
  MPI_CHANGE() addEvent(CURRENT, "E_MPI_Iprobe_failed", thread_id, probe_str);

  free(probe_str);
}

void __handle_cancel_mpi_request(app_ptr req)
{
  #warning TODO

  struct mpi_request * mpi_req = __mpi_find_mpi_req(CUR_RANK, req, mpi_req_none);
  if(!mpi_req) {
    /* this means the request was already freed, let's skip this one */
    return ;
  }
  mpi_req->status = mpi_req_status_none;
}

void handle_mpi_cancel()
{
  FUNC_NAME;
  app_ptr req;
  GET_PARAM_PACKED_1(CUR_EV, req);
  __handle_cancel_mpi_request(req);
}

void handle_mpi_stop_wait()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr req;
  GET_PARAM_PACKED_1(CUR_EV, req);

  if(!__handle_mpi_test_success(req, CUR_TIME(CUR_INDEX)))
    return;

  __ezt_mpi_leave_function(MPI_WAIT_ID);

  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}


void handle_mpi_stop_wait_failed()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr req;
  GET_PARAM_PACKED_1(CUR_EV, req);
  __handle_cancel_mpi_request(req);

  __ezt_mpi_leave_function(MPI_WAIT_ID);
MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}


void handle_mpi_stop_waitany()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  int nb_reqs;
  int index;
  GET_PARAM_PACKED_2(CUR_EV, nb_reqs, index);

  /* process each request*/
  int i;
  uint64_t timestamp = CUR_TIME(CUR_INDEX);

  for(i=0; i<nb_reqs; i++) {
    wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
    app_ptr mpi_req;
    GET_PARAM_PACKED_1(CUR_EV, mpi_req);

    if(index == i) {
      /* the successful request */
      if(__handle_mpi_test_success(mpi_req, timestamp))
	continue;
    } else {
      if(mpi_req == p_info->__MPI_REQUEST_NULL)
      continue;
      /* the other requests */
      struct mpi_request * req = __mpi_find_pending_mpi_req(CUR_RANK, mpi_req, mpi_req_none);
      if (!req) {
	continue;
      }
      assert(req);
      struct mpi_p2p_msg_t* msg = __mpi_find_p2p_message_by_mpi_req(CUR_RANK, req);
      assert(msg);

      if(req->req_type == mpi_req_recv) {
	/*  this is a receive request */
	msg->times[start_rwait] = TIME_INIT;
      } else {
	/*  this is a send request */
	msg->times[start_swait] = timestamp;
      }
    }
  }

  __ezt_mpi_leave_function(MPI_WAITANY_ID);

  MPI_CHANGE() popState(NS_TO_MS(timestamp), "ST_Thread", thread_id);
}

struct mpi_stop_waitall_replay {
  int nb_reqs;
  int i;
  uint64_t timestamp;
  app_ptr mpi_req;
};

void handle_mpi_stop_waitall( struct mpi_stop_waitall_replay *r)
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  int nb_reqs;
  if(r)
  nb_reqs = r->nb_reqs;
  else
  GET_PARAM_PACKED_1(CUR_EV, nb_reqs);

  uint64_t timestamp = CUR_TIME(CUR_INDEX);
  /* process each request*/
  int i = 0;
  if(r) {
    i = r->i;
    timestamp = r->timestamp;
  }

  for(; i<nb_reqs; i++) {

    if(!(r && r->i == i)) {
      /* unless this is the replay first look, fetch next event */
      wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);
    }

    app_ptr mpi_req;
    GET_PARAM_PACKED_1(CUR_EV, mpi_req);

    if(r && (r->i == i)) {
      assert(mpi_req == r->mpi_req);
    }

    if(mpi_req == p_info->__MPI_REQUEST_NULL)
      continue;

    if(! __handle_mpi_test_success(mpi_req, timestamp)) {
      /* The matching isend has not occured yet. We need to wait until it happens */

      struct mpi_stop_waitall_replay* replay = malloc(sizeof(struct mpi_stop_waitall_replay));
      replay->nb_reqs = nb_reqs;
      replay->i = i;
      replay->mpi_req = mpi_req;
      replay->timestamp = timestamp;
      ask_for_replay(CUR_INDEX, (void (*)(void *)) handle_mpi_stop_waitall, replay);

      goto out;
    }
  }

  __ezt_mpi_leave_function(MPI_WAITALL_ID);
  MPI_CHANGE() popState(NS_TO_MS(timestamp), "ST_Thread", thread_id);

  out:
  if( r )
  free(r);
  return;
}

void handle_mpi_test_success()
{
  FUNC_NAME;
  app_ptr mpi_req;
  GET_PARAM_PACKED_1(CUR_EV, mpi_req);
  if(__handle_mpi_test_success(mpi_req, CUR_TIME(CUR_INDEX)))
    return;

}

void handle_mpi_start_BCast()
{
  /* some collective communications won't work with communicators != COMM_WORLD
   * (rank in the wrong communicator used)
   */
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  __ezt_mpi_enter_function(MPI_BCAST_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_BCast");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  int root;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, root, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_bcast,
					    comm, my_rank, data_size, NULL, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Gather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_GATHER_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Gather");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  int root;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, data_size, root);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  assert(comm->my_rank == my_rank);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_gather,
					    comm, my_rank, data_size, NULL, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);

}

void handle_mpi_start_Gatherv()
{
  handle_mpi_start_Gather();
}

void handle_mpi_start_Scatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_SCATTER_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Scatter");

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  int root;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, data_size, root);

  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_scatter,
					    comm, my_rank, data_size, NULL, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Scatterv()
{
  handle_mpi_start_Scatter();
}

void handle_mpi_start_Allgather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
__ezt_mpi_enter_function(MPI_ALLGATHER_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Allgather");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, comm_size, my_rank, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  assert(comm);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_allgather,
					    comm, my_rank, data_size, NULL, thread_id);

  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Allgatherv()
{
  handle_mpi_start_Allgather();
}

void handle_mpi_start_Alltoall()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_ALLTOALL_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Alltoall");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, comm_size, my_rank, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_alltoall,
					    comm, my_rank, data_size, NULL, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Alltoallv()
{
  handle_mpi_start_Alltoall();
}

void handle_mpi_start_Reduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_REDUCE_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Reduce");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  int root;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, data_size, root);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_reduce,
					    comm, my_rank, data_size, NULL, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Allreduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_ALLREDUCE_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Allreduce");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, comm_size, my_rank, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_allreduce,
					    comm, my_rank, data_size, NULL, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Reduce_scatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_REDUCE_SCATTER_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_reduce_scatter");
  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, comm_size, my_rank, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_reduce_scatter,
					    comm, my_rank, data_size, NULL, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_Scan()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_SCAN_ID);

  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Scan");

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, comm_size, my_rank, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_scan,
					    comm, my_rank, data_size, NULL, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
}

void handle_mpi_start_barrier()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  MPI_CHANGE() pushState(CURRENT, "ST_Thread", thread_id, "STV_MPI_Barrier");
  __ezt_mpi_enter_function(MPI_BARRIER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, my_rank, comm_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_barrier,
					    comm, my_rank, 0, NULL, thread_id);

  __mpi_barrier_start_generic(msg, my_rank);
}


void handle_mpi_stop_BCast()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_bcast,
					    comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_BCAST_ID);

  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Gather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  assert(comm->my_rank == my_rank);
  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_gather,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_GATHER_ID);

  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Gatherv()
{
  handle_mpi_stop_Gather();
}

void handle_mpi_stop_Scatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_scatter,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_SCATTER_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Scatterv()
{
  handle_mpi_stop_Scatter();
}

void handle_mpi_stop_Allgather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_allgather,
      comm, my_rank, NULL, thread_id);
  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_ALLGATHER_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Allgatherv()
{
  handle_mpi_stop_Allgather();
}

void handle_mpi_stop_Alltoall()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_alltoall,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_ALLTOALL_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Alltoallv()
{
  handle_mpi_stop_Alltoall();
}

void handle_mpi_stop_Reduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_reduce,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank)) {
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;
  }

  __ezt_mpi_leave_function(MPI_REDUCE_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Allreduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_allreduce,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_ALLREDUCE_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Reduce_scatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_reduce_scatter,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_REDUCE_SCATTER_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

void handle_mpi_stop_Scan()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, comm_size, my_rank);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_scan,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  __ezt_mpi_leave_function(MPI_SCAN_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

static void mpi_synchronize_processes(struct mpi_coll_msg_t* msg, int my_rank) {
  uint64_t my_stop_time = msg->times[my_rank][stop_coll];
  int i;
  for(i=0; i<msg->comm[my_rank]->comm_size; i++) {
    if(i != my_rank) {
      /* search for the max(times[start_coll], my_stop_coll) */
      if(msg->times[i][start_coll] > my_stop_time) {
	my_stop_time = msg->times[i][start_coll];
      }
    }
  }

  /* If one of the processes arrived after process #my_rank left, there's a
   * synchronisation problem: we need to delay process #my_rank so that it
   * leaves the collective operation after all the processes arrived.
   */
  if(my_stop_time > msg->times[my_rank][stop_coll]) {
    int global_rank = ezt_get_global_rank(msg->comm[my_rank], my_rank);
    msg->times[my_rank][stop_coll] += add_delay_to_trace(global_rank,
	msg->times[my_rank][stop_coll],
	my_stop_time,
	msg->thread_ids[my_rank]);
  }
}

void handle_mpi_stop_barrier()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  GET_PARAM_PACKED_3(CUR_EV, comm_id, my_rank, comm_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_coll_msg_t* msg = __leave_coll(CUR_TIME(CUR_INDEX), mpi_coll_barrier,
      comm, my_rank, NULL, thread_id);

  if(__mpi_barrier_stop_generic(msg, my_rank))
    /* one of the process that belong to this collective didn't reach it.
     * We need to wait until it reach the begining of the collective before
     * pursuing.
     */
    return;

  mpi_synchronize_processes(msg, my_rank);

  __ezt_mpi_leave_function(MPI_BARRIER_ID);
  MPI_CHANGE() popState(CURRENT, "ST_Thread", thread_id);
}

/* non-blocking collective */
void handle_mpi_Ibcast()
{
  /* some collective communications won't work with communicators != COMM_WORLD
   * (rank in the wrong communicator used)
   */
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IBCAST_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  int root;
  GET_PARAM_PACKED_6(CUR_EV, comm_id, comm_size, my_rank, req, data_size, root);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_bcast,
					    comm, my_rank, data_size, mpi_req, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Ibcast");
}

void handle_mpi_Ibarrier()
{
  /* some collective communications won't work with communicators != COMM_WORLD
   * (rank in the wrong communicator used)
   */
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IBARRIER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  GET_PARAM_PACKED_4(CUR_EV, comm_id, my_rank, comm_size, req);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_barrier,
					    comm, my_rank, 0, mpi_req, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Ibarrier");
}

void handle_mpi_Igather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IGATHER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  int root;
  GET_PARAM_PACKED_6(CUR_EV, comm_id, comm_size, my_rank, req, data_size, root);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_gather,
					    comm, my_rank, data_size, mpi_req, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Igather");
}

void handle_mpi_Igatherv()
{
  handle_mpi_Igather();
}

void handle_mpi_Iscatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_ISCATTER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  int data_size;
  app_ptr req;
  int root;
  GET_PARAM_PACKED_6(CUR_EV, comm_id, comm_size, my_rank, req, data_size, root);

  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_scatter,
					    comm, my_rank, data_size, mpi_req, thread_id);

  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Iscatter");
}

void handle_mpi_Iscatterv()
{
  handle_mpi_Iscatter();
}

void handle_mpi_Iallgather()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IALLGATHER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, req, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_allgather,
					    comm, my_rank, data_size, mpi_req, thread_id);

  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Iallgather");
}

void handle_mpi_Iallgatherv()
{
  handle_mpi_Iallgather();
}

void handle_mpi_Ialltoall()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IALLTOALL_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, req, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_alltoall,
					    comm, my_rank, data_size, mpi_req, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Ialltoall");
}

void handle_mpi_Ialltoallv()
{
  handle_mpi_Ialltoall();
}

void handle_mpi_Ireduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IREDUCE_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  int root;
  GET_PARAM_PACKED_6(CUR_EV, comm_id, comm_size, my_rank, req, data_size, root);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_reduce,
					    comm, my_rank, data_size, mpi_req, thread_id);
  msg->root_process = root;
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Ireduce");
}

void handle_mpi_Iallreduce()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IALLREDUCE_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, req, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_allreduce,
					    comm, my_rank, data_size, mpi_req, thread_id);
  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Iallreduce");
}

void handle_mpi_Ireduce_scatter()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_IREDUCE_SCATTER_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, req, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);

  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX),
					    mpi_coll_reduce_scatter, comm,
					    my_rank, data_size, mpi_req, thread_id);

  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Ireduce_scatter");
}

void handle_mpi_Iscan()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  __ezt_mpi_enter_function(MPI_ISCAN_ID);

  app_ptr comm_id;
  int my_rank;
  int comm_size;
  app_ptr req;
  int data_size;
  GET_PARAM_PACKED_5(CUR_EV, comm_id, comm_size, my_rank, req, data_size);
  struct ezt_mpi_comm* comm = ezt_find_communicator(CUR_INDEX, comm_id);
  struct mpi_request * mpi_req = __mpi_new_mpi_request(CUR_RANK,
						       req,
						       mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __enter_coll(CUR_TIME(CUR_INDEX), mpi_coll_scan,
					    comm, my_rank, data_size, mpi_req, thread_id);

  __mpi_barrier_start_generic(msg, my_rank);
  MPI_CHANGE() pushState (CURRENT, "ST_Thread", thread_id, "STV_MPI_Iscan");
}

struct mpi_coll_msg_t* __mpi_find_coll_message_by_mpi_req(int rank,
							  struct mpi_request *mpi_req);

static void handle_mpi_stop_nonblocking_collective_generic() {

  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);

  app_ptr app_req;
  GET_PARAM_PACKED_1(CUR_EV, app_req);

  struct mpi_request * mpi_req = __mpi_find_pending_mpi_req(CUR_RANK,
							    app_req,
							    mpi_req_coll);
  assert(mpi_req);

  struct mpi_coll_msg_t* msg = __mpi_find_coll_message_by_mpi_req(CUR_RANK,
								  mpi_req);
  assert(msg);

  MPI_CHANGE() popState (CURRENT, "ST_Thread", thread_id);
}

/* non-blocking collective */
void handle_mpi_stop_ibcast() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IBCAST_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_ibarrier() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IBARRIER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_igather() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IGATHER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_igatherv() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IGATHER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iscatter() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_ISCATTER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iscatterv() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_ISCATTER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iallgather() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IALLGATHER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iallgatherv() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IALLGATHER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_ialltoall() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IALLTOALL_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_ialltoallv() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IALLTOALL_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_ireduce() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IREDUCE_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iallreduce() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IALLREDUCE_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_ireduce_scatter() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_IREDUCE_SCATTER_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

void handle_mpi_stop_iscan() {
  FUNC_NAME;
  __ezt_mpi_leave_function(MPI_ISCAN_ID);
  handle_mpi_stop_nonblocking_collective_generic();
}

/* end of non-blocking collective */

void handle_mpi_spawn()
{
  FUNC_NAME;
  /* This event happens when a process calls MPI_Comm_spawn
   * We have to:
   * - add the process info (PID, etc.) to the spawn_list
   * - create a few links
   */
  int pid;
  int nb_spawned;
  GET_PARAM_PACKED_2(CUR_EV, pid, nb_spawned);

  char *link_prefix = NULL;
  int ret = asprintf(&link_prefix, "%s_", CUR_ID);
  assert(ret>=0);
  int i;

  /* add the current spawn to the list of pending spawns */
  struct mpi_spawn_t *spawn = malloc(sizeof(struct mpi_spawn_t));
  spawn->nb_children = nb_spawned;
  spawn->start_time = CUR_TIME(CUR_INDEX);
  spawn->ppid = pid;
  spawn->parent_trace = CUR_TRACE;

  struct ezt_list_token_t* token = malloc(sizeof(struct ezt_list_token_t));
  token->data = spawn;
  ezt_list_add(&spawn_list, token);

  /* start one link per spawned process */
  for(i=0; i<nb_spawned; i++) {
    char *link_id = NULL;
    ret = asprintf(&link_id, "%s%d", link_prefix, i);
    assert(ret>=0);

    MPI_CHANGE() startLink (CURRENT, "L_MPI_SPAWN", "C_Prog", CUR_ID, NULL,
	link_id, link_id);
    free(link_id);
  }

  free(link_prefix);
}

void handle_mpi_send_init()
{
  FUNC_NAME;

  app_ptr buffer;
  int len;
  int dest;
  int src = CUR_RANK;

  GET_PARAM_PACKED_3(CUR_EV, buffer, len, dest);

  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);

  uint32_t tag;
  app_ptr comm;
  app_ptr mpi_req;
  GET_PARAM_PACKED_3(CUR_EV, tag, comm, mpi_req);

  int actual_dest = ezt_get_global_rank_generic(CUR_INDEX, comm, dest);
  assert(actual_dest!= -1);

  struct mpi_pers_req_t __attribute__((unused)) *req = __pers_init(CURRENT, mpi_req_send, buffer,
      src, actual_dest, len, tag, mpi_req);

  DECLARE_CUR_THREAD(cur_thread);
}

void handle_mpi_bsend_init() {
  handle_mpi_send_init();
}

void handle_mpi_rsend_init()
{
  handle_mpi_send_init();
}

void handle_mpi_ssend_init()
{
  handle_mpi_send_init();
}

void handle_mpi_recv_init()
{
  FUNC_NAME;
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  app_ptr buffer;
  int len;
  int src;
  int dest = CUR_RANK;
  GET_PARAM_PACKED_3(CUR_EV, buffer, len, src);

  wait_for_an_event(CUR_TRACE->id, EZTRACE_MPI_Info);

  uint32_t tag;
  app_ptr comm;
  app_ptr mpi_req;
  GET_PARAM_PACKED_3(CUR_EV, tag, comm, mpi_req);

  int actual_src = src;
  if(src != p_info->__MPI_ANY_SOURCE) {
    actual_src = ezt_get_global_rank_generic(CUR_INDEX, comm, src);
    assert(actual_src!= -1);
  }

  struct mpi_pers_req_t __attribute__((unused)) *req = __pers_init(CURRENT, mpi_req_recv, buffer,
      actual_src, dest, len, tag, mpi_req);

  DECLARE_CUR_THREAD(cur_thread);
}

void handle_mpi_start()
{
  FUNC_NAME;
  DECLARE_THREAD_ID_STR(thread_id, CUR_INDEX, CUR_THREAD_ID);
  INIT_MPI_PROCESS_INFO(GET_PROCESS_INFO(CUR_INDEX), p_info);

  app_ptr mpi_req;
  GET_PARAM_PACKED_1(CUR_EV, mpi_req);
  struct mpi_pers_req_t* req = __pers_start(CURRENT,
      CUR_RANK,
      mpi_req);
  assert(req);

  if(req->type == mpi_req_send) {
  /* todo: mpi communicator */
    struct mpi_p2p_msg_t* msg = __mpi_send_generic( thread_id, CUR_RANK, req->dest, req->len, req->tag, req->mpi_req, p_info->__MPI_COMM_WORLD.comm_id);
    if(!IS_TIME_SET(msg->times[stop_isend]))
      msg->times[stop_isend] = CUR_TIME(CUR_INDEX);
  } else {
    struct mpi_p2p_msg_t* msg = __mpi_start_recv_generic( thread_id, req->src, CUR_RANK, req->len, req->tag, req->mpi_req, p_info->__MPI_COMM_WORLD.comm_id);
    if(!IS_TIME_SET(msg->times[stop_irecv]))
      msg->times[stop_irecv] = CUR_TIME(CUR_INDEX);
  }

  /* todo: add a send event here */
}

int eztrace_convert_mpi_init()
{
  if(get_mode() == EZTRACE_CONVERT) {
    /* Process send a message */
    addEntityValue("STV_MPI_Send", "ST_Thread", "Sending", GTG_ORANGE);
    addEntityValue("STV_MPI_Isend", "ST_Thread", "MPI_Isend", GTG_ORANGE);
    addEntityValue("STV_MPI_Ibsend", "ST_Thread", "MPI_Ibsend", GTG_ORANGE);
    addEntityValue("STV_MPI_Issend", "ST_Thread", "MPI_Issend", GTG_ORANGE);
    addEntityValue("STV_MPI_Irsend", "ST_Thread", "MPI_Irsend", GTG_ORANGE);

    addEntityValue("STV_MPI_Sendrecv", "ST_Thread", "SendRecv", GTG_ORANGE);
    /* Process receive a message */
    addEntityValue("STV_MPI_Recv", "ST_Thread", "Receiving", GTG_PURPLE);
    addEntityValue("STV_MPI_Irecv", "ST_Thread", "MPI_Irecv", GTG_PURPLE);

    addEntityValue("STV_MPI_Wait", "ST_Thread", "MPI_Wait", GTG_RED);

    addEntityValue("STV_MPI_Barrier", "ST_Thread", "MPI_Barrier", GTG_PINK);
    addEntityValue("STV_MPI_BCast", "ST_Thread", "MPI_BCast", GTG_PINK);
    addEntityValue("STV_MPI_Gather", "ST_Thread", "MPI_Gather", GTG_PINK);
    addEntityValue("STV_MPI_Scatter", "ST_Thread", "MPI_Scatter", GTG_PINK);
    addEntityValue("STV_MPI_Allgather", "ST_Thread", "MPI_Allgather", GTG_PINK);
    addEntityValue("STV_MPI_Alltoall", "ST_Thread", "MPI_Alltoall", GTG_PINK);
    addEntityValue("STV_MPI_Reduce", "ST_Thread", "MPI_Reduce", GTG_PINK);
    addEntityValue("STV_MPI_Allreduce", "ST_Thread", "MPI_Allreduce", GTG_PINK);
    addEntityValue("STV_MPI_reduce_scatter", "ST_Thread", "MPI_Reduce_scatter", GTG_PINK);
    addEntityValue("STV_MPI_Scan", "ST_Thread", "MPI_Scan", GTG_PINK);

    addEntityValue("STV_MPI_Ibarrier", "ST_Thread", "MPI_Ibarrier", GTG_PINK);
    addEntityValue("STV_MPI_Ibcast", "ST_Thread", "MPI_Ibcast", GTG_PINK);
    addEntityValue("STV_MPI_Igather", "ST_Thread", "MPI_Igather", GTG_PINK);
    addEntityValue("STV_MPI_Iscatter", "ST_Thread", "MPI_Iscatter", GTG_PINK);
    addEntityValue("STV_MPI_Iallgather", "ST_Thread", "MPI_Iallgather", GTG_PINK);
    addEntityValue("STV_MPI_Ialltoall", "ST_Thread", "MPI_Ialltoall", GTG_PINK);
    addEntityValue("STV_MPI_Ireduce", "ST_Thread", "MPI_Ireduce", GTG_PINK);
    addEntityValue("STV_MPI_Iallreduce", "ST_Thread", "MPI_Iallreduce", GTG_PINK);
    addEntityValue("STV_MPI_Ireduce_scatter", "ST_Thread", "MPI_Ireduce_scatter", GTG_PINK);
    addEntityValue("STV_MPI_Iscan", "ST_Thread", "MPI_Iscan", GTG_PINK);

    addEntityValue("STV_MPI_Probe", "ST_Thread", "MPI_Probe", GTG_PURPLE);

    addLinkType ("L_MPI_Coll", "MPI collective communication", "CT_Program", "CT_Thread", "CT_Thread");
    addLinkType ("L_MPI_P2P", "MPI point to point communication", "CT_Program", "CT_Thread", "CT_Thread");
    addLinkType ("L_MPI_SPAWN", "MPI SPAWN", "CT_Program", "CT_Thread", "CT_Thread");

    addEventType ("E_MPI_CommSend", "CT_Thread", "MPI Send");
    addEventType ("E_MPI_CommRecv", "CT_Thread", "MPI Recv");

    addEventType ("E_MPI_EndComm", "CT_Thread", "End of an MPI communication");

    addEventType ("E_MPI_Probe_success", "CT_Thread", "MPI_Probe");
    addEventType ("E_MPI_Iprobe_success", "CT_Thread", "MPI_IProbe success");
    addEventType ("E_MPI_Iprobe_failed", "CT_Thread", "MPI_IProbe failed");
  }

  ezt_list_new(&spawn_list);

  init_mpi_pers_messages();
  init_mpi_p2p_messages();
  init_mpi_coll_messages();
  init_mpi_stats();
  return 0;
}

/* return 1 if the event was handled */
int
handle_mpi_events(eztrace_event_t *ev) {
  switch (LITL_READ_GET_CODE(ev)) {
      case EZTRACE_MPI_INIT:
      handle_mpi_init ();
      break;
      case EZTRACE_MPI_DELETE_COMM:
      handle_mpi_delete_comm ();
      break;
      case EZTRACE_MPI_NEW_COMM:
      handle_mpi_new_comm ();
      break;
      case EZTRACE_MPI_START_SEND:
      nb_mpi_calls[MPI_SEND_ID]++;
      handle_mpi_start_send ();
      break;
      case EZTRACE_MPI_STOP_SEND:
      handle_mpi_stop_send ();
      break;
      case EZTRACE_MPI_START_RECV:
      nb_mpi_calls[MPI_RECV_ID]++;
      handle_mpi_start_recv ();
      break;
      case EZTRACE_MPI_STOP_RECV:
      handle_mpi_stop_recv ();
      break;

      case EZTRACE_MPI_CANCEL:
      handle_mpi_cancel ();
      break;

      case EZTRACE_MPI_START_SENDRECV:
      nb_mpi_calls[MPI_SENDRECV_ID]++;
      handle_mpi_start_sendrecv ();
      break;
      case EZTRACE_MPI_STOP_SENDRECV:
      handle_mpi_stop_sendrecv ();
      break;
      case EZTRACE_MPI_START_SENDRECV_REPLACE:
      nb_mpi_calls[MPI_SENDRECV_REPLACE_ID]++;
      handle_mpi_start_sendrecv_replace ();
      break;
      case EZTRACE_MPI_STOP_SENDRECV_REPLACE:
      handle_mpi_stop_sendrecv_replace ();
      break;

      case EZTRACE_MPI_START_BSEND:
      nb_mpi_calls[MPI_BSEND_ID]++;
      handle_mpi_start_bsend ();
      break;
      case EZTRACE_MPI_STOP_BSEND:
      handle_mpi_stop_bsend ();
      break;

      case EZTRACE_MPI_START_SSEND:
      nb_mpi_calls[MPI_SSEND_ID]++;
      handle_mpi_start_ssend ();
      break;
      case EZTRACE_MPI_STOP_SSEND:
      handle_mpi_stop_ssend ();
      break;

      case EZTRACE_MPI_START_RSEND:
      nb_mpi_calls[MPI_RSEND_ID]++;
      handle_mpi_start_rsend ();
      break;
      case EZTRACE_MPI_STOP_RSEND:
      handle_mpi_stop_rsend ();
      break;

      case EZTRACE_MPI_ISEND:
      nb_mpi_calls[MPI_ISEND_ID]++;
      handle_mpi_isend ();
      break;
      case EZTRACE_MPI_IBSEND:
      nb_mpi_calls[MPI_IBSEND_ID]++;
      handle_mpi_ibsend ();
      break;
      case EZTRACE_MPI_ISSEND:
      nb_mpi_calls[MPI_ISSEND_ID]++;
      handle_mpi_issend ();
      break;
      case EZTRACE_MPI_IRSEND:
      nb_mpi_calls[MPI_IRSEND_ID]++;
      handle_mpi_irsend ();
      break;
      case EZTRACE_MPI_IRECV:
      nb_mpi_calls[MPI_IRECV_ID]++;
      handle_mpi_irecv ();
      break;
      case EZTRACE_MPI_STOP_IRECV:
      handle_mpi_stop_irecv ();
      break;

      case EZTRACE_MPI_STOP_ISEND:
      handle_mpi_stop_isend ();
      break;
      case EZTRACE_MPI_STOP_IBSEND:
      handle_mpi_stop_ibsend ();
      break;
      case EZTRACE_MPI_STOP_ISSEND:
      handle_mpi_stop_issend ();
      break;
      case EZTRACE_MPI_STOP_IRSEND:
      handle_mpi_stop_irsend ();
      break;

      case EZTRACE_MPI_START_PUT:
      nb_mpi_calls[MPI_PUT_ID]++;
      handle_mpi_start_put();
      break;
      case EZTRACE_MPI_STOP_PUT:
      handle_mpi_stop_put();
      break;
      case EZTRACE_MPI_START_GET:
      nb_mpi_calls[MPI_GET_ID]++;
      handle_mpi_start_get();
      break;
      case EZTRACE_MPI_STOP_GET:
      handle_mpi_stop_get();
      break;

      case EZTRACE_MPI_START_WAIT:
      nb_mpi_calls[MPI_WAIT_ID]++;
      handle_mpi_start_wait();
      break;
      case EZTRACE_MPI_STOP_WAIT:
      handle_mpi_stop_wait();
      break;
      case EZTRACE_MPI_STOP_WAIT_FAILED:
      handle_mpi_stop_wait_failed();
      break;
      case EZTRACE_MPI_START_WAITANY:
      nb_mpi_calls[MPI_WAITANY_ID]++;
      handle_mpi_start_waitany();
      break;
      case EZTRACE_MPI_STOP_WAITANY:
      handle_mpi_stop_waitany();
      break;
      case EZTRACE_MPI_START_WAITALL:
      nb_mpi_calls[MPI_WAITALL_ID]++;
      handle_mpi_start_waitall();
      break;
      case EZTRACE_MPI_STOP_WAITALL:
      handle_mpi_stop_waitall(NULL);
      break;
      case EZTRACE_MPI_TEST_SUCCESS:
      nb_mpi_calls[MPI_TEST_ID]++;
      handle_mpi_test_success();
      break;

      case EZTRACE_MPI_START_PROBE:
      nb_mpi_calls[MPI_PROBE_ID]++;
      handle_mpi_start_probe();
      break;
      case EZTRACE_MPI_STOP_PROBE:
      handle_mpi_stop_probe();
      break;
      case EZTRACE_MPI_IPROBE_SUCCESS:
      nb_mpi_calls[MPI_IPROBE_ID]++;
      handle_mpi_iprobe_success();
      break;
      case EZTRACE_MPI_IPROBE_FAILED:
      handle_mpi_iprobe_failed();
      break;

      case EZTRACE_MPI_START_BCast:
      nb_mpi_calls[MPI_BCAST_ID]++;
      handle_mpi_start_BCast();
      break;
      case EZTRACE_MPI_START_Gather:
      nb_mpi_calls[MPI_GATHER_ID]++;
      handle_mpi_start_Gather();
      break;
      case EZTRACE_MPI_START_Gatherv:
      nb_mpi_calls[MPI_GATHERV_ID]++;
      handle_mpi_start_Gatherv();
      break;
      case EZTRACE_MPI_START_Scatter:
      nb_mpi_calls[MPI_SCATTER_ID]++;
      handle_mpi_start_Scatter();
      break;
      case EZTRACE_MPI_START_Scatterv:
      nb_mpi_calls[MPI_SCATTERV_ID]++;
      handle_mpi_start_Scatterv();
      break;
      case EZTRACE_MPI_START_Allgather:
      nb_mpi_calls[MPI_ALLGATHER_ID]++;
      handle_mpi_start_Allgather();
      break;
      case EZTRACE_MPI_START_Allgatherv:
      nb_mpi_calls[MPI_ALLGATHERV_ID]++;
      handle_mpi_start_Allgatherv();
      break;
      case EZTRACE_MPI_START_Alltoall:
      nb_mpi_calls[MPI_ALLTOALL_ID]++;
      handle_mpi_start_Alltoall();
      break;
      case EZTRACE_MPI_START_Alltoallv:
      nb_mpi_calls[MPI_ALLTOALLV_ID]++;
      handle_mpi_start_Alltoallv();
      break;
      case EZTRACE_MPI_START_Reduce:
      nb_mpi_calls[MPI_REDUCE_ID]++;
      handle_mpi_start_Reduce();
      break;
      case EZTRACE_MPI_START_Allreduce:
      nb_mpi_calls[MPI_ALLREDUCE_ID]++;
      handle_mpi_start_Allreduce();
      break;
      case EZTRACE_MPI_START_Reduce_scatter:
      nb_mpi_calls[MPI_REDUCE_SCATTER_ID]++;
      handle_mpi_start_Reduce_scatter();
      break;
      case EZTRACE_MPI_START_Scan:
      nb_mpi_calls[MPI_SCAN_ID]++;
      handle_mpi_start_Scan();
      break;
      case EZTRACE_MPI_START_BARRIER:
      nb_mpi_calls[MPI_BARRIER_ID]++;
      handle_mpi_start_barrier();
      break;

      case EZTRACE_MPI_STOP_BCast : handle_mpi_stop_BCast(); break;
      case EZTRACE_MPI_STOP_Gather : handle_mpi_stop_Gather(); break;
      case EZTRACE_MPI_STOP_Gatherv : handle_mpi_stop_Gatherv(); break;
      case EZTRACE_MPI_STOP_Scatter : handle_mpi_stop_Scatter(); break;
      case EZTRACE_MPI_STOP_Scatterv : handle_mpi_stop_Scatterv(); break;
      case EZTRACE_MPI_STOP_Allgather : handle_mpi_stop_Allgather(); break;
      case EZTRACE_MPI_STOP_Allgatherv : handle_mpi_stop_Allgatherv(); break;
      case EZTRACE_MPI_STOP_Alltoall : handle_mpi_stop_Alltoall(); break;
      case EZTRACE_MPI_STOP_Alltoallv : handle_mpi_stop_Alltoallv(); break;
      case EZTRACE_MPI_STOP_Reduce : handle_mpi_stop_Reduce(); break;
      case EZTRACE_MPI_STOP_Allreduce : handle_mpi_stop_Allreduce(); break;
      case EZTRACE_MPI_STOP_Reduce_scatter : handle_mpi_stop_Reduce_scatter(); break;
      case EZTRACE_MPI_STOP_Scan : handle_mpi_stop_Scan(); break;
      case EZTRACE_MPI_STOP_BARRIER : handle_mpi_stop_barrier(); break;

      case EZTRACE_MPI_IBCAST:
      nb_mpi_calls[MPI_IBCAST_ID]++;
      handle_mpi_Ibcast();
      break;
      case EZTRACE_MPI_IGATHER:
      nb_mpi_calls[MPI_IGATHER_ID]++;
      handle_mpi_Igather();
      break;
      case EZTRACE_MPI_IGATHERV:
      nb_mpi_calls[MPI_IGATHERV_ID]++;
      handle_mpi_Igatherv();
      break;
      case EZTRACE_MPI_ISCATTER:
      nb_mpi_calls[MPI_ISCATTER_ID]++;
      handle_mpi_Iscatter();
      break;
      case EZTRACE_MPI_ISCATTERV:
      nb_mpi_calls[MPI_ISCATTERV_ID]++;
      handle_mpi_Iscatterv();
      break;
      case EZTRACE_MPI_IALLGATHER:
      nb_mpi_calls[MPI_IALLGATHER_ID]++;
      handle_mpi_Iallgather();
      break;
      case EZTRACE_MPI_IALLGATHERV:
      nb_mpi_calls[MPI_IALLGATHERV_ID]++;
      handle_mpi_Iallgatherv();
      break;
      case EZTRACE_MPI_IALLTOALL:
      nb_mpi_calls[MPI_IALLTOALL_ID]++;
      handle_mpi_Ialltoall();
      break;
      case EZTRACE_MPI_IALLTOALLV:
      nb_mpi_calls[MPI_IALLTOALLV_ID]++;
      handle_mpi_Ialltoallv();
      break;
      case EZTRACE_MPI_IREDUCE:
      nb_mpi_calls[MPI_IREDUCE_ID]++;
      handle_mpi_Ireduce();
      break;
      case EZTRACE_MPI_IALLREDUCE:
      nb_mpi_calls[MPI_IALLREDUCE_ID]++;
      handle_mpi_Iallreduce();
      break;
      case EZTRACE_MPI_IREDUCE_SCATTER:
      nb_mpi_calls[MPI_IREDUCE_SCATTER_ID]++;
      handle_mpi_Ireduce_scatter();
      break;
      case EZTRACE_MPI_ISCAN:
      nb_mpi_calls[MPI_ISCAN_ID]++;
      handle_mpi_Iscan();
      break;
      case EZTRACE_MPI_IBARRIER:
      nb_mpi_calls[MPI_IBARRIER_ID]++;
      handle_mpi_Ibarrier();
      break;

      case EZTRACE_MPI_STOP_IBCAST:
      handle_mpi_stop_ibcast();
      break;
      case EZTRACE_MPI_STOP_IGATHER:
      handle_mpi_stop_igather();
      break;
      case EZTRACE_MPI_STOP_IGATHERV:
      handle_mpi_stop_igatherv();
      break;
      case EZTRACE_MPI_STOP_ISCATTER:
      handle_mpi_stop_iscatter();
      break;
      case EZTRACE_MPI_STOP_ISCATTERV:
      handle_mpi_stop_iscatterv();
      break;
      case EZTRACE_MPI_STOP_IALLGATHER:
      handle_mpi_stop_iallgather();
      break;
      case EZTRACE_MPI_STOP_IALLGATHERV:
      handle_mpi_stop_iallgatherv();
      break;
      case EZTRACE_MPI_STOP_IALLTOALL:
      handle_mpi_stop_ialltoall();
      break;
      case EZTRACE_MPI_STOP_IALLTOALLV:
      handle_mpi_stop_ialltoallv();
      break;
      case EZTRACE_MPI_STOP_IREDUCE:
      handle_mpi_stop_ireduce();
      break;
      case EZTRACE_MPI_STOP_IALLREDUCE:
      handle_mpi_stop_iallreduce();
      break;
      case EZTRACE_MPI_STOP_IREDUCE_SCATTER:
      handle_mpi_stop_ireduce_scatter();
      break;
      case EZTRACE_MPI_STOP_ISCAN:
      handle_mpi_stop_iscan();
      break;
      case EZTRACE_MPI_STOP_IBARRIER:
      handle_mpi_stop_ibarrier();
      break;

      case EZTRACE_MPI_SPAWN : handle_mpi_spawn(); break;
      case EZTRACE_MPI_SPAWNED : handle_mpi_spawned(); break;

      case EZTRACE_MPI_SEND_INIT:
      handle_mpi_send_init();
      break;
      case EZTRACE_MPI_BSEND_INIT:
      handle_mpi_bsend_init();
      break;
      case EZTRACE_MPI_RSEND_INIT:
      handle_mpi_rsend_init();
      break;
      case EZTRACE_MPI_SSEND_INIT:
      handle_mpi_ssend_init();
      break;
      case EZTRACE_MPI_RECV_INIT:
      handle_mpi_recv_init();
      break;
      case EZTRACE_MPI_START:
      nb_mpi_calls[MPI_START_ID]++;
      handle_mpi_start();
      break;

      default:
      return 0;
    }
    return 1;
  }

  int handle_mpi_stats(eztrace_event_t *ev) {
    recording_stats = 1;
    return handle_mpi_events(ev);
  }

#define FORMAT_BYTES(nb_bytes)						\
  ((uint64_t)nb_bytes<1024?"B":						\
   ((uint64_t)nb_bytes<1024*1024?"KB":					\
    ((uint64_t)nb_bytes<1024*1024*1024?"MB":				\
     ((uint64_t)nb_bytes<(uint64_t)1024*1024*1024*1024?"GB":		\
      ((uint64_t)nb_bytes<(uint64_t)1024*1024*1024*1024*1024?"TB":	\
       "PB")))))

  float VALUE_BYTES(uint64_t nb_bytes) {
    int i;
    uint64_t div=1;
    for(i=0; i<6; i++) {
      if((nb_bytes/div)<1024)
      return (nb_bytes/(double)div);
      div*=1024;
    }
    return (float)nb_bytes;
  }

  void print_mpi_stats() {
    printf ( "\nMPI:\n");
    printf ( "---\n");

    /* todo:
     * add:
     *  - communication pattern (matrix)
     *  - min/max/average message length
     *  - min/max/average communication duration
     */

    int i;
#if 0
    /* Print the of mpi_send, isend, issend, bcast, gather, scatter etc. */
    for(i=0; i<MPI_ID_SIZE; i++) {
      /* don't print anything if the function was not called */
      if(nb_mpi_calls[i]) {
	printf("\t");

	switch(i) {
	  case MPI_SEND_ID : printf("MPI_SEND             :"); break;
	  case MPI_RECV_ID : printf("MPI_RECV             :"); break;
	  case MPI_BSEND_ID : printf("MPI_BSEND            :"); break;
	  case MPI_SSEND_ID : printf("MPI_SSEND            :"); break;
	  case MPI_RSEND_ID : printf("MPI_RSEND            :"); break;
	  case MPI_ISEND_ID : printf("MPI_ISEND            :"); break;
	  case MPI_IBSEND_ID : printf("MPI_IBSEND           :"); break;
	  case MPI_ISSEND_ID : printf("MPI_ISSEND           :"); break;
	  case MPI_IRSEND_ID : printf("MPI_IRSEND           :"); break;
	  case MPI_IRECV_ID : printf("MPI_IRECV            :"); break;
	  case MPI_SENDRECV_ID : printf("MPI_SENDRECV         :"); break;
	  case MPI_SENDRECV_REPLACE_ID : printf("MPI_SENDRECV_REPLACE :"); break;
	  case MPI_START_ID : printf("MPI_START            :"); break;
	  case MPI_STARTALL_ID : printf("MPI_STARTALL         :"); break;
	  case MPI_WAIT_ID : printf("MPI_WAIT             :"); break;
	  case MPI_TEST_ID : printf("MPI_TEST             :"); break;
	  case MPI_WAITANY_ID : printf("MPI_WAITANY          :"); break;
	  case MPI_TESTANY_ID : printf("MPI_TESTANY          :"); break;
	  case MPI_WAITALL_ID : printf("MPI_WAITALL          :"); break;
	  case MPI_TESTALL_ID : printf("MPI_TESTALL          :"); break;
	  case MPI_WAITSOME_ID : printf("MPI_WAITSOME         :"); break;
	  case MPI_TESTSOME_ID : printf("MPI_TESTSOME         :"); break;
	  case MPI_PROBE_ID : printf("MPI_PROBE            :"); break;
	  case MPI_IPROBE_ID : printf("MPI_IPROBE           :"); break;
	  case MPI_BARRIER_ID : printf("MPI_BARRIER          :"); break;
	  case MPI_BCAST_ID : printf("MPI_BCAST            :"); break;
	  case MPI_GATHER_ID : printf("MPI_GATHER           :"); break;
	  case MPI_GATHERV_ID : printf("MPI_GATHERV          :"); break;
	  case MPI_SCATTER_ID : printf("MPI_SCATTER          :"); break;
	  case MPI_SCATTERV_ID : printf("MPI_SCATTERV         :"); break;
	  case MPI_ALLGATHER_ID : printf("MPI_ALLGATHER        :"); break;
	  case MPI_ALLGATHERV_ID : printf("MPI_ALLGATHERV       :"); break;
	  case MPI_ALLTOALL_ID : printf("MPI_ALLTOALL         :"); break;
	  case MPI_ALLTOALLV_ID : printf("MPI_ALLTOALLV        :"); break;
	  case MPI_REDUCE_ID : printf("MPI_REDUCE           :"); break;
	  case MPI_ALLREDUCE_ID : printf("MPI_ALLREDUCE        :"); break;
	  case MPI_REDUCE_SCATTER_ID : printf("MPI_REDUCE_SCATTER   :"); break;
	  case MPI_SCAN_ID : printf("MPI_SCAN             :"); break;
	  case MPI_IBARRIER_ID : printf("MPI_IBARRIER          :"); break;
	  case MPI_IBCAST_ID : printf("MPI_IBCAST            :"); break;
	  case MPI_IGATHER_ID : printf("MPI_IGATHER           :"); break;
	  case MPI_IGATHERV_ID : printf("MPI_IGATHERV          :"); break;
	  case MPI_ISCATTER_ID : printf("MPI_ISCATTER          :"); break;
	  case MPI_ISCATTERV_ID : printf("MPI_ISCATTERV         :"); break;
	  case MPI_IALLGATHER_ID : printf("MPI_IALLGATHER        :"); break;
	  case MPI_IALLGATHERV_ID : printf("MPI_IALLGATHERV       :"); break;
	  case MPI_IALLTOALL_ID : printf("MPI_IALLTOALL         :"); break;
	  case MPI_IALLTOALLV_ID : printf("MPI_IALLTOALLV        :"); break;
	  case MPI_IREDUCE_ID : printf("MPI_IREDUCE           :"); break;
	  case MPI_IALLREDUCE_ID : printf("MPI_IALLREDUCE        :"); break;
	  case MPI_IREDUCE_SCATTER_ID : printf("MPI_IREDUCE_SCATTER   :"); break;
	  case MPI_ISCAN_ID : printf("MPI_ISCAN             :"); break;
	  case MPI_GET_ID : printf("MPI_GET              :"); break;
	  case MPI_PUT_ID : printf("MPI_PUT              :"); break;
	}
	printf("%d calls\n", nb_mpi_calls[i]);
      }
    }
#else
    /* print per-thread statistics */
    int rank;
    unsigned thread_id;
    for (rank = 0; rank < NB_TRACES; rank++) {
      struct eztrace_container_t *process_cont = GET_PROCESS_CONTAINER(rank);
      for(thread_id = 0; thread_id<process_cont->nb_children; thread_id++) {
	struct eztrace_container_t *p_thread = process_cont->children[thread_id];
	struct thread_info_t* thread_info = (struct thread_info_t*) (p_thread->container_info);
	INIT_MPI_THREAD_INFO(thread_info, mpi_info);

	int should_print = 0;

	double mpi_duration = 0;
	for(i=0; i<MPI_ID_SIZE; i++) {
	  if(mpi_info->nb_mpi_calls[i]) {
	    should_print = 1 ;
	    mpi_duration += mpi_info->total_time_mpi_calls[i];
	  }
	}

	if(!should_print)
	  continue;

	double thread_duration = p_thread->end_timestamp - p_thread->start_timestamp;
	printf("Thread %s -- total duration: %lf ms, time spent in mpi: %lf ms (%lf %%)\n",
	       p_thread->name, thread_duration, mpi_duration, 100*mpi_duration/thread_duration);

	/* Print the of mpi_send, isend, issend, bcast, gather, scatter etc. */
	for(i=0; i<MPI_ID_SIZE; i++) {
	  /* don't print anything if the function was not called */
	  if(mpi_info->nb_mpi_calls[i]) {
	    printf("\t");

	    switch(i) {
	    case MPI_SEND_ID : printf("MPI_SEND             :"); break;
	    case MPI_RECV_ID : printf("MPI_RECV             :"); break;
	    case MPI_BSEND_ID : printf("MPI_BSEND            :"); break;
	    case MPI_SSEND_ID : printf("MPI_SSEND            :"); break;
	    case MPI_RSEND_ID : printf("MPI_RSEND            :"); break;
	    case MPI_ISEND_ID : printf("MPI_ISEND            :"); break;
	    case MPI_IBSEND_ID : printf("MPI_IBSEND           :"); break;
	    case MPI_ISSEND_ID : printf("MPI_ISSEND           :"); break;
	    case MPI_IRSEND_ID : printf("MPI_IRSEND           :"); break;
	    case MPI_IRECV_ID : printf("MPI_IRECV            :"); break;
	    case MPI_SENDRECV_ID : printf("MPI_SENDRECV         :"); break;
	    case MPI_SENDRECV_REPLACE_ID : printf("MPI_SENDRECV_REPLACE :"); break;
	    case MPI_START_ID : printf("MPI_START            :"); break;
	    case MPI_STARTALL_ID : printf("MPI_STARTALL         :"); break;
	    case MPI_WAIT_ID : printf("MPI_WAIT             :"); break;
	    case MPI_TEST_ID : printf("MPI_TEST             :"); break;
	    case MPI_WAITANY_ID : printf("MPI_WAITANY          :"); break;
	    case MPI_TESTANY_ID : printf("MPI_TESTANY          :"); break;
	    case MPI_WAITALL_ID : printf("MPI_WAITALL          :"); break;
	    case MPI_TESTALL_ID : printf("MPI_TESTALL          :"); break;
	    case MPI_WAITSOME_ID : printf("MPI_WAITSOME         :"); break;
	    case MPI_TESTSOME_ID : printf("MPI_TESTSOME         :"); break;
	    case MPI_PROBE_ID : printf("MPI_PROBE            :"); break;
	    case MPI_IPROBE_ID : printf("MPI_IPROBE           :"); break;
	    case MPI_BARRIER_ID : printf("MPI_BARRIER          :"); break;
	    case MPI_BCAST_ID : printf("MPI_BCAST            :"); break;
	    case MPI_GATHER_ID : printf("MPI_GATHER           :"); break;
	    case MPI_GATHERV_ID : printf("MPI_GATHERV          :"); break;
	    case MPI_SCATTER_ID : printf("MPI_SCATTER          :"); break;
	    case MPI_SCATTERV_ID : printf("MPI_SCATTERV         :"); break;
	    case MPI_ALLGATHER_ID : printf("MPI_ALLGATHER        :"); break;
	    case MPI_ALLGATHERV_ID : printf("MPI_ALLGATHERV       :"); break;
	    case MPI_ALLTOALL_ID : printf("MPI_ALLTOALL         :"); break;
	    case MPI_ALLTOALLV_ID : printf("MPI_ALLTOALLV        :"); break;
	    case MPI_REDUCE_ID : printf("MPI_REDUCE           :"); break;
	    case MPI_ALLREDUCE_ID : printf("MPI_ALLREDUCE        :"); break;
	    case MPI_REDUCE_SCATTER_ID : printf("MPI_REDUCE_SCATTER   :"); break;
	    case MPI_SCAN_ID : printf("MPI_SCAN             :"); break;
	    case MPI_IBARRIER_ID : printf("MPI_IBARRIER          :"); break;
	    case MPI_IBCAST_ID : printf("MPI_IBCAST            :"); break;
	    case MPI_IGATHER_ID : printf("MPI_IGATHER           :"); break;
	    case MPI_IGATHERV_ID : printf("MPI_IGATHERV          :"); break;
	    case MPI_ISCATTER_ID : printf("MPI_ISCATTER          :"); break;
	    case MPI_ISCATTERV_ID : printf("MPI_ISCATTERV         :"); break;
	    case MPI_IALLGATHER_ID : printf("MPI_IALLGATHER        :"); break;
	    case MPI_IALLGATHERV_ID : printf("MPI_IALLGATHERV       :"); break;
	    case MPI_IALLTOALL_ID : printf("MPI_IALLTOALL         :"); break;
	    case MPI_IALLTOALLV_ID : printf("MPI_IALLTOALLV        :"); break;
	    case MPI_IREDUCE_ID : printf("MPI_IREDUCE           :"); break;
	    case MPI_IALLREDUCE_ID : printf("MPI_IALLREDUCE        :"); break;
	    case MPI_IREDUCE_SCATTER_ID : printf("MPI_IREDUCE_SCATTER   :"); break;
	    case MPI_ISCAN_ID : printf("MPI_ISCAN             :"); break;
	    case MPI_GET_ID : printf("MPI_GET              :"); break;
	    case MPI_PUT_ID : printf("MPI_PUT              :"); break;
	    }

	    printf("%d calls -- %lf ms (%lf %% of the thread runtime)\n", mpi_info->nb_mpi_calls[i],
		   mpi_info->total_time_mpi_calls[i], 100*mpi_info->total_time_mpi_calls[i]/thread_duration);
	  }
	}

      }
    }
#endif

    printf("\n");

    print_mpi_msg_stats();
  }


#define QUOTE(name) #name
#define STR(macro) QUOTE(macro)

#ifndef MPI_MODULE_NAME
  #define MPI_MODULE_NAME mpi
#endif

#define MPI_MODULE_NAME_STR STR(MPI_MODULE_NAME)


  struct eztrace_convert_module mpi_module;
  void libinit(void) __attribute__ ((constructor));
  void libinit(void)
  {
    mpi_module.api_version = EZTRACE_API_VERSION;
    mpi_module.init = eztrace_convert_mpi_init;
    mpi_module.handle = handle_mpi_events;
    mpi_module.handle_stats = handle_mpi_stats;
    mpi_module.print_stats = print_mpi_stats;
    mpi_module.module_prefix = EZTRACE_MPI_EVENTS_ID;
    int res __attribute__ ((__unused__));
    res = asprintf(&mpi_module.name, MPI_MODULE_NAME_STR);
    res = asprintf(&mpi_module.description, "Module for MPI functions");
    mpi_module.token.data = &mpi_module;
    eztrace_convert_register_module(&mpi_module);

    int i;
    for(i=0; i<MPI_ID_SIZE; i++)
    nb_mpi_calls[i] = 0;
  }

  void libfinalize(void) __attribute__ ((destructor));
  void libfinalize(void)
  {
  }

#endif	/* USE_MPI */
