MPI_Session fail when running concurently with a MPI program

[TApplencourt]

So the program spawn a programing useing MPI_Session in the background and then a programing usingMPI_Init .

The behavior is non-deterministic (hang, Fatal error in internal_Comm_create_from_group: Message truncated,, etc..) . I use 2 nodes in the following run.

The work-around seem to be to set a different PMIX_NAMESPACE for the session.

Reproducer bellow:

applenco@x1922c5s0b0n0:~/project/p26.01/MPI_session_bug> tail -n +1 session.c world.c full_reproducer.sh
==> session.c <==
/*
 * session.c -- an MPI "Sessions model" program.
 *
 * It creates a Session, derives a communicator from the "mpi://WORLD"
 * process set, then idles (pause()) until it receives a signal. 
 * 
 * With -DNS_FIX the program appends a suffix to $PMIX_NAMESPACE before any
 * MPI call, giving its MPI instance an identity distinct from the sibling
 * world process. 
 */
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#define CK(call)                                                               \
    do {                                                                       \
        int _r = (call);                                                       \
        if (_r != MPI_SUCCESS) {                                               \
            char _es[MPI_MAX_ERROR_STRING];                                    \
            int _el;                                                           \
            MPI_Error_string(_r, _es, &_el);                                   \
            fprintf(stderr, "session: %s -> %d: %s\n", #call, _r, _es);        \
            return 1;                                                          \
        }                                                                      \
    } while (0)

int main(void) {
#ifdef NS_FIX
    /* Work-around: give this MPI instance its own PMIx namespace, distinct
     * from the sibling world process, so the two do not share a bootstrap
     * identity. Must happen before any MPI call. */
    const char *ns = getenv("PMIX_NAMESPACE");
    if (ns) {
        char new_ns[512];
        snprintf(new_ns, sizeof(new_ns), "%s_session", ns);
        setenv("PMIX_NAMESPACE", new_ns, 1);
    }
#endif

    MPI_Session sh = MPI_SESSION_NULL;
    MPI_Group wgroup = MPI_GROUP_NULL;
    MPI_Comm comm = MPI_COMM_NULL;
    MPI_Info sinfo = MPI_INFO_NULL;

    MPI_Info_create(&sinfo);
    MPI_Info_set(sinfo, "thread_level", "MPI_THREAD_SINGLE");

    CK(MPI_Session_init(sinfo, MPI_ERRORS_RETURN, &sh));
    CK(MPI_Group_from_session_pset(sh, "mpi://WORLD", &wgroup));
    CK(MPI_Comm_create_from_group(wgroup, "session_comm", MPI_INFO_NULL,
                                  MPI_ERRORS_RETURN, &comm));

    int rank = -1, size = -1;
    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &size);
    fprintf(stderr, "session: comm created (rank %d / %d), idling\n", rank, size);

    /* Idle at zero CPU until the launcher signals us once the world process
     * has finished. No timeout, no polling. */
    pause();

    MPI_Comm_free(&comm);
    MPI_Group_free(&wgroup);
    MPI_Info_free(&sinfo);
    MPI_Session_finalize(&sh);
    return 0;
}

==> world.c <==
/*
 * world.c -- a classic MPI "world model" program.
 *
 * Plain MPI_Init / MPI_COMM_WORLD / MPI_Finalize, exactly like a normal
 * application. On its own it always works.
 */
#include <mpi.h>
#include <stdio.h>

int main(void) {
    MPI_Init(NULL, NULL);

    int rank, size;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    printf("world: rank %d / %d\n", rank, size);
    fflush(stdout);

    MPI_Finalize();
    return 0;
}

==> full_reproducer.sh <==
#!/bin/bash
#
# full_reproducer.sh -- MPICH deadlock/abort when an MPI Sessions process runs
# concurrently with a classic-world MPI process that shares its PMIx identity.
#
# Per rank, one launcher forks a `session` helper (background) and a `world`
# app (foreground), so the two MPI instances inherit the same PMIx identity
# (PMIX_NAMESPACE / PMIX_RANK). It runs the scenario with the plain `session`
# (reproduces the bug) and with `session_fix` (-DNS_FIX, distinct
# PMIX_NAMESPACE -> works), and prints the exit code of each. A per-run
# `timeout` turns a hang into a non-zero code.
#
# Usage:  ./full_reproducer.sh [NRANKS] [PPN]      (default 128 64, needs >=2 nodes)

set -u

# We will call ourself as some pont. Guard it with --worked.
# This will run first session than hello_world
# We send kill to the session to unpause and exiting nicely
if [ "${1:-}" = "--worker" ]; then
    here="$2"; session_bin="$3"
    "$here/$session_bin" &
    helper=$!
    "$here/world"
    rc=$?
    kill "$helper" 2>/dev/null
    exit $rc
fi

NRANKS="${1:-2}"
PPN="${2:-4}"
TIMEOUT="${TIMEOUT:-20}"
HERE="$(cd "$(dirname "$0")" && pwd)"
SELF="$HERE/$(basename "$0")"

mpicc -O2 -o "$HERE/world"       "$HERE/world.c"            || exit 2
mpicc -O2 -o "$HERE/session"     "$HERE/session.c"          || exit 2
mpicc -O2 -o "$HERE/session_fix" -DNS_FIX "$HERE/session.c" || exit 2

run_case() {
    local session_bin="$1"
    # Sweep stragglers from a previous hung run (one pkill per node).
    mpirun -n "$NRANKS" -- pkill -9 -x 'world|session|session_fix' >/dev/null 2>&1

    # Run ourself
    echo "$session_bin: Running"
    timeout "$TIMEOUT" mpirun -n "$NRANKS" --ppn "$PPN" -- \
        "$SELF" --worker "$HERE" "$session_bin"
    echo "$session_bin: exit code $?"
}

run_case session
run_case session_fix
applenco@x1922c5s0b0n0:~/project/p26.01/MPI_session_bug> bash full_reproducer.sh
session: Running
x1922c5s0b0n0-hsn0.hsn.cm.sunspot.alcf.anl.gov: rank 0 died from signal 15
session: exit code 124
session_fix: Running
session: comm created (rank 0 / 1), idling
session: comm created (rank 0 / 1), idling
world: rank 0 / 2
world: rank 1 / 2
session_fix: exit code 0
applenco@x1922c5s0b0n0:~/project/p26.01/MPI_session_bug> bash full_reproducer.sh
session: Running
Abort(1014587022) on node 0: Fatal error in internal_Comm_create_from_group: Message truncated, error stack:
internal_Comm_create_from_group(109)..: MPI_Comm_create_from_group(group=0x88000002, stringtag=session_comm, MPI_INFO_NULL, errh=0x54000001, newcomm=0x7ffea9ee6b54) failed
MPIR_Comm_create_from_group_impl(549).:
MPIR_init_comm_world(37)..............:
MPIR_Comm_commit(822).................:
MPID_Comm_commit_pre_hook(160)........:
MPIDI_SHM_mpi_comm_commit_pre_hook(18):
MPIDI_SHM_comm_bootstrap(48)..........:
MPIDI_IPC_comm_bootstrap(64)..........:
MPIDI_GPU_comm_bootstrap(72)..........:
MPIDI_FD_comm_bootstrap(61)...........:
MPIDI_IPC_mpi_fd_init(231)............:
MPIR_Allgather_intra_brucks(70).......:
MPIC_Sendrecv(280)....................:
MPIC_Wait(99).........................:
MPIR_Wait(745)........................:
MPIDIG_copy_from_unexp_req(55)........: Message from rank 1 and tag 7 truncated; 4 bytes received but buffer size is 8
x1922c5s0b0n0-hsn0.hsn.cm.sunspot.alcf.anl.gov: rank 0 died from signal 15
session: exit code 124
session_fix: Running
session: comm created (rank 0 / 1), idling
session: comm created (rank 0 / 1), idling
world: rank 0 / 2
world: rank 1 / 2
session_fix: exit code 0