OpenMPI Segmentation Fault at runtime using Intel XE 2011 SP1

OpenMPI Segmentation Fault at runtime using Intel XE 2011 SP1

albertofalzone's picture

Hello,
trying to use OpenMPI 1.2.8 and higher version as such as 1.4.1, 1.4.2 (but also with 1.4.3) with Intel Parallel Studio XE 2011 SP1 released in Sep 2011, after successfully build (with the same Intel compiler) and installation, running the simple command mpicc -V I got a "Segmentation fault" error message.
No change also setting the stack size to unlimited value.
Also, all OFED layer has been rebuilt with the same compiler in order have aligned the binaries of libverbs and so on, but the behaviour was the same.
In addiction, shutting down the openibd service, the simple command works, and starting the service again, I got the segmentation fault.
Other MPI flavors does not present this issue, i.e. mvapich , mvapich2.
For what reason I got this error message with OpenMPI ?
Is there an OpenMPI version that is Intel 2011 SP1 (12.1.0.233) compliant ?
Below a dump of strace command hoping that can help to evaluate this issue:

[root@cresco2x015 ~]# icc -V

Intel C Intel 64 Compiler XE for applications running on Intel
64, Version 12.1.0.233 Build 20110811

Copyright (C) 1985-2011 Intel Corporation. All rights reserved.

FOR NON-COMMERCIAL USE ONLY

[root@cresco2x015 ~]# which icc

/opt/intel12/composer_xe_2011_sp1.6.233/bin/intel64/icc

[root@cresco2x015 ~]#

[root@cresco2x015 ~]# which mpicc

/usr/mpi/intel/openmpi-1.4.1/bin/mpicc

[root@cresco2x015 ~]# mpicc -V

Segmentation fault

[root@cresco2x015 ~]#

[root@cresco2x015 ~]# strace mpicc -V

execve("/usr/mpi/intel/openmpi-1.4.1/bin/mpicc", ["mpicc", "-V"], [/* 38
vars */]) = 0

brk(0) = 0x181f5000

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3764e93000

uname({sys="Linux", node="cresco2x015.portici.enea.it", ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3764e94000

access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/tls/x86_64/libopen-pal.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/usr/mpi/intel/openmpi-1.4.1/lib64/tls/x86_64", 0x7fff2199e7b0) =
-1 ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/tls/libopen-pal.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/usr/mpi/intel/openmpi-1.4.1/lib64/tls", 0x7fff2199e7b0) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/x86_64/libopen-pal.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/usr/mpi/intel/openmpi-1.4.1/lib64/x86_64", 0x7fff2199e7b0) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libopen-pal.so.0", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\300\\345\\0\\0\\0\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=1198231, ...}) = 0

mmap(NULL, 2706888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b3764e95000

mprotect(0x2b3764f04000, 2093056, PROT_NONE) = 0

mmap(0x2b3765103000, 16384, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x2b3765103000

mmap(0x2b3765107000, 142792, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3765107000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libdl.so.2", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libdl.so.2", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=8192, ...}) = 0

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/gcc433/lib64/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT
(No such file or directory)

stat("/opt/gcc433/lib64/tls/x86_64", 0x7fff2199e780) = -1 ENOENT (No
such file or directory)

open("/opt/gcc433/lib64/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)

stat("/opt/gcc433/lib64/tls", 0x7fff2199e780) = -1 ENOENT (No such file
or directory)

open("/opt/gcc433/lib64/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No
such file or directory)

stat("/opt/gcc433/lib64/x86_64", 0x7fff2199e780) = -1 ENOENT (No such
file or directory)

open("/opt/gcc433/lib64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file
or directory)

stat("/opt/gcc433/lib64", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/hypre/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No
such file or directory)

stat("/opt/hypre/lib/tls/x86_64", 0x7fff2199e780) = -1 ENOENT (No such
file or directory)

open("/opt/hypre/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)

stat("/opt/hypre/lib/tls", 0x7fff2199e780) = -1 ENOENT (No such file or
directory)

open("/opt/hypre/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)

stat("/opt/hypre/lib/x86_64", 0x7fff2199e780) = -1 ENOENT (No such file
or directory)

open("/opt/hypre/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)

stat("/opt/hypre/lib", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/libsbml/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No
such file or directory)

stat("/opt/libsbml/lib/tls/x86_64", 0x7fff2199e780) = -1 ENOENT (No such
file or directory)

open("/opt/libsbml/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)

stat("/opt/libsbml/lib/tls", 0x7fff2199e780) = -1 ENOENT (No such file
or directory)

open("/opt/libsbml/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No
such file or directory)

stat("/opt/libsbml/lib/x86_64", 0x7fff2199e780) = -1 ENOENT (No such
file or directory)

open("/opt/libsbml/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file
or directory)

stat("/opt/libsbml/lib", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/tls/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/tls/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/tls/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/tls",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/x86_64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/x86_64",
0x7fff2199e780) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libdl.so.2",
O_RDONLY) = -1 ENOENT (No such file or directory)

stat("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64",
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0

open("/etc/ld.so.cache", O_RDONLY) = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=213932, ...}) = 0

mmap(NULL, 213932, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b376512a000

close(3) = 0

open("/lib64/libdl.so.2", O_RDONLY) = 3

read(3, "\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\20\\16
\\2029\\0\\0\\0"..., 832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=23360, ...}) = 0

mmap(0x3982200000, 2109696, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3982200000

mprotect(0x3982202000, 2097152, PROT_NONE) = 0

mmap(0x3982402000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x3982402000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libnsl.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libnsl.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("/opt/hypre/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("/opt/libsbml/lib/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("tls/x86_64/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("tls/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libnsl.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libnsl.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/lib64/libnsl.so.1", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\240@\\200\\2149\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=114352, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b376515f000

mmap(0x398c800000, 2194096, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x398c800000

mprotect(0x398c815000, 2093056, PROT_NONE) = 0

mmap(0x398ca14000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x398ca14000

mmap(0x398ca16000, 6832, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x398ca16000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libutil.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libutil.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libutil.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("/opt/hypre/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("/opt/libsbml/lib/libutil.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("tls/x86_64/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("tls/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libutil.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libutil.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/lib64/libutil.so.1", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\200\\16`\\2219\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=18152, ...}) = 0

mmap(0x3991600000, 2105616, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3991600000

mprotect(0x3991602000, 2093056, PROT_NONE) = 0

mmap(0x3991801000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x3991801000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libm.so.6", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libm.so.6", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libm.so.6", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("/opt/hypre/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/libsbml/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("tls/x86_64/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("tls/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libm.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libm.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/lib64/libm.so.6", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0`>\\340\\2019\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=615136, ...}) = 0

mmap(0x3981e00000, 2629848, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3981e00000

mprotect(0x3981e82000, 2093056, PROT_NONE) = 0

mmap(0x3982081000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x81000) = 0x3982081000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libgcc_s.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libgcc_s.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libgcc_s.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libgcc_s.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libgcc_s.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libgcc_s.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libgcc_s.so.1",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libgcc_s.so.1", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0`'\\0\\0\\0\\0\\0\\0"..., 832)
= 832

fstat(3, {st_mode=S_IFREG|0644, st_size=478623, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3765160000

mmap(NULL, 2188824, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b3765161000

mprotect(0x2b3765177000, 2097152, PROT_NONE) = 0

mmap(0x2b3765377000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x2b3765377000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libpthread.so.0", O_RDONLY) =
-1 ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libpthread.so.0", O_RDONLY) =
-1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("/opt/hypre/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("/opt/libsbml/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such
file or directory)

open("tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libpthread.so.0",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/lib64/libpthread.so.0", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\200W`\\2029\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=145592, ...}) = 0

mmap(0x3982600000, 2204528, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3982600000

mprotect(0x3982616000, 2093056, PROT_NONE) = 0

mmap(0x3982815000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x3982815000

mmap(0x3982817000, 13168, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3982817000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libc.so.6", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libc.so.6", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/../compiler/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/ipp/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mkl/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/tbb/lib/intel64//cc4.1.0_libc2.4_kernel2.6.16.21/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/gcc433/lib64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("/opt/hypre/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/libsbml/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file
or directory)

open("tls/x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("tls/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/debugger/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/mpirt/lib/intel64/libc.so.6",
O_RDONLY) = -1 ENOENT (No such file or directory)

open("/lib64/libc.so.6", O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0p\\332\\241\\2019\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=1713160, ...}) = 0

mmap(0x3981a00000, 3494168, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3981a00000

mprotect(0x3981b4c000, 2097152, PROT_NONE) = 0

mmap(0x3981d4c000, 20480, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14c000) = 0x3981d4c000

mmap(0x3981d51000, 16664, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3981d51000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libimf.so", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libimf.so", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libimf.so",
O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\300\\301\\0\\0\\0\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=3152633, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3765378000

mmap(NULL, 3976400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b3765379000

mprotect(0x2b3765600000, 1048576, PROT_NONE) = 0

mmap(0x2b3765700000, 278528, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x287000) = 0x2b3765700000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libsvml.so", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libsvml.so", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libsvml.so",
O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0\\240u\\1\\0\\0\\0\\0\\0"...,
832) = 832

fstat(3, {st_mode=S_IFREG|0755, st_size=7220500, ...}) = 0

mmap(NULL, 7810736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b3765744000

mprotect(0x2b3765db5000, 1048576, PROT_NONE) = 0

mmap(0x2b3765eb5000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x671000) = 0x2b3765eb5000

close(3) = 0

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libintlc.so.5", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/usr/mpi/intel/openmpi-1.4.1/lib64/libintlc.so.5", O_RDONLY) = -1
ENOENT (No such file or directory)

open("/opt/intel12/composer_xe_2011_sp1.6.233/compiler/lib/intel64/libintlc.so.5",
O_RDONLY) = 3

read(3,
"\\177ELF\\2\\1\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\3\\0>\\0\\1\\0\\0\\0@p\\0\\0\\0\\0\\0\\0"..., 832)
= 832

fstat(3, {st_mode=S_IFREG|0755, st_size=295094, ...}) = 0

mmap(NULL, 1321856, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2b3765eb7000

mprotect(0x2b3765ef7000, 1044480, PROT_NONE) = 0

mmap(0x2b3765ff6000, 12288, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3f000) = 0x2b3765ff6000

mmap(0x2b3765ff9000, 2944, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x2b3765ff9000

close(3) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3765ffa000

mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2b3765ffb000

arch_prctl(ARCH_SET_FS, 0x2b3765ffb620) = 0

mprotect(0x3981d4c000, 16384, PROT_READ) = 0

mprotect(0x3982815000, 4096, PROT_READ) = 0

mprotect(0x3982081000, 4096, PROT_READ) = 0

mprotect(0x3991801000, 4096, PROT_READ) = 0

mprotect(0x398ca14000, 4096, PROT_READ) = 0

mprotect(0x3982402000, 4096, PROT_READ) = 0

mprotect(0x398181b000, 4096, PROT_READ) = 0

munmap(0x2b376512a000, 213932) = 0

set_tid_address(0x2b3765ffb6b0) = 23781

set_robust_list(0x2b3765ffb6c0, 0x18) = 0

futex(0x7fff2199f2dc, FUTEX_WAKE_PRIVATE, 1) = 0

rt_sigaction(SIGRTMIN, {0x3982605370, [], SA_RESTORER|SA_SIGINFO,
0x398260e4c0}, NULL, 8) = 0

rt_sigaction(SIGRT_1, {0x39826052b0, [],
SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x398260e4c0}, NULL, 8) = 0

rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0

getrlimit(RLIMIT_STACK, {rlim_cur=RLIM_INFINITY,
rlim_max=RLIM_INFINITY}) = 0

stat("/sys/class/infiniband", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0

stat("/sys/class/infiniband", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0

--- SIGSEGV (Segmentation fault) @ 0 (0) ---

+++ killed by SIGSEGV +++

[root@cresco2x015 ~]#

8 posts / 0 new
Last post
For more complete information about compiler optimizations, see our Optimization Notice.
hubert-haberstock (Intel)'s picture

I didn't fully investigate your trace, but it sounds like being the same issuerecently found with the Intel Composer XE Update 6 (formerly known as SP1) and OpenMPI.
Please disable optimization for

mvapich2-1.7rc2/src/mpid/ch3/channels/common/src/memory/ptmalloc2/mvapich_malloc.c by adding
#pragma optimize("", off)
on top of the file. Let me know if this helps.
Regards, Hubert

hlrs's picture

Dear Hubert,
well, this bug caused some headache on my side: running any Open MPI binary compiled intel v12.1 on a linux x86-64 machine SEGFAULTED in ptmalloc2's code in opal/mca/memory/linux/malloc.c:

[c++]/* remove from unsorted list */
unsorted_chunks(av)->bk = bck;
bck->fd = unsorted_chunks(av);[/c++]

Your reply came just in time ,-]
Inserting the #pragma optimize("", off) fixed this.

FYI, this has been commited to ompi-trunk and will be moved to Open MPI v1.5 (https://svn.open-mpi.org/trac/ompi/ticket/2883).

hubert-haberstock (Intel)'s picture

Thanks for feedback. This is a bug in the vectorizer of the 12.1 compiler and will be fixed in one of the next releases. Until the bugfix is implemented you need to applythis workaround.
Regards, Hubert.

larry-baker's picture
I started to look at adding a test for the Intel compiler version around the #pragma that disables optimization for OpenMPI and I found the __ICC and__INTEL_COMPILER predefined macros (compiler version no.) are not properly defined: $ icc -E -dD hello.c | grep __INTEL_COMPILER #define __INTEL_COMPILER 9999 #define __INTEL_COMPILER_BUILD_DATE 20110811 $ icc -E -dD hello.c | grep __ICC #define __ICC 9999 $ icc -v icc version 12.1.0 (gcc version 4.1.2 compatibility)
hubert-haberstock (Intel)'s picture

The build date is currently the only suitable macro.This allows to check for the Intel Compiler and for specific compiler versions. Makes sense? Regards, Hubert.

larry-baker's picture

This is a bug, right?

The 12.1 compiler documentation athttp://software.intel.com/sites/products/documentation/hpc/composerxe/en... says __ICC and__INTEL_COMPILER are predefined compiler version no. macros.

For the previous compiler releases I have, they sort of are (you cannot distinguish the last point release from the macros -- also a bug, I think):

Version 11.1
#define __ICC 1110
#define __INTEL_COMPILER 1110
#define __INTEL_COMPILER_BUILD_DATE 20101201

Version 12.0.3
#define __ICC 1200
#define __INTEL_COMPILER 1200
#define __INTEL_COMPILER_BUILD_DATE 20110309

Version 12.0.4
#define __ICC 1200
#define __INTEL_COMPILER 1200
#define __INTEL_COMPILER_BUILD_DATE 20110427

Version 12.0.5
#define __ICC 1200
#define __INTEL_COMPILER 1200
#define __INTEL_COMPILER_BUILD_DATE 20110719

While for the 12.1.0 compiler release, they are nonsense (9999):

icc version 12.1.0 (gcc version 4.1.2 compatibility)
#define __ICC 9999
#define __INTEL_COMPILER 9999
#define __INTEL_COMPILER_BUILD_DATE 20110811

Has this bug already been reported? Either the documentation needs to be fixed, or the macros need to be fixed. They can't both be correct the way they are now.

hubert-haberstock (Intel)'s picture

Yes, it's a bug that will be fixed in the next upcoming compiler release. Regards, Hubert.

Login to leave a comment.