我正在使用process_vm_readv在MPI中将数据从一个进程获取到另一个进程。我发现在为process_vm_readv提供一定数量的iovec(在本例中为1024)之后,该程序将开始垃圾。我不确定发生了什么,内核是否耗尽了内存?或我的代码有问题。还是process_vm_readv对iovec有上限?我为iovec自生成了一个矢量模式(每16个字节中有8个字节)。程序将一直运行,直到两个线程上的此模式都填充了1GB。分别为sbuf和rbuf分配了1GB的内存。该程序位于24GB以上的计算机上。
void do_test( int slen, int rlen, int scount, int rcount, void *sbuf, void *rbuf ){
int rank, err;
double timers[REP];
MPI_Win win;
pid_t pid;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if( rank == 0 ){
MPI_Win_create( NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win );
int send_iovcnt;
struct iovec *send_iov;
struct iovec *iov = malloc( sizeof(struct iovec) * scount );
for( int p = 0; p < scount; p++ ){
iov[p].iov_base = (char*)rbuf + p * 16;
iov[p].iov_len = 8;
}
MPI_Recv( &pid, sizeof(pid_t), MPI_BYTE, 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
MPI_Recv( &send_iovcnt, 1, MPI_INT, 1, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
send_iov = malloc( sizeof(struct iovec) * send_iovcnt );
MPI_Recv( send_iov, sizeof(struct iovec) * send_iovcnt, MPI_BYTE, 1, 2, MPI_COMM_WORLD, MPI_STATUS_IGNORE );
for( int i = 0; i < REP; i++ ){
cache_flush();
timers[i] = MPI_Wtime();
MPI_Win_fence( 0, win );
process_vm_readv( pid, iov, send_iovcnt, send_iov, send_iovcnt, 0 );
MPI_Win_fence( 0, win );
cache_flush();
timers[i] = MPI_Wtime() - timers[i];
}
free(send_iov);
free(iov);
print_result( 8 * scount, REP, timers );
} else if( rank == 1 ){
MPI_Win_create( sbuf, slen, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win );
struct iovec *iov = malloc( sizeof(struct iovec) * rcount );
for( int p = 0; p < rcount; p++ ){
iov[p].iov_base = (char*)sbuf + p * 16;
iov[p].iov_base = 8;
}
pid = getpid();
MPI_Send( &pid, sizeof(pid_t), MPI_BYTE, 0, 0, MPI_COMM_WORLD );
MPI_Send( &rcount, 1, MPI_INT, 0, 1, MPI_COMM_WORLD );
MPI_Send( iov, rcount * sizeof(struct iovec), MPI_BYTE, 0, 2, MPI_COMM_WORLD );
for( int i = 0; i < REP; i++ ){
cache_flush();
MPI_Win_fence( 0, win );
MPI_Win_fence( 0, win );
}
free(iov);
}
在man page of process_vm_readv(2)
中找到以下文本:
process_vm_readv(2)
和liovcnt
参数中指定的值必须小于或等于riovcnt
(在IOV_MAX
中定义,或可通过调用<limits.h>
进行访问。)
在我的Linux系统上,sysconf(_SC_IOV_MAX)
的值(最终在IOV_MAX
中定义)为1024。