VScode jupyter 内核在使用 f2py 子例程将大数据写入 2D fortran 数组时死机

问题描述 投票:0回答:0

我不能在特定情况下将 fortran-

read
转换为 2D fortran 数组(
ifort
+
f2py
编译 -> 在 Jupyter VScode 中加载并执行该子例程)。

我怀疑

ifort
的编译阶段可能有问题。 有什么解决办法吗?

  1. 我写 Fortran90 代码,有一行
    open
    读取 fortran 未格式化的文件。
!rur/readr.f90
module readr
    use omp_lib
    implicit none

    real(kind=8),    dimension(:,:),   allocatable :: real_table
    integer(kind=4), dimension(:,:),   allocatable :: integer_table
    integer(kind=8), dimension(:,:),   allocatable :: long_table
    integer(kind=1), dimension(:,:),   allocatable :: byte_table
    integer :: nhvar
    real(kind=8) :: aexp

contains
!#####################################################################
    subroutine read_part(repo, iout, cpu_list, mode, verbose, longint, nthread)
!#####################################################################
        implicit none

        integer :: i, j, icpu, idim, ipart
        integer :: ncpu, ndim
        integer :: npart, nstar_int, nsink, ncursor
        integer(kind=8) :: npart_tot, nstar, npart_c

        integer :: part_n, nreal, nint, nbyte, nlong, nchem, nthread
        integer :: pint
        logical :: ok

        character(len=128) :: file_path

        character(len=128),    intent(in) :: repo
        integer,               intent(in) :: iout
        integer, dimension(:), intent(in) :: cpu_list
        character(len=10),     intent(in) :: mode
        logical,               intent(in) :: verbose
        logical,               intent(in) :: longint
        integer(kind=4), dimension(:),     allocatable :: cursors_temp, cursors

        part_n = 30

        file_path = part_filename(repo, iout, cpu_list(1), mode)

        ! Step 1: Verify there is file
        inquire(file=file_path, exist=ok)
        if ( .not. ok ) then
            print *, file_path, ' not found.'
            stop
        endif

        ! Step 2: Count the total number of particles.
        open(unit=part_n, file=file_path, status='old', form='unformatted')
        read(part_n) ncpu
        read(part_n) ndim
        call skip_read(part_n, 2)
        if(longint) then
            read(part_n) nstar
        else
            read(part_n) nstar_int
            nstar = nstar_int
        end if
        call skip_read(part_n, 2)
        read(part_n) nsink
        close(part_n)

        npart_tot = 0
        allocate(cursors(1:SIZE(cpu_list)))
        allocate(cursors_temp(1:SIZE(cpu_list)))
        cursors_temp = 0
        !$OMP PARALLEL DO SHARED(npart_tot, cursors_temp) PRIVATE(i, icpu, npart, part_n) NUM_THREADS(nthread)
        do i = 1, SIZE(cpu_list)
            icpu = cpu_list(i)
            part_n = 30 + omp_get_thread_num()
            call get_npart(part_n, part_filename(repo, iout, icpu, mode), npart)
            !$OMP ATOMIC
            npart_tot = npart_tot + npart
            cursors_temp(i) = npart
        end do
        !$OMP END PARALLEL DO

        ncursor = 1
        do i = 1, SIZE(cpu_list)
            cursors(i) = ncursor
            ncursor = ncursor + cursors_temp(i)
        end do
        deallocate(cursors_temp)

        ! Set coulum spaces of each datatype for different versions of RAMSES
        if(nstar > 0 .or. nsink > 0) then
            nreal = 2*ndim + 3
        else
            nreal = 2*ndim + 1
        end if
        nint = 3
        nbyte = 0

        if(longint) then
            nint = nint-1
            nlong = 1
        else
            nlong = 0
        end if

        call close()

        ! Allocate space for particle data
        allocate(real_table(1:nreal, 1:npart_tot))
        allocate(integer_table(1:nint, 1:npart_tot))
        allocate(byte_table(1:nbyte, 1:npart_tot))
        if(longint)allocate(long_table(1:nlong, 1:npart_tot))

        ! Step 3: Read the actual particle data
        ! Current position for particle
        if(verbose)write(6, '(a)', advance='no') 'Progress: '
        !$OMP PARALLEL DO &
        !$OMP SHARED(integer_table, long_table, real_table, byte_table,ndim,nstar,nsink,nchem) &
        !$OMP PRIVATE(i,j, icpu,npart,part_n,npart_c,idim,pint) &
        !$OMP NUM_THREADS(nthread)
        do i = 1, SIZE(cpu_list)
            !$OMP CRITICAL
            if(verbose)call progress_bar(i, SIZE(cpu_list))
            !$OMP END CRITICAL
            icpu = cpu_list(i)
            part_n = 30 + omp_get_thread_num()
            npart_c = cursors(i)
            open(unit=part_n, file=part_filename(repo, iout, icpu, mode), status='old', form='unformatted')
            ! Skip headers
            call skip_read(part_n, 2)
            read(part_n) npart
            call skip_read(part_n, 5)

            ! Read position(3), velocity(3), mass
            do idim = 1, 2*ndim+1
                read(part_n) real_table(idim, npart_c:npart_c+npart-1) ! <---- ERROR
            end do

            (...)

            close(part_n)
        end do
        !$OMP END PARALLEL DO
        deallocate(cursors)
    end subroutine read_part
  1. 我使用
    f2py
    gfortran
    ifort
  2. 编译它
#!/bin/bash

F2PY=f2py
f='readr.f90'
bn=$(basename "$f" .f90)

# (gfortran case)
FORT=gfortran
$F2PY -c -lgomp --f90exec=$FORT --f77exec=$FORT $f -m $bn --opt='-O3 -x f95-cpp-input'

# (ifort case)
FORT=ifort
$F2PY -m $bn -liomp5 --f90exec=$FORT --f77exec=$FORT --opt='-O3 -fpp -m64 -free' -c $f
  1. 我在 VScode 的 Jupyter 扩展中导入它
#test.ipynb
...
from rur.readr import readr
readr.read_part(...)
  1. 内核在打开大文件(>80MB)时死机。 确切地说,Jupyter 内核死于
    read(part_n) real_table(idim, npart_c:npart_c+npart-1)
    。 由于内核死机时它不会打印任何错误消息,我无法提供任何进一步的信息。

我检查了这些:

  • gfortran
    案例每次都很好,但是
    ifort
    是个问题

  • 如果我运行 python,而不是 jupyter,那么它运行良好

  • 打开小文件就可以了

  • 设置

    ulimit -s unlimited
    export OMP_STACKSIZE='1G'

  • 我在setting.json中增加了max_buffer_size: “jupyter.jupyterCommandLineArguments”:[

      "--NotebookApp.iopub_msg_rate_limit=1000000000",
      "--NotebookApp.max_buffer_size=10240000000"
    

    ]

  • 它在

    read(part_n) real_table(idim, naprt_c:npart_c+npart-1)
    失败。但是当我读取一维数组时它会起作用,例如:
    read(part_n) real_table1D(naprt_c:npart_c+npart-1)

  • 这里是版本:

$ ifort --version
ifort (IFORT) 19.0.4.243 20190416
Copyright (C) 1985-2019 Intel Corporation.  All rights reserved.

$ f2py -h
...
Version:     1.22.4
numpy Version: 1.22.4
Requires:    Python 3.5 or higher.
License:     NumPy license (see LICENSE.txt in the NumPy source code)
Copyright 1999 - 2011 Pearu Peterson all rights reserved.

$ python3 --version
Python 3.10.4

$ conda list
...
ipython                   8.11.0                   pypi_0    pypi
ipython_genutils          0.2.0              pyhd3eb1b0_1  
ipywidgets                8.0.4                    pypi_0    pypi
...
jupyter                   1.0.0                    pypi_0    pypi
jupyter-client            8.0.3                    pypi_0    pypi
jupyter-console           6.4.4                    pypi_0    pypi
jupyter-core              5.2.0                    pypi_0    pypi
jupyter-events            0.6.3                    pypi_0    pypi
jupyter-server            1.23.4                   pypi_0    pypi
jupyter-server-terminals  0.4.4                    pypi_0    pypi
jupyter_client            7.2.2           py310h06a4308_0  
jupyter_console           6.4.4           py310h06a4308_0  
jupyter_core              4.10.0          py310hff52083_0    conda-forge
jupyter_server            1.23.4          py310h06a4308_0  
jupyterlab                3.5.3                    pypi_0    pypi
jupyterlab-pygments       0.2.2                    pypi_0    pypi
jupyterlab-server         2.19.0                   pypi_0    pypi
jupyterlab-widgets        3.0.5                    pypi_0    pypi
jupyterlab_pygments       0.1.2                      py_0  
jupyterlab_server         2.19.0          py310h06a4308_0  
jupyterlab_widgets        1.0.0              pyhd3eb1b0_1  

python visual-studio-code jupyter-notebook intel-fortran f2py
© www.soinside.com 2019 - 2024. All rights reserved.