使用OpenCL优化而不是多处理池映射

问题描述 投票:-1回答:1

在我的代码的第一个版本中,我使用了python的Multiprocessing库,该库适用于16个线程的主函数MAIN_LOOP,如下所示:

def MAIN_LOOP(lll, seed=None):
    global aa
    global bb
    aa, bb = 0,0
    if paramo == 0:
        C_ij_GG, C_ij_LL, C_ij_GL = np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange)))
    C_ij_GG_up, C_ij_LL_up, C_ij_GL_up = np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange)))
    C_ij_GG_dw, C_ij_LL_dw, C_ij_GL_dw = np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange))), np.zeros((len(zrange), len(zrange)))
    while aa < len(zrange):
        while bb < len(zrange):
            if paramo == 0:
                C_ij_GG[aa][bb], C_ij_LL[aa][bb], C_ij_GL[aa][bb] = Pobs_C(zpm, zrange[aa], zrange[bb], h[2], Omega_m[2], Omega_DE[2], w0[2], wa[2], C_IA, A_IA[2], n_IA[2], B_IA[2], E_tab, R_tab, DG_tab, DG_tab, WG_tab, W_tab, WIA_tab, l[lll], P_dd_C, R_tab(z_pk)) 
            C_ij_GG_up[aa][bb], C_ij_LL_up[aa][bb], C_ij_GL_up[aa][bb] = Pobs_C(zpm, zrange[aa], zrange[bb], h[0], Omega_m[0], Omega_DE[0], w0[0], wa[0], C_IA, A_IA[0], n_IA[0], B_IA[0], E_tab_up, R_tab_up, DG_tab, DG_tab_up, WG_tab_up, W_tab_up, WIA_tab_up, l[lll], P_dd_C_up, R_tab_up(z_pk))
            C_ij_GG_dw[aa][bb], C_ij_LL_dw[aa][bb], C_ij_GL_dw[aa][bb] = Pobs_C(zpm, zrange[aa], zrange[bb], h[3], Omega_m[3], Omega_DE[3], w0[3], wa[3], C_IA, A_IA[3], n_IA[3], B_IA[3], E_tab_dw, R_tab_dw, DG_tab, DG_tab_dw, WG_tab_dw, W_tab_dw, WIA_tab_dw, l[lll], P_dd_C_dw, R_tab_dw(z_pk))
            bb=bb+1
        bb=0
        aa=aa+1

    if paramo == 0:
        aa, bb = 0,0 
        outGG=open(pre_CC_path[0]+CC_path[2]+"/COVAR_fid_"+str(l[lll]),'w')
        outLL=open(pre_CC_path[1]+CC_path[2]+"/COVAR_fid_"+str(l[lll]),'w')
        outGL=open(pre_CC_path[2]+CC_path[2]+"/COVAR_fid_"+str(l[lll]),'w')
        while aa < len(C_ij_GG):
            while bb < len(C_ij_GG):
                outGG.write(str("%.16e" % C_ij_GG[aa][bb]))
                outGG.write(str(' '))
                outLL.write(str("%.16e" % C_ij_LL[aa][bb]))
                outLL.write(str(' '))
                outGL.write(str("%.16e" % C_ij_GL[aa][bb]))
                outGL.write(str(' '))
                bb=bb+1
            outGG.write(str('\n'))
            outLL.write(str('\n'))
            outGL.write(str('\n'))
            bb=0
            aa=aa+1
        outGG.close()
        outLL.close()
        outGL.close()

    aa, bb = 0,0            
    outGGU=open(pre_CC_path[0]+CC_path[0]+"/COVAR_up_"+str(l[lll]),'w')
    outGGD=open(pre_CC_path[0]+CC_path[3]+"/COVAR_dw_"+str(l[lll]),'w')
    outLLU=open(pre_CC_path[1]+CC_path[0]+"/COVAR_up_"+str(l[lll]),'w')
    outLLD=open(pre_CC_path[1]+CC_path[3]+"/COVAR_dw_"+str(l[lll]),'w')
    outGLU=open(pre_CC_path[2]+CC_path[0]+"/COVAR_up_"+str(l[lll]),'w')
    outGLD=open(pre_CC_path[2]+CC_path[3]+"/COVAR_dw_"+str(l[lll]),'w')
    while aa < len(C_ij_GG_up):
        while bb < len(C_ij_GG_up):
            outGGU.write(str("%.16e" % C_ij_GG_up[aa][bb]))
            outGGU.write(str(' '))
            outGGD.write(str("%.16e" % C_ij_GG_dw[aa][bb]))
            outGGD.write(str(' '))
            outLLU.write(str("%.16e" % C_ij_LL_up[aa][bb]))
            outLLU.write(str(' '))
            outLLD.write(str("%.16e" % C_ij_LL_dw[aa][bb]))
            outLLD.write(str(' '))
            outGLU.write(str("%.16e" % C_ij_GL_up[aa][bb]))
            outGLU.write(str(' '))
            outGLD.write(str("%.16e" % C_ij_GL_dw[aa][bb]))
            outGLD.write(str(' '))
            bb=bb+1
        outGGU.write(str('\n'))
        outGGD.write(str('\n'))
        outLLU.write(str('\n'))
        outLLD.write(str('\n'))
        outGLU.write(str('\n'))
        outGLD.write(str('\n'))
        bb=0
        aa=aa+1
    outGGU.close()
    outGGD.close()
    outLLU.close()
    outLLD.close()
    outGLU.close()
    outGLD.close()
    lll=lll+1

lll = range(len(l))    
if __name__ == '__main__':          
    pool = mp.Pool(16)
    pool.map(MAIN_LOOP, lll)

并行化版本位于末尾,即带有:

if __name__ == '__main__':          
    pool = mp.Pool(16)
    pool.map(MAIN_LOOP, lll)

现在,我正在尝试使用另一种优化方法,并且尝试通过GPU / OpenCL进行优化:

所以,我没有用这个平行的multiprocessing pool代码部分,而是用:]代替了。

# NEW VERSION : with OpenCL

if __name__ == '__main__':          
  # GPU/OPenCL VERSION
  # Select a device
  ctx = cl.create_some_context()
  queue = cl.CommandQueue(ctx)
  # Kernel
  prg = cl.Program(ctx, """
  typedef int T;

  // Extern MAIN_LOOP function
  void MAIN_LOOP(__global T* in);

  __kernel
  void
  gpu_map(__global T* in, 
         const size_t n)
  {
    unsigned gid = get_global_id(0);

    // Call MAIN_LOOP with global_id
    MAIN_LOOP(in[gid]);
  }
  """).build()

  # Output compiler
  os.environ['PYOPENCL_COMPILER_OUTPUT'] = '1'
  # Allocate memory on the device and copy the content of our numpy array
  mf = cl.mem_flags
  # Get kernel function
  lll_np = np.array(lll, dtype=np.uint32)
  # Create input numpy
  lll_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=lll_np)
  #Get kernel function
  my_knl= prg.gpu_map
  my_knl.set_scalar_arg_dtypes([None, np.int32])
  my_knl(queue, lll_np.shape, None, lll_g, len(lll_np))

不幸的是,无法构建内核程序,出现以下错误:

Traceback (most recent call last):X2_non_flat_GPU_dev.py
  File "X2_non_flat_GPU_OpenCL_dev.py", line 671, in <module>
    """).build()
  File "/Users/fab/Library/Python/2.7/lib/python/site-packages/pyopencl/__init__.py", line 510, in build
    options_bytes=options_bytes, source=self._source)
  File "/Users/fab/Library/Python/2.7/lib/python/site-packages/pyopencl/__init__.py", line 554, in _build_and_catch_errors
    raise err
pyopencl._cl.RuntimeError: clBuildProgram failed: BUILD_PROGRAM_FAILURE - clBuildProgram failed: BUILD_PROGRAM_FAILURE - clBuildProgram failed: BUILD_PROGRAM_FAILURE

Build on <pyopencl.Device 'AMD Radeon Pro Vega 20 Compute Engine' on 'Apple' at 0x1021d00>:

Error returned by cvms_element_build_from_source
(options: -I /Users/fab/Library/Python/2.7/lib/python/site-packages/pyopencl/cl)
(source saved as /var/folders/y7/5dtgdjld5fxd3c1qm9hknlm40000gn/T/tmpg3pfTx.cl)

关于这些错误的任何线索?谢谢

UPDATE 1:

类似的悬赏开始于我的代码基准测试:Different ways to optimize with GPU PyOpenCL a python code : extern function inside kernel GPU/PyOpenCL

从运行时的角度来看,有关代码部分的更多信息是贪婪的。但是,这种赏金更多地是关于可以找到一种优化方法的一般想法。

在我的代码的第一个版本中,我使用了python的Multiprocessing库,该库应用于16个线程的主函数MAIN_LOOP上,如下所示:def MAIN_LOOP(lll,seed = None):global aa global ...

numpy gpu opencl python-multiprocessing pyopencl
1个回答
0
投票

在OpenCL中,您有一个主机(在您的情况下,您的CPU执行Python代码)和几个设备(通常是单个GPU)。

主机使设备执行的入队操作:

© www.soinside.com 2019 - 2024. All rights reserved.