计算气候数据百分位数

Question

import os
import numpy as np
import xarray as xr

data_folder1 = 'K:\\reanalysis\ERA5\\100m_speed\\100m u\\1979-2022'
data_folder2 = 'K:\\reanalysis\ERA5\\100m_speed\\100m v\\1979-2022'

file_names_u = os.listdir(data_folder1)
file_names_v = os.listdir(data_folder2)

def read_wind(file_path_u, file_path_v, lat_value, lon_value):
    with xr.open_dataset(file_path_u) as ds:
        u_wind = ds['u100'].sel(latitude=lat_value, longitude=lon_value).values
    with xr.open_dataset(file_path_v) as ds:
        v_wind = ds['v100'].sel(latitude=lat_value, longitude=lon_value).values
    return u_wind, v_wind

lat_value = 0
lon_value = 90
all_u_winds = []
all_v_winds = []

for file_name_u, file_name_v in zip(file_names_u, file_names_v):
    file_path_u = os.path.join(data_folder1, file_name_u)
    file_path_v = os.path.join(data_folder2, file_name_v)
    u_wind, v_wind = read_wind(file_path_u, file_path_v, lat_value, lon_value)
    all_u_winds.extend(u_wind)
    all_v_winds.extend(v_wind)

all_u_winds = np.array(all_u_winds)
all_v_winds = np.array(all_v_winds)

# 计算风速
all_windspeeds = np.sqrt(np.square(all_u_winds) + np.square(all_v_winds))

percentile_50 = np.percentile(all_windspeeds, 50)
print(f"The 50% percentile of the wind speed values is: {percentile_50}")

我想计算我的nc文件中44年的经纬度网格点的风速百分位数。我的一个文件有1.4G，总共有528个u和528个v文件，但是用我现在的代码计算速度非常慢。您能帮我看看如何改进吗

Answer 1

我建议使用 pyinstrument 等分析器来查找代码中最慢的部分。我已将所需的行添加到您的代码中以使用 pyinstrument（见下文）。从一些数据文件开始。找到并修复主要瓶颈后，尝试使用更多数据文件运行代码。

import os
import numpy as np
import xarray as xr
from pyinstrument import Profiler

profiler = Profiler()
profiler.start()

data_folder1 = 'K:\\reanalysis\ERA5\\100m_speed\\100m u\\1979-2022'
data_folder2 = 'K:\\reanalysis\ERA5\\100m_speed\\100m v\\1979-2022'

file_names_u = os.listdir(data_folder1)
file_names_v = os.listdir(data_folder2)

def read_wind(file_path_u, file_path_v, lat_value, lon_value):
    with xr.open_dataset(file_path_u) as ds:
        u_wind = ds['u100'].sel(latitude=lat_value, longitude=lon_value).values
    with xr.open_dataset(file_path_v) as ds:
        v_wind = ds['v100'].sel(latitude=lat_value, longitude=lon_value).values
    return u_wind, v_wind

lat_value = 0
lon_value = 90
all_u_winds = []
all_v_winds = []

for file_name_u, file_name_v in zip(file_names_u, file_names_v):
    file_path_u = os.path.join(data_folder1, file_name_u)
    file_path_v = os.path.join(data_folder2, file_name_v)
    u_wind, v_wind = read_wind(file_path_u, file_path_v, lat_value, lon_value)
    all_u_winds.extend(u_wind)
    all_v_winds.extend(v_wind)

all_u_winds = np.array(all_u_winds)
all_v_winds = np.array(all_v_winds)

# 计算风速
all_windspeeds = np.sqrt(np.square(all_u_winds) + np.square(all_v_winds))

percentile_50 = np.percentile(all_windspeeds, 50)
print(f"The 50% percentile of the wind speed values is: {percentile_50}")

profiler.stop()
profiler.print()