在 OS/X 上找不到 Hadoop 本机库

问题描述 投票:0回答:5

我已经从github下载了

hadoop
源代码并使用
native
选项进行编译:

mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true

然后我将

.dylib
文件复制到 $HADOOP_HOME/lib

cp -p hadoop-common-project/hadoop-common/target/hadoop-common-2.7.1/lib/native/*.dylib /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/lib

LD_LIBRARY_PATH已更新并重新启动hdfs:

 echo $LD_LIBRARY_PATH
 /usr/local/Cellar/hadoop/2.7.2/libexec/lib:
 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/lib:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home//jre/lib

(注意:这也意味着 docker-spark 上的 Hadoop“无法为您的平台加载本机 hadoop 库”错误的答案? 对我不起作用..)

但是

checknative
仍然统一返回
false
:

$stop-dfs.sh && start-dfs.sh && hadoop checknative
16/06/13 16:12:32 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Stopping namenodes on [sparkbook]
sparkbook: stopping namenode
localhost: stopping datanode
Stopping secondary namenodes [0.0.0.0]
0.0.0.0: stopping secondarynamenode
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Starting namenodes on [sparkbook]
sparkbook: starting namenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-namenode-sparkbook.out
localhost: starting datanode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-datanode-sparkbook.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-secondarynamenode-sparkbook.out
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Native library checking:
hadoop:  false
zlib:    false
snappy:  false
lz4:     false
bzip2:   false
openssl: false
macos hadoop hadoop-native-library
5个回答
24
投票

要在全新安装的 macOS 10.12 上使其正常工作,我必须执行以下操作:

  1. 使用 homebrew 安装构建依赖项:

    brew install cmake maven openssl [email protected] snappy
    
  2. 查看hadoop源代码

    git clone https://github.com/apache/hadoop.git
    cd hadoop
    git checkout rel/release-2.7.3
    
  3. 将以下补丁应用到构建中:

    diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
    index 942b19c..8b34881 100644
    --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
    +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
    @@ -16,6 +16,8 @@
     # limitations under the License.
     #
    
    +SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl)
    +
     cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
    
     # Default to release builds
    @@ -116,8 +118,8 @@ set(T main/native/src/test/org/apache/hadoop)
     GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)
    
     SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
    -set_find_shared_library_version("1")
    -find_package(BZip2 QUIET)
    +set_find_shared_library_version("1.0")
    +find_package(BZip2 REQUIRED)
     if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
         GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
         set(BZIP2_SOURCE_FILES
    diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
    index d2ddf89..ac8e351 100644
    --- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
    +++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
    @@ -17,4 +17,8 @@
     <!-- Put site-specific property overrides in this file. -->
    
     <configuration>
    +<property>
    +<name>io.compression.codec.bzip2.library</name>
    +<value>libbz2.dylib</value>
    +</property>
     </configuration>
    diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
    index 34c0110..70f23a4 100644
    --- a/hadoop-tools/hadoop-pipes/pom.xml
    +++ b/hadoop-tools/hadoop-pipes/pom.xml
    @@ -52,7 +52,7 @@
                         <mkdir dir="${project.build.directory}/native"/>
                         <exec executable="cmake" dir="${project.build.directory}/native" 
                             failonerror="true">
    -                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
    +                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/>
                         </exec>
                         <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                           <arg line="VERBOSE=1"/>
    
  4. 从源代码构建hadoop:

    mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true
    
  5. 运行hadoop时指定

    JAVA_LIBRARY_PATH

    $ JAVA_LIBRARY_PATH=/usr/local/opt/openssl/lib:/opt/local/lib:/usr/lib hadoop-dist/target/hadoop-2.7.3/bin/hadoop checknative -a
    16/10/14 20:16:32 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library libbz2.dylib
    16/10/14 20:16:32 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
    Native library checking:
    hadoop:  true /Users/admin/Desktop/hadoop/hadoop-dist/target/hadoop-2.7.3/lib/native/libhadoop.dylib
    zlib:    true /usr/lib/libz.1.dylib
    snappy:  true /usr/local/lib/libsnappy.1.dylib
    lz4:     true revision:99
    bzip2:   true /usr/lib/libbz2.1.0.dylib
    openssl: true /usr/local/opt/openssl/lib/libcrypto.dylib
    

6
投票

上面 @andrewdotn 的回复中缺少一些步骤:

1) 对于步骤 (3),通过将发布的文本添加到文本文件来创建补丁,例如“patch.txt”,然后执行“git apply patch.txt”

2) 除了按照 javadba 的指示复制文件之外,某些应用程序还要求您设置:

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/native
export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:${HADOOP_HOME}/lib/native

1
投票

所需的步骤是将

*.dylib
git
源构建目录复制到您平台的
$HADOOP_HOME/<common dir>lib
目录中。对于通过
OS/X
安装的
brew
,它是:

cp /git/hadoop/hadoop-dist/target/hadoop-2.7.1/lib/native/ /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/

我们现在可以在那里看到所需的库:

$ll /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/*.dylib
-rwxr-xr-x  1 macuser  staff  149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib
-rwxr-xr-x  1 macuser  staff  149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.1.0.0.dylib

现在

hadoop checknative
命令可以工作了:

$hadoop checknative

6/06/15 09:10:59 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
Native library checking:
hadoop:  true /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib
zlib:    true /usr/lib/libz.1.dylib
snappy:  false
lz4:     true revision:99
bzip2:   false
openssl: false build does not support openssl.

1
投票

作为 @andrewdotn 答案的更新,这里是与 Hadoop 2.8.1 一起使用的

patch.txt
文件:

diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index c93bfe78546..e8918f9ca29 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
@@ -20,6 +20,8 @@
 # CMake configuration.
 #

+SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl)
+
 cmake_minimum_required(VERSION 2.6 FATAL_ERROR)

 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..)
@@ -50,8 +52,8 @@ get_filename_component(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME)

 # Look for bzip2.
 set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-hadoop_set_find_shared_library_version("1")
-find_package(BZip2 QUIET)
+hadoop_set_find_shared_library_version("1.0")
+find_package(BZip2 REQUIRED)
 if(BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES)
     get_filename_component(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME)
     set(BZIP2_SOURCE_FILES
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
index d2ddf893e49..ac8e351f1c8 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml
@@ -17,4 +17,8 @@
 <!-- Put site-specific property overrides in this file. -->

 <configuration>
+<property>
+<name>io.compression.codec.bzip2.library</name>
+<value>libbz2.dylib</value>
+</property>
 </configuration>
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 8aafad0f7eb..d4832542265 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -55,7 +55,7 @@
                     <mkdir dir="${project.build.directory}/native"/>
                     <exec executable="cmake" dir="${project.build.directory}/native" 
                         failonerror="true">
-                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+                      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                       <arg line="VERBOSE=1"/>

0
投票

我们有 Hadoop 3.2.2 的补丁吗? 请注意,我使用的是 Mac m2 苹果硅。 @andrewdotn,也请提供苹果硅的步骤。英特尔的步骤对此不起作用。

© www.soinside.com 2019 - 2024. All rights reserved.