apache hadoop-2.6.0-CDH5.4.1 安装笔记，hadoop2.6.0安装

apache hadoop-2.6.0-CDH5.4.1 安装

1.安装Oracle Java 8

sudo add-apt-repository ppa:webupd8team/java
sudo apt-get update
sudo apt-get install oracle-java8-installer

sudo vi /etc/profile

#set java environment
export JAVA_HOME=/usr/lib/jvm/oracle-java8-installer
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

source /etc/profile

验证java环境是否配好
java -version

---

2.实验机器角色分配

主机名	ip	角色
Master	10.5.0.196	MameNode、ResourceManager、SecondaryNameNode
Slave1	10.5.0.231	DataNode、NodeManager
Slave2	10.5.0.232	DataNode、NodeManager
Slave3	10.5.0.233	DataNode、NodeManager

配置Master
修改 /etc/hostname 内容为Master （Slave1为Slave1…）

/etc/hosts 添加内容行

10.5.0.196      Master 
10.5.0.231      Slave1
10.5.0.232      Slave2
10.5.0.233      Slave3

---

3.建立hadoop用户

useradd hadoop
passwd hadoop
设置管理员权限
vi /etc/sudoers

`#` User privilege specification
root    ALL=(ALL:ALL) ALL
hadoop  ALL=(ALL:ALL) ALL

---

4. Master上配置SSH免密码登录

sudo apt-get install openssh-server
sudo su hadoop
cd /home/hadoop
ssh-keygen -t rsa(一路回车 生成密钥)
cd .ssh
cp id_rsa.pub authorized_keys
或者 cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys

将公钥拷到Slave上
ssh-copy-id -i $HOME/.ssh/id_rsa.pub hadoop@Slave1
#或者 scp authorized_keys hadoop@Slave1:/home/hadoop/.ssh/

---

5.下载、安装hadoop-cdh

wget http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.4.1.tar.gz
tar -zxvf hadoop-2.6.0-cdh5.4.1.tar.gz -C /usr/local/cloudera/hadoop-2.6.0-cdh5.4.1
sudo chown hadoop:hadoop -R /usr/local/cloudera
sudo mkdir -p /usr/local/cloudera/hadoop_tmp/hdfs/namenode
sudo mkdir -p /usr/local/cloudera/hadoop_tmp/hdfs/datanode
sudo chown hadoop:hadoop -R /usr/local/hadoop_tmp

配置 ~/.bashrc
sudo vi $HOME/.bashrc

添加下列内容到文件尾部

`#`HADOOP VARIABLES START
export JAVA_HOME=/usr/lib/jvm/oracle-java8-installer
export HADOOP_HOME=/usr/local/cloudera/hadoop-2.6.0-cdh5.4.1
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
`#`HADOOP VARIABLES END

source ~/.bashrc

cd $HADOOP_HOME/etc/hadoop

sudo vi hadoop-env.sh
修改JAVA_HOME的值
export JAVA_HOME=/usr/lib/jvm/oracle-java8-installer

sudo vi slaves
Slave1
Slave2
Slave3

sudo vi masters
Master

---

6.配置.XML文件

core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml

1. core-site.xml

<configuration>
<!-- file system properties -->
<property>
        <name>fs.default.name</name>
        <value>hdfs://Master:9000</value>
</property>

<property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
</property>
</configuration>

2. hdfs-site.xml

/hdfs/namenode、/hdfs/datanode 目录需要自己新建

<configuration>
<property>
        <name>dfs.replication</name>
        <value>1</value>
</property>

<property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/usr/local/cloudera/hadoop_tmp/hdfs/namenode