Hadoop Installation on Linux，hadoopinstallation
1. Operation System
[hadoop@karei hadoop]$ uname -a
Linux karei 2.6.18-371.4.1.el5 #1 SMP Wed Jan 8 18:42:07 EST 2014 x86_64 x86_64 x86_64 GNU/Linux
[hadoop@karei hadoop]$
2. Cluster Info
[root@karei ~]# clustat
3. Install JDK, and set environment variables for Java
a. Install
rpm -ihv jdk-8u45-linux-i586.rpm
b. Set environment variables
export JAVA_HOME=/usr/java/jdk1.8.0_45
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
Note:
User: vi ~/.bash_profile --> source ~/.bash_profile
System: vi /etc/profile --> source /etc/profile
4. Download Hadoop from http://mirror.netinch.com/pub/apache/hadoop/common/
5. Decompression installation hadoop-2.7.0.tar.gz
a. gunzip hadoop-2.7.0.tar.gz
b. tar -xvf hadoop-2.7.0.tar.gz -C /usr/local/
c. cd /usr/local/
d. mv hadoop-2.7.0 hadoop
6. Set environment variables for Hadoop
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
#export JAVA_LIBRARY_PATH=$HADOOP_HOME/lib/native:$JAVA_LIBRARY_PATH
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
#export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
#export HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$HADOOP_HOME/lib"
Note:
User: vi ~/.bash_profile --> source ~/.bash_profile
System: vi /etc/profile --> source /etc/profile
7. Create user and group for Hadoop for all nodes of cluster
a. Create group
groupadd bigdata
b. Create user
adduser hadoop
c. Add user into group
usermod -a -G bigdata hadoop
d. Set password for user
passwd hadoop
e. chown Hadoop home directory
chown hadoop:bigdata /usr/local/hadoop
8. Set RSA as Hadoop user for all nodes of cluster
a. ssh-keygen -t rsa
b. ssh-copy-id <username>@<remote-host>
9. Configure nodes of cluster at file /etc/hosts
[root@karei ~]# cat /etc/hosts 127.0.0.1 localhost.localdomain localhost ::1 localhost6.localdomain6 localhost6 100.99.167.53 gerra.bigdata.hadoop.net gerra 100.99.167.54 hemei.bigdata.hadoop.net hemei 100.99.167.55 karei.bigdata.hadoop.net karei 100.99.167.56 lephi.bigdata.hadoop.net lephi [root@karei ~]#
10. Configure masters and slaves under directory $HADOOP_HOME/etc/hadoop
[root@karei hadoop]#
[root@karei hadoop]# cat $HADOOP_HOME/etc/hadoop/masters
karei
hemei
[root@karei hadoop]#
[root@karei hadoop]#
[root@karei hadoop]# cat $HADOOP_HOME/etc/hadoop/slaves
gerra
lephi
[root@karei hadoop]#
11. Configure Hadoop
a. Configure $HADOOP_HOME/etc/hadoop/core-site.xml
<?xml version="1.0"?>
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://karei:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/hadoop/tmp</value>
<description>Abase for other temporary directories.</description>
</property>
<property>
<name>hadoop.proxyuser.hadoop.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hadoop.groups</name>
<value>*</value>
</property>
</configuration>
b. Configure $HADOOP_HOME/etc/hadoop/yarn-site.xml
<?xml version="1.0"?>
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>karei:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>hemei:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>karei:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>karei:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>karei:8088</value>
</property>
</configuration>
c. Configure $HADOOP_HOME/etc/hadoop/mapred-site.xml
<?xml version="1.0"?>
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>karei:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>karei:19888</value>
</property>
</configuration>
d. Configure $HADOOP_HOME/etc/hadoop/hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>hemei:9001</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop/tmp/hdfs/datanode</value>
<description>DataNode directory for storing data chunks.</description>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop/tmp/hdfs/namenode</value>
<description>NameNode directory for namespace and transaction logs storage.</descri