QuickReference/env.md at aaeed68ca9c94e0c004cc41bb4783090e739f708

7.1 KiB

Raw Blame History

title	top_img	top_img_height	date	tags
Hadoop集群搭建基础环境	/img/site01.jpg	800px	2024-08-01 09:10:40	hadoop

防火墙关闭

# 在 6 台主机执行
systemctl stop firewalld
systemctl disable firewalld

配置yum源

下载 repo 文件： Centos-7.repo 并上传到/tmp，进入到/tmp

备份并且替换系统的repo文件

	cp Centos-7.repo /etc/yum.repos.d/ 
  cd /etc/yum.repos.d/ 
  mv CentOS-Base.repo CentOS-Base.repo.bak 
  mv Centos-7.repo CentOS-Base.repo

将nn1上的CentOS-Base.repo拷贝到其他主机

scp /etc/yum.repos.d/CentOS-Base.repo root@nn2:/etc/yum.repos.d
scp /etc/yum.repos.d/CentOS-Base.repo root@nn3:/etc/yum.repos.d
scp /etc/yum.repos.d/CentOS-Base.repo root@s1:/etc/yum.repos.d
scp /etc/yum.repos.d/CentOS-Base.repo root@s2:/etc/yum.repos.d
scp /etc/yum.repos.d/CentOS-Base.repo root@s3:/etc/yum.repos.d

执行yum源更新命令

    yum clean all
    yum makecache 
    yum update -y

安装常用软件

yum install -y openssh-server vim gcc gcc-c++ glibc-headers bzip2-devel lzo-devel curl wget openssh-clients zlib-devel autoconf automake cmake libtool openssl-devel fuse-devel snappy-devel telnet unzip zip net-tools.x86_64 firewalld systemd ntp unrar bzip2

JDK安装

注意需要在六台机器依次执行

上传到/tmp目录下，安装

cd /tmp
rpm -ivh jdk-8u144-linux-x64.rpm

配置环境变量

ln -s /usr/java/jdk1.8.0_144/ /usr/java/jdk1.8
echo 'export JAVA_HOME=/usr/java/jdk1.8' >> /etc/profile.d/myEnv.sh 
echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile.d/myEnv.sh 
source /etc/profile 
java -version

修改主机名和主机名映射

vim /etc/hostname

6台机器分别为nn1、nn2、nn3、s1、s2、s3

vim /etc/hosts

修改为

192.168.1.30 nn1
192.168.1.31 nn2
192.168.1.32 nn3
192.168.1.33 s1
192.168.1.34 s2
192.168.1.35 s3

创建hadoop用户

#创建hadoop用户 
useradd hadoop 
#给hadoop用户设置密码: 12345678 
passwd hadoop

禁止非 wheel 组用户切换到root，配置免密切换root

修改/etc/pam.d/su配置

sed -i 's/#auth\t\trequired\tpam_wheel.so/auth\t\trequired\tpam_wheel.so/g' '/etc/pam.d/su' 
sed -i 's/#auth\t\tsufficient\tpam_wheel.so/auth\t\tsufficient\tpam_wheel.so/g' '/etc/pam.d/su'

修改/etc/login.defs文件

echo "SU_WHEEL_ONLY yes" >> /etc/login.defs

添加用户到管理员，禁止普通用户su 到 root

#把hadoop用户加到wheel组里
gpasswd -a hadoop wheel
#查看wheel组里是否有hadoop用户
cat /etc/group | grep wheel

给hadoop用户，配置SSH密钥

配置hadoop用户ssh免密码登录到hadoop

仅在nn1执行这段脚本命令即可但是 su - hadoop , mkdir ~/.ssh 需要在其他主机执行一下

#切换到hadoop用户 
su - hadoop
#生成ssh公私钥 
ssh-keygen -t rsa -f ~/.ssh/id_rsa -P ''
ssh-copy-id nn1
ssh-copy-id nn2
ssh-copy-id nn3
ssh-copy-id s1
ssh-copy-id s2
ssh-copy-id s3
scp /home/hadoop/.ssh/id_rsa hadoop@nn2:/home/hadoop/.ssh
scp /home/hadoop/.ssh/id_rsa hadoop@nn3:/home/hadoop/.ssh
scp /home/hadoop/.ssh/id_rsa hadoop@s1:/home/hadoop/.ssh
scp /home/hadoop/.ssh/id_rsa hadoop@s2:/home/hadoop/.ssh
scp /home/hadoop/.ssh/id_rsa hadoop@s3:/home/hadoop/.ssh

配置hadoop用户ssh免密码登录到root

同上

ssh-copy-id root@nn1
ssh-copy-id root@ nn2
ssh-copy-id root@nn3
ssh-copy-id root@s1
ssh-copy-id root@s2
ssh-copy-id root@s3
scp /home/hadoop/.ssh/id_rsa root@nn2:/root/.ssh
scp /home/hadoop/.ssh/id_rsa root@nn3:/root/.ssh
scp /home/hadoop/.ssh/id_rsa root@s1:/root/.ssh
scp /home/hadoop/.ssh/id_rsa root@s2:/root/.ssh
scp /home/hadoop/.ssh/id_rsa root@s3:/root/.ssh

脚本配置

ips

vim /home/hadoop/bin/ips

nn1 
nn2
nn3
s1 
s2 
s3

ssh_all.sh

vim /home/hadoop/bin/ssh_all.sh

#! /bin/bash
# 进入到当前脚本所在目录
cd `dirname $0`
# 获取当前脚本所在目录
dir_path=`pwd`
#echo $dir_path
# 读ips文件得到数组(里面是一堆主机名)
ip_arr=(`cat $dir_path/ips`)
# 遍历数组里的主机名
for ip in ${ip_arr[*]}
do
        # 拼接ssh命令: ssh hadoop@nn1.hadoop ls
        cmd_="ssh hadoop@${ip} \"$*\"  "
        echo $cmd_
        # 通过eval命令 执行 拼接的ssh 命令
        if eval ${cmd_} ; then
                echo "OK"
        else
                echo "FAIL"
        fi
done

ssh_root.sh

#! /bin/bash
# 进入到当前脚本所在目录
cd `dirname $0`
# 获取当前脚本所在目录
dir_path=`pwd`
#echo $dir_path
# 读ips文件得到数组(里面是一堆主机名)
ip_arr=(`cat $dir_path/ips`)
# 遍历数组里的主机名
for ip in ${ip_arr[*]}
do
        # 拼接ssh命令: ssh hadoop@nn1.hadoop ls
        cmd_="ssh  hadoop@${ip} ~/bin/exe.sh \"$*\""
        echo $cmd_
        # 通过eval命令 执行 拼接的ssh 命令
        if eval ${cmd_} ; then
                echo "OK"
        else
                echo "FAIL"
        fi
done

scp_all.sh

#! /bin/bash
# 进入到当前脚本所在目录
cd `dirname $0`
# 获取当前脚本所在目录
dir_path=`pwd`
#echo $dir_path
# 读ips文件得到数组(里面是一堆主机名)
ip_arr=(`cat $dir_path/ips`)
# 源
source_=$1
# 目标
target=$2
# 遍历数组里的主机名
for ip in ${ip_arr[*]}
do
        # 拼接scp命令: scp 源 hadoop@nn1.hadoop:目标
        cmd_="scp -r ${source_} hadoop@${ip}:${target}"
        echo $cmd_
        # 通过eval命令 执行 拼接的scp 命令
        if eval ${cmd_} ; then
                echo "OK"
        else
                echo "FAIL"
        fi
done

exe.sh

#切换到root用户执行cmd命令
cmd=$*
su - << EOF
$cmd
EOF

赋予执行权限

chmod +x ssh_all.sh 
chmod +x scp_all.sh
chmod +x ssh_root.sh
chmod +x exe.sh

分发到其他主机

./ssh_all.sh mkdir /home/hadoop/bin
./scp_all.sh /home/hadoop/bin/ips /home/hadoop/bin/
./scp_all.sh /home/hadoop/bin/exe.sh /home/hadoop/bin/
./scp_all.sh /home/hadoop/bin/ssh_all.sh /home/hadoop/bin/
./scp_all.sh /home/hadoop/bin/scp_all.sh /home/hadoop/bin/
./scp_all.sh /home/hadoop/bin/ssh_root.sh /home/hadoop/bin/

将 /home/hadoop/bin添加到hadoop用户的环境变量，需要切换到hadoop用户

echo 'export PATH=$PATH:/home/hadoop/bin' >> ~/.bashrc && source ~/.bashrc
scp_all.sh /home/hadoop/.bashrc /home/hadoop/
ssh_all.sh source ~/.bashrc

7.1 KiB Raw Blame History Unescape Escape