hdfsstart.sh 1.83 KB
Newer Older
Shao-Ching Huang's avatar
Shao-Ching Huang committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
#!/bin/sh

source /u/local/Modules/default/init/modules.sh
export MODULEPATH=$MODULEPATH:/u/local/Modules/modulefiles
module load java
module load hadoop

source /u/local/bin/set_qrsh_env.sh
export HADOOP_CONF_BASE=$HOME/.hadoop
mkdir -p $HADOOP_CONF_BASE
rm -rf $TMPDIR/hadoop-*

echo "NUMBER of HOSTS = $NHOSTS"
echo "TMPDIR      = $TMPDIR"

##==export HADOOP_CONF_DIR=${SGE_O_WORKDIR}/conf.${JOB_ID}
dt=$(date '+%Y-%m-%d-%H-%M-%S');
export HADOOP_CONF_DIR=$HADOOP_CONF_BASE/conf.${JOB_ID}
export HADOOP_LOG_DIR=$HADOOP_CONF_BASE/logs.${JOB_ID}
export YARN_CONF_DIR=$HADOOP_CONF_BASE/conf.${JOB_ID}
export YARN_LOG_DIR=$HADOOP_CONF_BASE/logs.${JOB_ID}

echo "Configuration directory is ${HADOOP_CONF_DIR}"
echo "HADOOP LOG directory is ${HADOOP_LOG_DIR}"
echo "YARN LOG directory is ${YARN_LOG_DIR}"

/bin/cp -r ${HADOOP_HOME}/etc/hadoop/ ${HADOOP_CONF_DIR}


#sleep 10

port=9000
count=0
master=`/bin/cat $PE_HOSTFILE | /bin/sed -n 1p | /bin/cut -f1 -d" " `
####master=`hostname`

/u/local/etc/sge.cfg/portcheck.pl $master $port
while [[ $? -eq 0 && ${count}  -lt 20 ]]
do
     port=`/usr/bin/expr ${port} + 2`
     count=`/usr/bin/expr ${count} + 1`
     echo "${master}  ${port}  $count"
     /u/local/etc/sge.cfg/portcheck.pl $master $port
done

coreport=$port
echo "Core Port: $coreport"
port=`/usr/bin/expr ${port} + 1`
count=0
/u/local/etc/sge.cfg/portcheck.pl $master $port
while [[ $? -eq 0 && ${count}  -lt 20 ]]
do
     port=`/usr/bin/expr ${port} + 2`
     count=`/usr/bin/expr ${count} + 1`
     echo "${master}  ${port}  $count"
     /u/local/etc/sge.cfg/portcheck.pl $master $port
done

#echo "${master}  ${port}  $count"
mapport=$port
echo "Map Port: $mapport"


python /u/systems/SGE/hoffman2/scripts/hadoop/2.x/hdfsprolog.py $PE_HOSTFILE $TMPDIR $NHOSTS $HADOOP_CONF_DIR $coreport $mapport

hdfs namenode -format -force

start-dfs.sh
start-yarn.sh