????5??????mapred-site.xml???
<configuration>
/*hadoop??map-reduce???п????????3????????mapred-site.xml?????“mapreduce.framework.name”??????????????"classic"."yarn"????“local”*/
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
/*MapReduce JobHistory Server???*/
<property>
<name>mapreduce.jobhistory.address</name>
<value>Master:10020</value>
</property>
/*MapReduce JobHistory Server web UI ???*/
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>Master:19888</value>
</property>
</configuration>
??????4??????namenode??datanode???????????????·??
????1??????namenode??datanode????
????mkdir /hdfs/namenode
????mkdir /hdfs/datanode
????2????????????λ??????????hdfs-site.xml???????????????????????
<configuration>
/*???????????????*/
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>Master:9001</value>
</property>
/*???????????*/
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/hdfs/namenode</value>
</property>
/*????datanode??????洢??*/
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/hdfs/datanode</value>
</property>
/*?????????*/
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
/*??dfs.webhdfs.enabled?????????true???????????webhdfs??LISTSTATUS??LISTFILESTATUS??????г????????????????????????Щ?????????namenode?????*/
<proeprty>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
</configuration>
??????5??????Master??Slave???
????1??Master???????????????????????
????Master /*Master????????????*/
????2??????Slaves???????????????
????/*Slave???????????*/
????Slave1
????Slave2
????Slave3
????Slave4
????Slave5
??????6????Hadoop??????????pssh????????????
????./pssh -h hosts.txt -r /hadoop /
??????7???????Namenode????Hadoop?????￡?
????./bin/hadoop namenode -format
??????8?????Hadoop
????./sbin/start-all.sh
??????9??????????ú???????
????jps
????DataNode
????ResourceManager
????Jps
????NodeManager
????NameNode
????SecondaryNameNode
????ps-aux|grep *DataNode* /*??DataNode????*/
????5 ???Spark
????1??????spark
????2?????
????3??????
??????conf/Spark-env.sh?????????????????ò???
????export SCALA_HOME=/paht/to/scala-2.10.4
????export SPARK_WORKER_MEMORY=7g
????export SPARK_MASTER_IP=172.16.0.140
????export MASTER=spark://172.16.0.140:7077
????4??????slaves???
??????conf/slaves???????5??Worker????????????????????????slaves?????
????Slave1
????Slave2
????Slave3
????Slave4
????Slave5
????6 ??????
??????1??Spark???????
????1????Spark???????Spark
????./sbin/start-all.sh
????2?????Spark
????./sbin/stop-all.sh
??????2??Hadoop?????????
????1????Hadoop???????Hadoop
????2?????Hadoop
??????3????????????
????1?????????μ?Master????????
????2??????ssh???Worker???