28 lines
782 B
Scala
28 lines
782 B
Scala
package com.aisi.spark
|
|
|
|
import org.apache.spark.{SparkConf, SparkContext}
|
|
|
|
object WordCount {
|
|
def main(args: Array[String]): Unit = {
|
|
val config = new SparkConf()
|
|
config.setMaster("local")
|
|
config.setAppName("SH_wordcount")
|
|
|
|
// 禁用权限设置
|
|
config.set("spark.hadoop.dfs.permissions", "false")
|
|
|
|
val sc = new SparkContext(config)
|
|
|
|
// 使用 Windows 文件路径格式
|
|
val rdd = sc.textFile("D:/JetBrainsToolProject/IntelJ IDEA/HaiNiuProjects/Spark/data/word.txt")
|
|
|
|
val rdd1 = rdd.flatMap(_.split(" "))
|
|
val rdd2 = rdd1.map((_, 1))
|
|
val rdd3 = rdd2.groupBy(_._1)
|
|
val rdd4 = rdd3.mapValues(_.size)
|
|
|
|
// 保存结果到本地路径
|
|
rdd4.saveAsTextFile("D:/JetBrainsToolProject/IntelJ IDEA/HaiNiuProjects/Spark/data/res")
|
|
}
|
|
}
|