package com.aisi.spark

import org.apache.spark.{SparkConf, SparkContext}

object WordCount {
  def main(args: Array[String]): Unit = {
    val config = new SparkConf()
    config.setMaster("local")
    config.setAppName("SH_wordcount")

    // 禁用权限设置
    config.set("spark.hadoop.dfs.permissions", "false")

    val sc = new SparkContext(config)

    // 使用 Windows 文件路径格式
    val rdd = sc.textFile("D:/JetBrainsToolProject/IntelJ IDEA/HaiNiuProjects/Spark/data/word.txt")

    val rdd1 = rdd.flatMap(_.split(" "))
    val rdd2 = rdd1.map((_, 1))
    val rdd3 = rdd2.groupBy(_._1)
    val rdd4 = rdd3.mapValues(_.size)

    // 保存结果到本地路径
    rdd4.saveAsTextFile("D:/JetBrainsToolProject/IntelJ IDEA/HaiNiuProjects/Spark/data/res")
  }
}