Spark手把手:[e2-spk-s02]

docker run -v /home/etadm/docker/spark/e2spkv01:/home:rw
-p 8088:8088 -p 8042:8042
-h sandbox
-it sequenceiq/spark:1.6.0 bash
2 . 5

spark-shell
--master local
--jars /home/e2-spk-s02/jars/spark-csv_2.10-1.4.0.jar
,/home/e2-spk-s02/jars/commons-csv-1.1.jar
2 . 6

// Create a object container
case class Word(text: String)
val fileName = "README.md"
val docs = sc.textFile(fileName)
val lower = docs.map(line => line.toLowerCase())
val words = lower.flatMap(line => line.split("s+"))
// Convert RDD to Dataframe using "Case class"
val words_df = words.map(Word(_)).toDF()
words_df.registerTempTable("words") // Register as a [TABLE]
val topWords = sqlContext.sql(
"SELECT text, count(text) AS n
FROM words
GROUP BY text
ORDER BY n DESC
LIMIT 10"
)
topWords.foreach(println)
5 . 3

case class Word(text: String)
val fileName = "README.md"
val docs = sc.textFile(fileName)
val lower = docs.map(line => line.toLowerCase())
val words = lower.flatMap(line => line.split("s+"))
val words_df = words.map(Word(_)).toDF()
words_df.registerTempTable("words")
val topWords = sqlContext.sql("SELECT text, count(text) AS n FROM words GROUP BY text ORDER BY
topWords.foreach(println)
5 . 19

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions._
val sc = new SparkContext(conf)
// Spark SQL DataFrame, SQLContext
val sqlContext = new SQLContext(sc)

// RDD DataFrame
import sqlContext.implicits._
7 . 4

// Parquet DataFrame
val df = sqlContext.read.parquet("people.parquet")

// DataFrame stdout
// Displays the content of the DataFrame to stdout
df.show()
7 . 5

// DataFrame stdout
df.show()
// Schema
df.printSchema()
// "name"
df.select("name").show()
// "age" +1
df.select(df("name"), df("age") + 1).show()
// 21 people
df.filter(df("age") > 21).show()
// age count
df.groupBy("age").count().show()
7 . 6

//
df.registerTempTable("people")
// DataFrame stdout
sqlContext.sql("SELECT * FROM people").show()
// Schema
sqlContext.sql("SELECT * FROM people").printSchema()
// "name"
sqlContext.sql("SELECT name FROM people").show()
// "age" +1
sqlContext.sql("SELECT name, (age + 1) as age FROM people").show()
// 21 people
sqlContext.sql("SELECT * FROM people WHERE age > 21").show()
// age count
sqlContext.sql("SELECT age, count(age) as count FROM people Group By age").show()
7 . 7

// case class Schema
case class Person(name: String, age: Int)
// DataFrame
val df = sc.textFile("people.txt").map(_.split(",")).map(p => Person(p(0), p(1).trim.toInt)).t
df.registerTempTable("people")
val teenagers = sqlContext.sql("SELECT name, age FROM people WHERE age >= 13 AND age <= 19")
// SQL query DataFrame, normal RDD operation
teenagers.map( t => "Name: " + t(0)).collect().foreach(println)
//
teenagers.map (t => "Name: " + t.getAs[String]("name")).collect().foreach(println)
// row.getValueMap[T] Map[String, T]
teenagers.map (_.getValuesMap[Any](List("name", "age"))).collect().foreach(println)
7 . 8

// , "parquet" Spark
val df = sqlContext.read.load("users.parquet")
// DataFrame "parquet"
df.select("name", "favorite_color").write.save("namesAndFavColors.parquet")
7 . 9

val df = sqlContext.read.format("json").load("people.json")
// DataFrame "parquet"
df.select("name", "age").write.save("namesAndAges.parquet")
7 . 10

import org.apache.spark.sql.SaveMode
val df = sqlContext.read.load("users.parquet")

// DataFrame "parquet" ( SaveMode.Overwrite)
df.select("name", "favorite_color").write
.mode(SaveMode.Overwrite).save("namesAndFavColors.parquet")
7 . 11

//define the schema using a case class
case class Auction(auctionid: String, bid: Float, bidtime: Float, bidder: String, bidderrate:
// ebay auction
val ebayText = sc.textFile("ebay.csv")
// Auction
val ebay = ebayText.map(_.split(",")).map(p => Auction(p(0), p(1).toFloat, p(2).toFloat, p(3),
// DataFrame
val auction = ebay.toDF()
auction.registerTempTable("auction")
// ?
val count = auction.select("auctionid").distinct.count
System.out.println(count)

// (item)
val results =sqlContext.sql("SELECT auctionid, item, count(bid) as bid_count FROM auction GRO
results.show()

// ( / / ) 8 . 4

import com.databricks.spark.csv
// 3rd party library "CSV" Dataframe
val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true") // Use first line of all files as header
.option("inferSchema", "true") // Automatically infer data types
.load("sfpd.csv")
// Schema
df.printSchema
// Distinct Category
df.select("Category").distinct().collect().foreach(println)
// temp table
df.registerTempTable("sfpd")
//
sqlContext.sql("SELECT distinct Category FROM sfpd").collect().foreach(println)
// Top 10
sqlContext.sql("SELECT Resolution , count(Resolution) as rescount FROM sfpd group by Resolutio
// Top 10
sqlContext.sql("SELECT Category , count(Category) as catcount FROM sfpd group by Category orde
8 . 6

Spark手把手:[e2-spk-s02]

Recommended

Recommended

More Related Content

What's hot

What's hot (20)

Viewers also liked

Viewers also liked (20)

Similar to Spark手把手:[e2-spk-s02]

Similar to Spark手把手:[e2-spk-s02] (20)

More from Erhwen Kuo

More from Erhwen Kuo (19)

Recently uploaded

Recently uploaded (20)

Spark手把手:[e2-spk-s02]