'''
--------------------------------------------------------------------------------
福利数据解析
基础数据分析，标题分词，词频统计
-----------------------------------
by:obaby
email: root@obaby.org.cn
blog:http://www.h4ck.org.cn
===================================
参考链接：https://sparkbyexamples.com/pyspark/select-columns-from-pyspark-dataframe/
-------------------------------------------------------------------------------
'''
import jieba
# 通过spark read csv格式文件，从csv header解析数据结构
csv = spark.read.option("header",True).csv("hdfs://localhost:9000/data2/porn_data_movie.csv")
# 数据格式
csv.printSchema()
root
 |-- id: string (nullable = true)
 |-- create: string (nullable = true)
 |-- update: string (nullable = true)
 |-- name: string (nullable = true)
 |-- describe: string (nullable = true)
 |-- source_id: string (nullable = true)
 |-- publish_time: string (nullable = true)
 |-- play_count: string (nullable = true)
 |-- good_count: string (nullable = true)
 |-- bad_count: string (nullable = true)
 |-- link_count: string (nullable = true)
 |-- comment_count: string (nullable = true)
 |-- designation: string (nullable = true)
 |-- category_id: string (nullable = true)
 |-- porn_site_id: string (nullable = true)
 |-- uploader_id: string (nullable = true)
 |-- producer: string (nullable = true)