spark处理广告数据中搜索词与bidword的大小写全角繁体问题

环境：spark 1.4.1， jpinyin-1.1.3.jar

spark-shell –executor-memory 10G –total-executor-cores 10 –jars ~/huangqiang/jpinyin-1.1.3.jar

import com.github.stuxuhai.jpinyin.ChineseHelper

val hiveCtx = new org.apache.spark.sql.hive.HiveContext(sc)

val hql1 = hiveCtx.sql(“select logdate, query, custid, groupid, bidword, quality, price from search_join_log where logdate <=’2016-01-07′ and logdate >= ‘2016-01-01′ and adtype=’2′ and adid > 0 and channel in(’16’,’40’,’51’,’52’,’78’,’80’,’72’,’73’,’81’)”)

def qj2Bj(str: String): String = {
var b = str.toCharArray
val c = b.map{i =>
if(i == ‘\\\\u3000’){
“”
}else if(i > ‘\\\\uFF00’ && i < ‘\\\\uFF5F’){
(i – 65248).toChar
}else{
i.toChar
}
}
return c.mkString
}

val norm_search_join_log = hql1.map{r =>
val q_bc = qj2Bj(r(1).toString.toLowerCase())
val b_bc = qj2Bj(r(4).toString.toLowerCase())
val q_sim = ChineseHelper.convertToSimplifiedChinese(q_bc)
val b_sim = ChineseHelper.convertToSimplifiedChinese(b_bc)
(r(0).toString, q_sim, r(2).toString, r(3).toString, b_sim, r(5).toString, r(6).toString)
}