upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/memcached/ComplementByMemcache.scala
1package cn.piflow.bundle.memcached
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import com.danga.MemCached.{MemCachedClient, SockIOPool}
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12import scala.collection.mutable
13
14class ComplementByMemcache extends ConfigurableStop {
15 override val authorEmail: String = "yangqidong@cnic.cn"
16 override val description: String = "Complement by Memcache"
17 val inportList: List[String] = List(PortEnum.DefaultPort.toString)
18 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
19
20 var servers:String=_ //Server address and port number,If you have multiple servers, use "," segmentation.
21 var keyFile:String=_ //The field you want to use as a query condition
22 var weights:String=_ //Weight of each server
23 var maxIdle:String=_ //Maximum processing time
24 var maintSleep:String=_ //Main thread sleep time
25 var nagle:String=_ //If the socket parameter is true, the data is not buffered and sent immediately.
26 var socketTO:String=_ //Socket timeout during blocking
27 var socketConnectTO:String=_ //Timeout control during connection establishment
28 var replaceField:String=_ //The fields you want to get
29
30
31 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
32 val session: SparkSession = pec.get[SparkSession]()
33 val inDF: DataFrame = in.read()
34
35 val mcc: MemCachedClient =getMcc()
36
37 val replaceFields: mutable.Buffer[String] = replaceField.split(",").toBuffer
38
39 val rowArr: Array[Row] = inDF.collect()
40 val fileNames: Array[String] = inDF.columns
41 val data: Array[Map[String, String]] = rowArr.map(row => {
42 var rowStr: String = row.toString().substring(1,row.toString().length-1)
43 val dataARR: Array[String] = rowStr.split(",")
44 var map: Map[String, String] = Map()
45 for (x <- (0 until fileNames.size)) {
46 map += (fileNames(x) -> dataARR(x))
47 }
48 map
49 })
50
51 val finalData: Array[Map[String, String]] = data.map(eachData => {
52 var d: Map[String, String] = eachData
53 val anyRef: AnyRef = mcc.get(d.get(keyFile).get)
54 if(anyRef.getClass.toString.equals("class scala.Some")){
55 val map: Map[String, String] = anyRef.asInstanceOf[Map[String, String]]
56 for (f <- replaceFields) {
57 d += (f -> map.get(f).get.toString)
58 }
59 }
60 d
61 })
62
63 var arrKey: Array[String] = Array()
64 val rows: List[Row] = finalData.toList.map(map => {
65 arrKey = map.keySet.toArray
66 val values: Iterable[AnyRef] = map.values
67 val seq: Seq[AnyRef] = values.toSeq
68 val seqSTR: Seq[String] = values.toSeq.map(x=>x.toString)
69 val row: Row = Row.fromSeq(seqSTR)
70 row
71 })
72 val rowRDD: RDD[Row] = session.sparkContext.makeRDD(rows)
73
74 val fields: Array[StructField] = arrKey.map(d=>StructField(d,StringType,nullable = true))
75 val schema: StructType = StructType(fields)
76 val df: DataFrame = session.createDataFrame(rowRDD,schema)
77
78 out.write(df)
79 }
80
81
82 def getMcc(): MemCachedClient = {
83 val pool: SockIOPool = SockIOPool.getInstance()
84 var serversArr:Array[String]=servers.split(",")
85 pool.setServers(serversArr)
86
87 if(weights.length>0){
88 val weightsArr: Array[Integer] = "3".split(",").map(x=>{new Integer(x.toInt)})
89 pool.setWeights(weightsArr)
90 }
91 if(maxIdle.length>0){
92 pool.setMaxIdle(maxIdle.toInt)
93 }
94 if(maintSleep.length>0){
95 pool.setMaintSleep(maintSleep.toInt)
96 }
97 if(nagle.length>0){
98 pool.setNagle(nagle.toBoolean)
99 }
100 if(socketTO.length>0){
101 pool.setSocketTO(socketTO.toInt)
102 }
103 if(socketConnectTO.length>0){
104 pool.setSocketConnectTO(socketConnectTO.toInt)
105 }
106
107 pool.initialize()
108 val mcc: MemCachedClient = new MemCachedClient()
109 mcc
110 }
111
112 override def setProperties(map: Map[String, Any]): Unit = {
113 servers = MapUtil.get(map,"servers").asInstanceOf[String]
114 keyFile = MapUtil.get(map,"keyFile").asInstanceOf[String]
115 weights = MapUtil.get(map,"weights").asInstanceOf[String]
116 maxIdle = MapUtil.get(map,"maxIdle").asInstanceOf[String]
117 maintSleep = MapUtil.get(map,"maintSleep").asInstanceOf[String]
118 nagle = MapUtil.get(map,"nagle").asInstanceOf[String]
119 socketTO = MapUtil.get(map,"socketTO").asInstanceOf[String]
120 socketConnectTO = MapUtil.get(map,"socketConnectTO").asInstanceOf[String]
121 replaceField = MapUtil.get(map,"replaceField").asInstanceOf[String]
122
123 }
124
125 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
126 var descriptor : List[PropertyDescriptor] = List()
127
128 val servers=new PropertyDescriptor().name("servers").displayName("servers").description("Server address and port number,If you have multiple servers, use , segmentation.").defaultValue("").required(true)
129 descriptor = servers :: descriptor
130 val keyFile=new PropertyDescriptor().name("keyFile").displayName("keyFile").description("The field you want to use as a query condition").defaultValue("").required(true)
131 descriptor = keyFile :: descriptor
132 val weights=new PropertyDescriptor().name("weights").displayName("weights").description("Weight of each server,If you have multiple servers, use , segmentation.").defaultValue("").required(false)
133 descriptor = weights :: descriptor
134 val maxIdle=new PropertyDescriptor().name("maxIdle").displayName("maxIdle").description("Maximum processing time").defaultValue("").required(false)
135 descriptor = maxIdle :: descriptor
136 val maintSleep=new PropertyDescriptor().name("maintSleep").displayName("maintSleep").description("Main thread sleep time").defaultValue("").required(false)
137 descriptor = maintSleep :: descriptor
138 val nagle=new PropertyDescriptor().name("nagle").displayName("nagle").description("If the socket parameter is true, the data is not buffered and sent immediately.").defaultValue("").required(false)
139 descriptor = nagle :: descriptor
140 val socketTO=new PropertyDescriptor().name("socketTO").displayName("socketTO").description("Socket timeout during blocking").defaultValue("").required(false)
141 descriptor = socketTO :: descriptor
142 val socketConnectTO=new PropertyDescriptor().name("socketConnectTO").displayName("socketConnectTO").description("Timeout control during connection establishment").defaultValue("").required(false)
143 descriptor = socketConnectTO :: descriptor
144 val replaceField=new PropertyDescriptor().name("replaceField").displayName("replaceField").description("The field you want to replace . use , segmentation.").defaultValue("").required(true)
145 descriptor = replaceField :: descriptor
146
147 descriptor
148 }
149
150 override def getIcon(): Array[Byte] = {
151 ImageUtil.getImage("icon/memcache/ComplementByMemcache.png")
152 }
153
154 override def getGroup(): List[String] = {
155 List(StopGroup.Memcache.toString)
156 }
157
158 override def initialize(ctx: ProcessContext): Unit = {
159
160 }
161
162}