Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/memcached/ComplementByMemcache.scala

1package cn.piflow.bundle.memcached
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import com.danga.MemCached.{MemCachedClient, SockIOPool}
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12import scala.collection.mutable
13
14class ComplementByMemcache extends ConfigurableStop {
15  override val authorEmail: String = "yangqidong@cnic.cn"
16  override val description: String = "Complement by Memcache"
17  val inportList: List[String] = List(PortEnum.DefaultPort.toString)
18  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
19
20  var servers:String=_            //Server address and port number,If you have multiple servers, use "," segmentation.
21  var keyFile:String=_            //The field you want to use as a query condition
22  var weights:String=_            //Weight of each server
23  var maxIdle:String=_            //Maximum processing time
24  var maintSleep:String=_         //Main thread sleep time
25  var nagle:String=_              //If the socket parameter is true, the data is not buffered and sent immediately.
26  var socketTO:String=_           //Socket timeout during blocking
27  var socketConnectTO:String=_    //Timeout control during connection establishment
28  var replaceField:String=_            //The fields you want to get
29
30
31  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
32    val session: SparkSession = pec.get[SparkSession]()
33    val inDF: DataFrame = in.read()
34
35    val mcc: MemCachedClient =getMcc()
36
37    val replaceFields: mutable.Buffer[String] = replaceField.split(",").toBuffer
38
39    val rowArr: Array[Row] = inDF.collect()
40    val fileNames: Array[String] = inDF.columns
41    val data: Array[Map[String, String]] = rowArr.map(row => {
42      var rowStr: String = row.toString().substring(1,row.toString().length-1)
43      val dataARR: Array[String] = rowStr.split(",")
44      var map: Map[String, String] = Map()
45      for (x <- (0 until fileNames.size)) {
46        map += (fileNames(x) -> dataARR(x))
47      }
48      map
49    })
50
51    val finalData: Array[Map[String, String]] = data.map(eachData => {
52      var d: Map[String, String] = eachData
53      val anyRef: AnyRef = mcc.get(d.get(keyFile).get)
54      if(anyRef.getClass.toString.equals("class scala.Some")){
55        val map: Map[String, String] = anyRef.asInstanceOf[Map[String, String]]
56        for (f <- replaceFields) {
57          d += (f -> map.get(f).get.toString)
58        }
59      }
60      d
61    })
62
63    var arrKey: Array[String] = Array()
64    val rows: List[Row] = finalData.toList.map(map => {
65      arrKey = map.keySet.toArray
66      val values: Iterable[AnyRef] = map.values
67      val seq: Seq[AnyRef] = values.toSeq
68      val seqSTR: Seq[String] = values.toSeq.map(x=>x.toString)
69      val row: Row = Row.fromSeq(seqSTR)
70      row
71    })
72    val rowRDD: RDD[Row] = session.sparkContext.makeRDD(rows)
73
74    val fields: Array[StructField] = arrKey.map(d=>StructField(d,StringType,nullable = true))
75    val schema: StructType = StructType(fields)
76    val df: DataFrame = session.createDataFrame(rowRDD,schema)
77
78    out.write(df)
79  }
80
81
82  def getMcc(): MemCachedClient = {
83    val pool: SockIOPool = SockIOPool.getInstance()
84    var serversArr:Array[String]=servers.split(",")
85    pool.setServers(serversArr)
86
87    if(weights.length>0){
88      val weightsArr: Array[Integer] = "3".split(",").map(x=>{new Integer(x.toInt)})
89      pool.setWeights(weightsArr)
90    }
91    if(maxIdle.length>0){
92      pool.setMaxIdle(maxIdle.toInt)
93    }
94    if(maintSleep.length>0){
95      pool.setMaintSleep(maintSleep.toInt)
96    }
97    if(nagle.length>0){
98      pool.setNagle(nagle.toBoolean)
99    }
100    if(socketTO.length>0){
101      pool.setSocketTO(socketTO.toInt)
102    }
103    if(socketConnectTO.length>0){
104      pool.setSocketConnectTO(socketConnectTO.toInt)
105    }
106
107    pool.initialize()
108    val mcc: MemCachedClient = new MemCachedClient()
109    mcc
110  }
111
112  override def setProperties(map: Map[String, Any]): Unit = {
113    servers = MapUtil.get(map,"servers").asInstanceOf[String]
114    keyFile = MapUtil.get(map,"keyFile").asInstanceOf[String]
115    weights = MapUtil.get(map,"weights").asInstanceOf[String]
116    maxIdle = MapUtil.get(map,"maxIdle").asInstanceOf[String]
117    maintSleep = MapUtil.get(map,"maintSleep").asInstanceOf[String]
118    nagle = MapUtil.get(map,"nagle").asInstanceOf[String]
119    socketTO = MapUtil.get(map,"socketTO").asInstanceOf[String]
120    socketConnectTO = MapUtil.get(map,"socketConnectTO").asInstanceOf[String]
121    replaceField = MapUtil.get(map,"replaceField").asInstanceOf[String]
122
123  }
124
125  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
126    var descriptor : List[PropertyDescriptor] = List()
127
128    val servers=new PropertyDescriptor().name("servers").displayName("servers").description("Server address and port number,If you have multiple servers, use , segmentation.").defaultValue("").required(true)
129    descriptor = servers :: descriptor
130    val keyFile=new PropertyDescriptor().name("keyFile").displayName("keyFile").description("The field you want to use as a query condition").defaultValue("").required(true)
131    descriptor = keyFile :: descriptor
132    val weights=new PropertyDescriptor().name("weights").displayName("weights").description("Weight of each server,If you have multiple servers, use , segmentation.").defaultValue("").required(false)
133    descriptor = weights :: descriptor
134    val maxIdle=new PropertyDescriptor().name("maxIdle").displayName("maxIdle").description("Maximum processing time").defaultValue("").required(false)
135    descriptor = maxIdle :: descriptor
136    val maintSleep=new PropertyDescriptor().name("maintSleep").displayName("maintSleep").description("Main thread sleep time").defaultValue("").required(false)
137    descriptor = maintSleep :: descriptor
138    val nagle=new PropertyDescriptor().name("nagle").displayName("nagle").description("If the socket parameter is true, the data is not buffered and sent immediately.").defaultValue("").required(false)
139    descriptor = nagle :: descriptor
140    val socketTO=new PropertyDescriptor().name("socketTO").displayName("socketTO").description("Socket timeout during blocking").defaultValue("").required(false)
141    descriptor = socketTO :: descriptor
142    val socketConnectTO=new PropertyDescriptor().name("socketConnectTO").displayName("socketConnectTO").description("Timeout control during connection establishment").defaultValue("").required(false)
143    descriptor = socketConnectTO :: descriptor
144    val replaceField=new PropertyDescriptor().name("replaceField").displayName("replaceField").description("The field you want to replace .  use , segmentation.").defaultValue("").required(true)
145    descriptor = replaceField :: descriptor
146
147    descriptor
148  }
149
150  override def getIcon(): Array[Byte] = {
151    ImageUtil.getImage("icon/memcache/ComplementByMemcache.png")
152  }
153
154  override def getGroup(): List[String] = {
155    List(StopGroup.Memcache.toString)
156  }
157
158  override def initialize(ctx: ProcessContext): Unit = {
159
160  }
161
162}