upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/graphx/LoadGraph.scala
1package cn.piflow.bundle.graphx
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.graphx.{GraphLoader, PartitionStrategy}
9class LoadGraph extends ConfigurableStop {
10
11 val authorEmail: String = "06whuxx@163.com"
12 val description: String = "Load data and construct a graphx"
13 val inportList: List[String] = List(Port.DefaultPort)
14
15
16 var edgePort : String = "edges"
17 var vertexPort : String = "vertex"
18 val outportList: List[String] = List(edgePort,vertexPort)
19
20
21 var dataPath:String = _
22
23 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24 val spark = pec.get[SparkSession]()
25 val sc=spark.sparkContext
26
27 import spark.sqlContext.implicits._
28 var graph=GraphLoader
29 .edgeListFile(sc,dataPath,true)
30 .partitionBy(PartitionStrategy.RandomVertexCut)
31 //TODO:can not transfer EdgeRdd to Dataset
32 out.write(edgePort,graph.edges.toDF())
33 out.write(vertexPort,graph.vertices.toDF())
34
35 }
36
37 def initialize(ctx: ProcessContext): Unit = {
38
39 }
40
41 def setProperties(map : Map[String, Any]): Unit = {
42 dataPath = MapUtil.get(map,"dataPath").asInstanceOf[String]
43 }
44
45 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46 var descriptor : List[PropertyDescriptor] = List()
47 val dataPath = new PropertyDescriptor()
48 .name("dataPath")
49 .displayName("Data_Path")
50 .defaultValue("")
51 .allowableValues(Set(""))
52 .required(true)
53 .example("hdfs://192.168.3.138:8020/work/test/test.csv")
54 descriptor = dataPath :: descriptor
55 descriptor
56 }
57
58 override def getIcon(): Array[Byte] = {
59 ImageUtil.getImage("icon/graphx/LoadGraph.png")
60 }
61
62 override def getGroup(): List[String] = {
63 List(StopGroup.GraphX.toString)
64 }
65
66}