Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/graphx/LoadGraph.scala

1package cn.piflow.bundle.graphx
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.graphx.{GraphLoader, PartitionStrategy}
9class LoadGraph extends ConfigurableStop {
10
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Load data and construct a graphx"
13  val inportList: List[String] = List(Port.DefaultPort)
14
15
16  var edgePort : String = "edges"
17  var vertexPort : String = "vertex"
18  val outportList: List[String] = List(edgePort,vertexPort)
19
20
21  var dataPath:String = _
22
23  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24    val spark = pec.get[SparkSession]()
25    val sc=spark.sparkContext
26
27    import spark.sqlContext.implicits._
28    var graph=GraphLoader
29      .edgeListFile(sc,dataPath,true)
30      .partitionBy(PartitionStrategy.RandomVertexCut)
31    //TODO:can not transfer EdgeRdd to Dataset
32    out.write(edgePort,graph.edges.toDF())
33    out.write(vertexPort,graph.vertices.toDF())
34
35  }
36
37  def initialize(ctx: ProcessContext): Unit = {
38
39  }
40
41  def setProperties(map : Map[String, Any]): Unit = {
42    dataPath = MapUtil.get(map,"dataPath").asInstanceOf[String]
43  }
44
45  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46    var descriptor : List[PropertyDescriptor] = List()
47    val dataPath = new PropertyDescriptor()
48      .name("dataPath")
49      .displayName("Data_Path")
50      .defaultValue("")
51      .allowableValues(Set(""))
52      .required(true)
53      .example("hdfs://192.168.3.138:8020/work/test/test.csv")
54    descriptor = dataPath :: descriptor
55    descriptor
56  }
57
58  override def getIcon(): Array[Byte] = {
59    ImageUtil.getImage("icon/graphx/LoadGraph.png")
60  }
61
62  override def getGroup(): List[String] = {
63    List(StopGroup.GraphX.toString)
64  }
65
66}