Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/rdf/RdfToDF.scala

1package cn.piflow.bundle.rdf
2
3import java.util.regex.{Matcher, Pattern}
4
5import cn.piflow.bundle.util.Entity
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
9import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
10import org.apache.spark.rdd.RDD
11import org.apache.spark.sql.types.{DataTypes, StringType, StructField, StructType}
12import org.apache.spark.sql.{Row, SparkSession}
13
14class RdfToDF extends ConfigurableStop{
15  override val authorEmail: String = "xiaomeng7890@gmail.com"
16
17  override val description: String = "convert *.n3 RDF file to CSV(DataFrame) file"
18
19  var rdfFilepath : String = _
20  var isFront : String = _
21  var PRegex : String = _
22  var RRegex : String = _
23  var ERegex : String = _
24  var RSchema : String = _
25  var IDName : String = _
26  var LABELName : String = _
27
28  var entityPort : String = "entityOut"
29  var relationshipPort : String = "relationshipOut"
30
31  override def setProperties(map: Map[String, Any]): Unit = {
32    isFront = MapUtil.get(map, "isFromFront").asInstanceOf[String]
33    PRegex = MapUtil.get(map, "propertyRegex").asInstanceOf[String]
34    RRegex = MapUtil.get(map, "relationshipRegex").asInstanceOf[String]
35    ERegex = MapUtil.get(map, "entityRegex").asInstanceOf[String]
36    RSchema = MapUtil.get(map, "relationshipSchema").asInstanceOf[String]
37    IDName = MapUtil.get(map, "entityIdName").asInstanceOf[String]
38    LABELName = MapUtil.get(map, "entityLabelName").asInstanceOf[String]
39    if (isFront == "false")
40      rdfFilepath = MapUtil.get(map, "filePath").asInstanceOf[String]
41  }
42
43  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
44
45    var descriptor: List[PropertyDescriptor] = List()
46    val filePath = new PropertyDescriptor()
47      .name("filePath")
48      .displayName("inputHDFSFilePath")
49      .description("The path of the input rdf file")
50      .defaultValue("")
51      .required(true)
52
53
54    val isFromFront = new PropertyDescriptor()
55      .name("isFromFront")
56      .displayName("isFromFront")
57      .description("identify the file path source(should have same schema)")
58      .allowableValues(Set("true", "false"))
59      .defaultValue("false")
60      .required(true)
61
62    val propertyRegex = new PropertyDescriptor()
63      .name("propertyRegex")
64      .displayName("property regex")
65      .description("define the propertyRegex to parse the n3 file's property lines\r\n" +
66        "this regex string should be fully named and regular\r\n" +
67        "you need to SPECIFIC five value's name \r\n" +
68        "1.prefix 2.id 3.pprefix 4.name 5.value" +
69        "the form should be like this : \r\n" +
70        "(?<prefix>...?<id>... ?<pprefix>...?<name> ?<value>...)\r\n" +
71        "check the default value carefully to knowledge the right structure")
72      .defaultValue("<(?<prefix>http:\\/\\/[^>]+\\/)(?<id>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?<pprefix>http:\\/\\/[^>]+\\/)(?<name>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \"(?<value>.+)\" \\.")
73      .required(true)
74
75    val relationshipRegex = new PropertyDescriptor()
76      .name("relationshipRegex")
77      .displayName("relationship regex")
78      .description("define the propertyRegex to parse the n3 file's relationship lines\r\n" +
79        "this regex string should be fully named and regular\r\n" +
80        "you need to SPECIFIC six value's name \r\n" +
81        "1.prefix1 2.id1 3.tprefix 4.type 5.prefix2 6.id2" +
82        "the form should be like this : \r\n" +
83        "(?<prefix1>...?<id1>... ?<tprefix>...?<type> ?<prefix2>...?<id2>)\r\n" +
84        "check the default value carefully to knowledge the right structure")
85      .defaultValue("<(?<prefix1>http:\\/\\/[^>]+\\/)(?<id1>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?<tprefix>http:\\/\\/[^>]+\\/)(?<type>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)(?<!#type)> <(?<prefix2>http:\\/\\/[^>]+\\/)(?<id2>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \\.")
86      .required(true)
87
88    val entityRegex = new PropertyDescriptor()
89      .name("entityRegex")
90      .displayName("entity regex")
91      .description("define the propertyRegex to parse the n3 file's entity lines\r\n" +
92        "this regex string should be fully named and regular\r\n" +
93        "you need to SPECIFIC four value's name \r\n" +
94        "1.prefix 2.id 4.lprefix 5.label" +
95        "the form should be like this : \r\n" +
96        "(?<prefix>...?<id>... ... ?<lprefix>...?<label>)\r\n" +
97        "check the default value carefully to knowledge the right structure")
98      .defaultValue("(<(?<prefix>http:\\/\\/[^>]+\\/)(?<id>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?:http:\\/\\/[^>]+\\/)(?:[^\\/][-A-Za-z0-9._#$%^&*!@~]+)(?:#type)> <(?<lprefix>http:\\/\\/[^>]+\\/)(?<label>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \\.")
99      .required(true)
100
101    val relationshipSchema = new PropertyDescriptor()
102      .name("relationshipSchema")
103      .displayName("relationship's schema")
104      .description("define the schema of relationship, as a user, \r\n" +
105        "you should ponder the name of start id and end id\r\n" +
106        "make sure your schema looks like the default value")
107      .defaultValue("ENTITY_ID:START_ID,role,ENTITY_ID:END_ID,RELATION_TYPE:TYPE")
108      .required(true)
109
110    val entityIdName = new PropertyDescriptor()
111      .name("entityIdName")
112      .displayName("entity's id")
113      .description("define the id of entity, as a user, \r\n" +
114        "you should ponder the style like \'id\' + :ID\r\n" +
115        "make sure your schema looks like the default value")
116      .defaultValue("ENTITY_ID:ID")
117      .required(true)
118
119    val entityLabelName = new PropertyDescriptor()
120      .name("entityLabelName")
121      .displayName("entity's label")
122      .description("define the label of entity, as a user, \r\n" +
123        "you should ponder the style like \'label\' + :LABEL\r\n" +
124        "make sure your schema looks like the default value")
125      .defaultValue("ENTITY_TYPE:LABEL")
126      .required(true)
127
128    descriptor = filePath :: descriptor
129    descriptor = isFromFront :: descriptor
130    descriptor = propertyRegex :: descriptor
131    descriptor = entityRegex :: descriptor
132    descriptor = relationshipRegex :: descriptor
133    descriptor = relationshipSchema :: descriptor
134    descriptor = entityIdName :: descriptor
135    descriptor = entityLabelName :: descriptor
136    descriptor
137  }
138
139  override def getIcon(): Array[Byte] = {
140    ImageUtil.getImage("icon/rdf/RDF2DF.png")
141  }
142
143  override def getGroup(): List[String] = {
144    List(StopGroup.RDFGroup.toString)
145  }
146
147  override def initialize(ctx: ProcessContext): Unit = {
148
149  }
150
151  override def perform(in: JobInputStream,
152                       out: JobOutputStream,
153                       pec: JobContext): Unit = {
154
155    val entityRegexPattern : Pattern = Pattern.compile(ERegex)
156    val relationRegexPattern : Pattern = Pattern.compile(RRegex)
157    val propertyRegexPattern : Pattern = Pattern.compile(PRegex)
158    val spark = pec.get[SparkSession]()
159    val sc = spark.sparkContext
160    val sq = spark.sqlContext
161    var hdfsFile : RDD[String] = sc.emptyRDD[String]
162    //in
163    if (isFront == "true") {
164      val inDF : Array[String] = in
165        .read()
166        .collect()
167        .map(r => r.getAs[String](1))
168      var index = 0
169      val iterator : Iterator[String] = inDF.iterator
170
171      while(iterator.hasNext) {//every row's first col should be the exact hdfs path of the n3 file
172        index match {
173          case 0 => hdfsFile = sc.textFile(iterator.next())
174            index += 1
175          case 1000 =>
176            println(hdfsFile.count()) //in some case the num of file will excess 10w, use this way to reduce the depth of DAG
177            index = 1
178          case _ => hdfsFile = hdfsFile.union(sc.textFile(iterator.next))
179            index += 1
180        }
181      }
182    } else {
183      hdfsFile = sc.textFile(rdfFilepath)
184    }
185
186    val entityRdd = hdfsFile
187      .filter(s => entityRegexPattern.matcher(s).find() ||
188        propertyRegexPattern.matcher(s).find())
189
190    val relationshipRdd = hdfsFile
191      .filter(s => relationRegexPattern.matcher(s).find())
192
193    val settleUpEntityRdd = entityRdd.map(s => {
194      val me = entityRegexPattern.matcher(s)
195      val mp = propertyRegexPattern.matcher(s)
196      if (me.find()) {
197        (me.group("id"), me.group())
198      } else {
199        mp.find()
200        (mp.group("id"), mp.group())
201      }
202    })
203      .groupByKey() //main
204      .values
205      .map(s => s.toList)
206
207    val entitySchema : Set[String] = settleUpEntityRdd
208      .map(s => s.filter(l => propertyRegexPattern.matcher(l).find()))
209      .map(s => {
210        s.map(line => {
211          val m = propertyRegexPattern.matcher(line)
212          m.find()
213          m.group("name")
214        })
215      })
216      .map(l => {
217        l.toSet
218      })
219      .reduce(_ ++ _)
220
221
222    val finalEntitySchema = IDName + "," +
223      entitySchema.reduce((a, b) => a + "," + b) + "," + LABELName
224
225
226    val entityObjectRdd = settleUpEntityRdd.map(l => {
227      val en = l.filter(s => entityRegexPattern.matcher(s).find()).head
228      val label = get(entityRegexPattern.matcher(en),"label")
229      val id = get(entityRegexPattern.matcher(en),"id")
230      val prop = l
231        .filter(s => ?(propertyRegexPattern, s))
232        .map(s => (
233          get(propertyRegexPattern.matcher(s),"name")
234            ->
235            get(propertyRegexPattern.matcher(s),"value").replace("\"", " ")
236          )
237        )
238        .groupBy(i => i._1)
239        .map(f => (f._1, f._2.map(_._2).toArray))
240        .toArray
241        .toMap
242
243      new Entity(id, label, prop, entitySchema.toSeq)
244    })
245
246    val entityRowRdd = entityObjectRdd.map(_.getEntityRow)
247
248    val sampleEntity = entityObjectRdd.first()
249
250    val relationshipRowRdd = relationshipRdd.map(s => Seq(
251      get(relationRegexPattern.matcher(s),"id1") ,
252      get(relationRegexPattern.matcher(s),"type") ,
253      get(relationRegexPattern.matcher(s),"id2") ,
254      get(relationRegexPattern.matcher(s),"type")
255    )
256    ).map(s => Row(s))
257
258
259
260    var sampleSchema : Seq[String] = sampleEntity.schema
261    sampleSchema +:= IDName
262    sampleSchema :+= LABELName
263
264    val entityDFSList: Seq[StructField] = for {
265      s <- sampleSchema
266      p <- sampleEntity.propSeq
267    } yield {
268      p match {
269        case _: String => StructField(s, StringType, nullable = true)
270        case _: Array[String] => StructField(s,
271          DataTypes.createArrayType(StringType),
272          nullable = true)
273        case _ => StructField(s, StringType, nullable = true)
274      }
275    }
276
277    val relationshipDFSchema : StructType = StructType(RSchema.split(",").map(x => x.trim)
278      .map(i => StructField(i, StringType, nullable = true)))
279
280    val entityDFSchema : StructType = StructType(entityDFSList)
281
282    val entityDF = sq.createDataFrame(entityRowRdd, entityDFSchema)
283    val relationDF = sq.createDataFrame(relationshipRowRdd, relationshipDFSchema)
284    //entityDF.show(1)
285    //relationDF.show(1)
286    out.write(entityPort, entityDF)
287    out.write(relationshipPort, relationDF)
288  }
289
290  def get(m : Matcher, name : String) : String = {
291    if (m.find()) m.group(name)
292    else ""
293  }
294
295  def ? (regex : Pattern, str : String) : Boolean = {
296    regex.matcher(str).find()
297  }
298
299  override val inportList: List[String] = List(Port.DefaultPort.toString)
300  override val outportList: List[String] = List(entityPort, relationshipPort)
301}