upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/rdf/RdfToDF.scala
1package cn.piflow.bundle.rdf
2
3import java.util.regex.{Matcher, Pattern}
4
5import cn.piflow.bundle.util.Entity
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
9import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
10import org.apache.spark.rdd.RDD
11import org.apache.spark.sql.types.{DataTypes, StringType, StructField, StructType}
12import org.apache.spark.sql.{Row, SparkSession}
13
14class RdfToDF extends ConfigurableStop{
15 override val authorEmail: String = "xiaomeng7890@gmail.com"
16
17 override val description: String = "convert *.n3 RDF file to CSV(DataFrame) file"
18
19 var rdfFilepath : String = _
20 var isFront : String = _
21 var PRegex : String = _
22 var RRegex : String = _
23 var ERegex : String = _
24 var RSchema : String = _
25 var IDName : String = _
26 var LABELName : String = _
27
28 var entityPort : String = "entityOut"
29 var relationshipPort : String = "relationshipOut"
30
31 override def setProperties(map: Map[String, Any]): Unit = {
32 isFront = MapUtil.get(map, "isFromFront").asInstanceOf[String]
33 PRegex = MapUtil.get(map, "propertyRegex").asInstanceOf[String]
34 RRegex = MapUtil.get(map, "relationshipRegex").asInstanceOf[String]
35 ERegex = MapUtil.get(map, "entityRegex").asInstanceOf[String]
36 RSchema = MapUtil.get(map, "relationshipSchema").asInstanceOf[String]
37 IDName = MapUtil.get(map, "entityIdName").asInstanceOf[String]
38 LABELName = MapUtil.get(map, "entityLabelName").asInstanceOf[String]
39 if (isFront == "false")
40 rdfFilepath = MapUtil.get(map, "filePath").asInstanceOf[String]
41 }
42
43 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
44
45 var descriptor: List[PropertyDescriptor] = List()
46 val filePath = new PropertyDescriptor()
47 .name("filePath")
48 .displayName("inputHDFSFilePath")
49 .description("The path of the input rdf file")
50 .defaultValue("")
51 .required(true)
52
53
54 val isFromFront = new PropertyDescriptor()
55 .name("isFromFront")
56 .displayName("isFromFront")
57 .description("identify the file path source(should have same schema)")
58 .allowableValues(Set("true", "false"))
59 .defaultValue("false")
60 .required(true)
61
62 val propertyRegex = new PropertyDescriptor()
63 .name("propertyRegex")
64 .displayName("property regex")
65 .description("define the propertyRegex to parse the n3 file's property lines\r\n" +
66 "this regex string should be fully named and regular\r\n" +
67 "you need to SPECIFIC five value's name \r\n" +
68 "1.prefix 2.id 3.pprefix 4.name 5.value" +
69 "the form should be like this : \r\n" +
70 "(?<prefix>...?<id>... ?<pprefix>...?<name> ?<value>...)\r\n" +
71 "check the default value carefully to knowledge the right structure")
72 .defaultValue("<(?<prefix>http:\\/\\/[^>]+\\/)(?<id>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?<pprefix>http:\\/\\/[^>]+\\/)(?<name>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \"(?<value>.+)\" \\.")
73 .required(true)
74
75 val relationshipRegex = new PropertyDescriptor()
76 .name("relationshipRegex")
77 .displayName("relationship regex")
78 .description("define the propertyRegex to parse the n3 file's relationship lines\r\n" +
79 "this regex string should be fully named and regular\r\n" +
80 "you need to SPECIFIC six value's name \r\n" +
81 "1.prefix1 2.id1 3.tprefix 4.type 5.prefix2 6.id2" +
82 "the form should be like this : \r\n" +
83 "(?<prefix1>...?<id1>... ?<tprefix>...?<type> ?<prefix2>...?<id2>)\r\n" +
84 "check the default value carefully to knowledge the right structure")
85 .defaultValue("<(?<prefix1>http:\\/\\/[^>]+\\/)(?<id1>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?<tprefix>http:\\/\\/[^>]+\\/)(?<type>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)(?<!#type)> <(?<prefix2>http:\\/\\/[^>]+\\/)(?<id2>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \\.")
86 .required(true)
87
88 val entityRegex = new PropertyDescriptor()
89 .name("entityRegex")
90 .displayName("entity regex")
91 .description("define the propertyRegex to parse the n3 file's entity lines\r\n" +
92 "this regex string should be fully named and regular\r\n" +
93 "you need to SPECIFIC four value's name \r\n" +
94 "1.prefix 2.id 4.lprefix 5.label" +
95 "the form should be like this : \r\n" +
96 "(?<prefix>...?<id>... ... ?<lprefix>...?<label>)\r\n" +
97 "check the default value carefully to knowledge the right structure")
98 .defaultValue("(<(?<prefix>http:\\/\\/[^>]+\\/)(?<id>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> <(?:http:\\/\\/[^>]+\\/)(?:[^\\/][-A-Za-z0-9._#$%^&*!@~]+)(?:#type)> <(?<lprefix>http:\\/\\/[^>]+\\/)(?<label>[^\\/][-A-Za-z0-9._#$%^&*!@~]+)> \\.")
99 .required(true)
100
101 val relationshipSchema = new PropertyDescriptor()
102 .name("relationshipSchema")
103 .displayName("relationship's schema")
104 .description("define the schema of relationship, as a user, \r\n" +
105 "you should ponder the name of start id and end id\r\n" +
106 "make sure your schema looks like the default value")
107 .defaultValue("ENTITY_ID:START_ID,role,ENTITY_ID:END_ID,RELATION_TYPE:TYPE")
108 .required(true)
109
110 val entityIdName = new PropertyDescriptor()
111 .name("entityIdName")
112 .displayName("entity's id")
113 .description("define the id of entity, as a user, \r\n" +
114 "you should ponder the style like \'id\' + :ID\r\n" +
115 "make sure your schema looks like the default value")
116 .defaultValue("ENTITY_ID:ID")
117 .required(true)
118
119 val entityLabelName = new PropertyDescriptor()
120 .name("entityLabelName")
121 .displayName("entity's label")
122 .description("define the label of entity, as a user, \r\n" +
123 "you should ponder the style like \'label\' + :LABEL\r\n" +
124 "make sure your schema looks like the default value")
125 .defaultValue("ENTITY_TYPE:LABEL")
126 .required(true)
127
128 descriptor = filePath :: descriptor
129 descriptor = isFromFront :: descriptor
130 descriptor = propertyRegex :: descriptor
131 descriptor = entityRegex :: descriptor
132 descriptor = relationshipRegex :: descriptor
133 descriptor = relationshipSchema :: descriptor
134 descriptor = entityIdName :: descriptor
135 descriptor = entityLabelName :: descriptor
136 descriptor
137 }
138
139 override def getIcon(): Array[Byte] = {
140 ImageUtil.getImage("icon/rdf/RDF2DF.png")
141 }
142
143 override def getGroup(): List[String] = {
144 List(StopGroup.RDFGroup.toString)
145 }
146
147 override def initialize(ctx: ProcessContext): Unit = {
148
149 }
150
151 override def perform(in: JobInputStream,
152 out: JobOutputStream,
153 pec: JobContext): Unit = {
154
155 val entityRegexPattern : Pattern = Pattern.compile(ERegex)
156 val relationRegexPattern : Pattern = Pattern.compile(RRegex)
157 val propertyRegexPattern : Pattern = Pattern.compile(PRegex)
158 val spark = pec.get[SparkSession]()
159 val sc = spark.sparkContext
160 val sq = spark.sqlContext
161 var hdfsFile : RDD[String] = sc.emptyRDD[String]
162 //in
163 if (isFront == "true") {
164 val inDF : Array[String] = in
165 .read()
166 .collect()
167 .map(r => r.getAs[String](1))
168 var index = 0
169 val iterator : Iterator[String] = inDF.iterator
170
171 while(iterator.hasNext) {//every row's first col should be the exact hdfs path of the n3 file
172 index match {
173 case 0 => hdfsFile = sc.textFile(iterator.next())
174 index += 1
175 case 1000 =>
176 println(hdfsFile.count()) //in some case the num of file will excess 10w, use this way to reduce the depth of DAG
177 index = 1
178 case _ => hdfsFile = hdfsFile.union(sc.textFile(iterator.next))
179 index += 1
180 }
181 }
182 } else {
183 hdfsFile = sc.textFile(rdfFilepath)
184 }
185
186 val entityRdd = hdfsFile
187 .filter(s => entityRegexPattern.matcher(s).find() ||
188 propertyRegexPattern.matcher(s).find())
189
190 val relationshipRdd = hdfsFile
191 .filter(s => relationRegexPattern.matcher(s).find())
192
193 val settleUpEntityRdd = entityRdd.map(s => {
194 val me = entityRegexPattern.matcher(s)
195 val mp = propertyRegexPattern.matcher(s)
196 if (me.find()) {
197 (me.group("id"), me.group())
198 } else {
199 mp.find()
200 (mp.group("id"), mp.group())
201 }
202 })
203 .groupByKey() //main
204 .values
205 .map(s => s.toList)
206
207 val entitySchema : Set[String] = settleUpEntityRdd
208 .map(s => s.filter(l => propertyRegexPattern.matcher(l).find()))
209 .map(s => {
210 s.map(line => {
211 val m = propertyRegexPattern.matcher(line)
212 m.find()
213 m.group("name")
214 })
215 })
216 .map(l => {
217 l.toSet
218 })
219 .reduce(_ ++ _)
220
221
222 val finalEntitySchema = IDName + "," +
223 entitySchema.reduce((a, b) => a + "," + b) + "," + LABELName
224
225
226 val entityObjectRdd = settleUpEntityRdd.map(l => {
227 val en = l.filter(s => entityRegexPattern.matcher(s).find()).head
228 val label = get(entityRegexPattern.matcher(en),"label")
229 val id = get(entityRegexPattern.matcher(en),"id")
230 val prop = l
231 .filter(s => ?(propertyRegexPattern, s))
232 .map(s => (
233 get(propertyRegexPattern.matcher(s),"name")
234 ->
235 get(propertyRegexPattern.matcher(s),"value").replace("\"", " ")
236 )
237 )
238 .groupBy(i => i._1)
239 .map(f => (f._1, f._2.map(_._2).toArray))
240 .toArray
241 .toMap
242
243 new Entity(id, label, prop, entitySchema.toSeq)
244 })
245
246 val entityRowRdd = entityObjectRdd.map(_.getEntityRow)
247
248 val sampleEntity = entityObjectRdd.first()
249
250 val relationshipRowRdd = relationshipRdd.map(s => Seq(
251 get(relationRegexPattern.matcher(s),"id1") ,
252 get(relationRegexPattern.matcher(s),"type") ,
253 get(relationRegexPattern.matcher(s),"id2") ,
254 get(relationRegexPattern.matcher(s),"type")
255 )
256 ).map(s => Row(s))
257
258
259
260 var sampleSchema : Seq[String] = sampleEntity.schema
261 sampleSchema +:= IDName
262 sampleSchema :+= LABELName
263
264 val entityDFSList: Seq[StructField] = for {
265 s <- sampleSchema
266 p <- sampleEntity.propSeq
267 } yield {
268 p match {
269 case _: String => StructField(s, StringType, nullable = true)
270 case _: Array[String] => StructField(s,
271 DataTypes.createArrayType(StringType),
272 nullable = true)
273 case _ => StructField(s, StringType, nullable = true)
274 }
275 }
276
277 val relationshipDFSchema : StructType = StructType(RSchema.split(",").map(x => x.trim)
278 .map(i => StructField(i, StringType, nullable = true)))
279
280 val entityDFSchema : StructType = StructType(entityDFSList)
281
282 val entityDF = sq.createDataFrame(entityRowRdd, entityDFSchema)
283 val relationDF = sq.createDataFrame(relationshipRowRdd, relationshipDFSchema)
284 //entityDF.show(1)
285 //relationDF.show(1)
286 out.write(entityPort, entityDF)
287 out.write(relationshipPort, relationDF)
288 }
289
290 def get(m : Matcher, name : String) : String = {
291 if (m.find()) m.group(name)
292 else ""
293 }
294
295 def ? (regex : Pattern, str : String) : Boolean = {
296 regex.matcher(str).find()
297 }
298
299 override val inportList: List[String] = List(Port.DefaultPort.toString)
300 override val outportList: List[String] = List(entityPort, relationshipPort)
301}