src/Csv.scala
1package xstream.examples.piflow
2
3import java.io.{InputStreamReader,OutputStreamWriter}
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import com.fasterxml.jackson.databind.JsonNode
6import org.apache.arrow.vector.types.pojo.{ArrowType,Schema}
7import org.apache.commons.csv.{CSVFormat,CSVPrinter}
8import xstream.execution.{StreamDescriptor,InferenceResult}
9import scala.jdk.CollectionConverters._
10
11private[piflow] object Csv {
12 import Support._
13 def format(p:JsonNode):CSVFormat={
14 val delimiter=text(p,"delimiter",",");require(delimiter.length==1,"delimiter must be one character")
15 CSVFormat.DEFAULT.builder().setDelimiter(delimiter.charAt(0)).setIgnoreEmptyLines(false).setRecordSeparator("\n").get()
16 }
17 def validate(p:JsonNode,parser:Boolean):Unit={params(p,Set.empty,if(parser)Set("header","delimiter","schema")else Set("header","delimiter"));bool(p,"header",true);format(p);if(parser)text(p,"schema","")}
18}
19class CsvParserOperator extends StreamOperator {
20 import Support._
21 override protected def bytesInput=true
22 protected def validate(p:JsonNode):Unit=Csv.validate(p,true)
23 private def read(in:java.io.InputStream,p:JsonNode)(consume:(Schema,Iterator[Vector[AnyRef]])=>Unit):Unit={
24 val decoder=StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT)
25 val reader=new InputStreamReader(in,decoder);val parser=Csv.format(p).parse(reader)
26 try {
27 val rows=parser.iterator().asScala
28 val first=if(rows.hasNext)Some(rows.next().asScala.toVector)else None
29 val declared=text(p,"schema","");val header=bool(p,"header",true)
30 val ns=if(header)first.getOrElse(columns(declared))else if(declared.nonEmpty)columns(declared)else first.toVector.flatMap(_.indices.map(i=>"_"+i))
31 require(ns.nonEmpty && ns.forall(_.nonEmpty) && ns.distinct.size==ns.size,"CSV requires unique, nonempty columns")
32 val s=schema(ns.map(n=>field(n,ArrowType.Utf8.INSTANCE)))
33 val values=(if(header)Iterator.empty else first.iterator)++rows.map(_.asScala.toVector)
34 consume(s,values.map{r=>check();require(r.size==ns.size,"CSV row width differs from schema");r.map(_.asInstanceOf[AnyRef])})
35 }finally parser.close()
36 }
37 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={
38 val declared=text(p,"schema","")
39 if(!bool(p,"header",true) && declared.nonEmpty)descriptor(Some(schema(columns(declared).map(n=>field(n,ArrowType.Utf8.INSTANCE)))))
40 else inputs("source").sample match {
41 case Some(s) if s.data.nonEmpty=>
42 var result:Schema=null
43 read(new java.io.ByteArrayInputStream(s.data.toArray),p){(schema,_)=>result=schema}
44 descriptor(Option(result))
45 case _=>descriptor(None)
46 }
47 }
48 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit=read(inputs("source").bytes,p){(s,rows)=>emitRows(s,rows,out.records(s))}
49}
50object CsvParser extends CsvParserOperator
51class CsvSaveOperator extends StreamOperator {
52 import Support._
53 protected def validate(p:JsonNode):Unit=Csv.validate(p,false)
54 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={inputs("source").schema.foreach(s=>fields(s).foreach(scalar));InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set("csv"),certainty="known")))}
55 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
56 val in=inputs("source");val s=in.schema;fields(s).foreach(scalar)
57 val writer=new OutputStreamWriter(out.bytes(),StandardCharsets.UTF_8);val printer=new CSVPrinter(writer,Csv.format(p))
58 if(bool(p,"header",true))printer.printRecord(names(s).asJava)
59 in.records(r=>{rows(r).foreach(row=>printer.printRecord(row.map{case null=>"";case b:java.lang.Boolean=>if(b)"True"else "False";case x=>x}.asJava));printer.flush()})
60 printer.flush()
61 }
62}
63object CsvSave extends CsvSaveOperator