Source code

Download source archive ↓

src/Csv.scala

1package xstream.examples.piflow
2
3import java.io.{InputStreamReader,OutputStreamWriter}
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import com.fasterxml.jackson.databind.JsonNode
6import org.apache.arrow.vector.types.pojo.{ArrowType,Schema}
7import org.apache.commons.csv.{CSVFormat,CSVPrinter}
8import xstream.execution.{StreamDescriptor,InferenceResult}
9import scala.jdk.CollectionConverters._
10
11private[piflow] object Csv {
12  import Support._
13  def format(p:JsonNode):CSVFormat={
14    val delimiter=text(p,"delimiter",",");require(delimiter.length==1,"delimiter must be one character")
15    CSVFormat.DEFAULT.builder().setDelimiter(delimiter.charAt(0)).setIgnoreEmptyLines(false).setRecordSeparator("\n").get()
16  }
17  def validate(p:JsonNode,parser:Boolean):Unit={params(p,Set.empty,if(parser)Set("header","delimiter","schema")else Set("header","delimiter"));bool(p,"header",true);format(p);if(parser)text(p,"schema","")}
18}
19class CsvParserOperator extends StreamOperator {
20  import Support._
21  override protected def bytesInput=true
22  protected def validate(p:JsonNode):Unit=Csv.validate(p,true)
23  private def read(in:java.io.InputStream,p:JsonNode)(consume:(Schema,Iterator[Vector[AnyRef]])=>Unit):Unit={
24    val decoder=StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT)
25    val reader=new InputStreamReader(in,decoder);val parser=Csv.format(p).parse(reader)
26    try {
27      val rows=parser.iterator().asScala
28      val first=if(rows.hasNext)Some(rows.next().asScala.toVector)else None
29      val declared=text(p,"schema","");val header=bool(p,"header",true)
30      val ns=if(header)first.getOrElse(columns(declared))else if(declared.nonEmpty)columns(declared)else first.toVector.flatMap(_.indices.map(i=>"_"+i))
31      require(ns.nonEmpty && ns.forall(_.nonEmpty) && ns.distinct.size==ns.size,"CSV requires unique, nonempty columns")
32      val s=schema(ns.map(n=>field(n,ArrowType.Utf8.INSTANCE)))
33      val values=(if(header)Iterator.empty else first.iterator)++rows.map(_.asScala.toVector)
34      consume(s,values.map{r=>check();require(r.size==ns.size,"CSV row width differs from schema");r.map(_.asInstanceOf[AnyRef])})
35    }finally parser.close()
36  }
37  protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={
38    val declared=text(p,"schema","")
39    if(!bool(p,"header",true) && declared.nonEmpty)descriptor(Some(schema(columns(declared).map(n=>field(n,ArrowType.Utf8.INSTANCE)))))
40    else inputs("source").sample match {
41      case Some(s) if s.data.nonEmpty=>
42        var result:Schema=null
43        read(new java.io.ByteArrayInputStream(s.data.toArray),p){(schema,_)=>result=schema}
44        descriptor(Option(result))
45      case _=>descriptor(None)
46    }
47  }
48  private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit=read(inputs("source").bytes,p){(s,rows)=>emitRows(s,rows,out.records(s))}
49}
50object CsvParser extends CsvParserOperator
51class CsvSaveOperator extends StreamOperator {
52  import Support._
53  protected def validate(p:JsonNode):Unit=Csv.validate(p,false)
54  protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={inputs("source").schema.foreach(s=>fields(s).foreach(scalar));InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set("csv"),certainty="known")))}
55  private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
56    val in=inputs("source");val s=in.schema;fields(s).foreach(scalar)
57    val writer=new OutputStreamWriter(out.bytes(),StandardCharsets.UTF_8);val printer=new CSVPrinter(writer,Csv.format(p))
58    if(bool(p,"header",true))printer.printRecord(names(s).asJava)
59    in.records(r=>{rows(r).foreach(row=>printer.printRecord(row.map{case null=>"";case b:java.lang.Boolean=>if(b)"True"else "False";case x=>x}.asJava));printer.flush()})
60    printer.flush()
61  }
62}
63object CsvSave extends CsvSaveOperator