src/Galaxy.scala
1package xstream.galaxy
2
3import java.io.{InputStream,OutputStream,InputStreamReader,OutputStreamWriter,BufferedReader,BufferedWriter}
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.Locale
6import java.util.concurrent.CancellationException
7import com.fasterxml.jackson.databind.JsonNode
8import scala.jdk.CollectionConverters._
9import xstream.operator.{Operator,ByteStreamOperator,OperatorContext}
10import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
11
12/** Galaxy text tools adapted to bounded, cancellable UTF-8 streams. See NOTICE. */
13abstract class GalaxyOperator(val operation:String) extends Operator {
14 private val delimiters=Map("tab"->"\t","comma"->",","dash"->"-","underscore"->"_","pipe"->"\\|","dot"->"\\.","whitespace"->"(?U)\\s+","space"->" ","colon"->":")
15 private def text(p:JsonNode,key:String,default:String):String={
16 val n=p.get(key);require(n==null || n.isTextual,key+" must be a string")
17 val s=if(n==null)default else n.asText;require(s.length<=4096,key+" is too long");s
18 }
19 private def columns(p:JsonNode):Vector[Int]={
20 val s=text(p,"columns","c1").replaceAll("\\s+","")
21 val result=s.split(",",-1).toVector.flatMap{c=>
22 require(c.matches("(?i)c[1-9][0-9]{0,5}(-c[1-9][0-9]{0,5})?"),"columns must use c1,c2 or c2-c4")
23 val ends=c.toLowerCase(Locale.ROOT).split("-c").map(_.stripPrefix("c").toInt)
24 val end=ends.last;require(end>=ends.head && end-ends.head<1024,"invalid or excessive column range")
25 (ends.head to end).map(_-1)
26 }
27 require(result.nonEmpty && result.size<=1024,"select between 1 and 1024 columns");result
28 }
29 private def allowed:Set[String]=operation match {
30 case "change_case"=>Set("columns","delimiter","casing")
31 case "cut_columns"=>Set("columns","delimiter")
32 case "merge_columns"=>Set("columns")
33 case "add_column"=>Set("value","iterate")
34 case _=>Set("character")
35 }
36 def validate(p:JsonNode):Unit={
37 require(p!=null && p.isObject,"parameters must be an object")
38 require(p.fieldNames.asScala.forall(allowed),"unknown parameter")
39 if(Set("change_case","cut_columns","merge_columns")(operation))columns(p)
40 if(Set("change_case","cut_columns")(operation))require(Set("tab","comma","dash","underscore","pipe","dot","whitespace")(text(p,"delimiter","tab")),"unsupported delimiter")
41 if(operation=="change_case")require(Set("upper","lower")(text(p,"casing","upper")),"casing must be upper or lower")
42 if(Set("condense_characters","convert_delimiters")(operation))require(delimiters.contains(text(p,"character","tab")),"unsupported character")
43 if(operation=="add_column"){
44 val v=text(p,"value","1");require(!v.exists(c=>c=='\n'||c=='\r'||c=='\t'),"value cannot contain tabs or line breaks")
45 require(!p.has("iterate") || p.get("iterate").isBoolean,"iterate must be boolean")
46 }
47 }
48 private def boundedJoin(fields:Seq[String],separator:String):String={
49 require(fields.foldLeft(0L)(_+_.length)+math.max(0,fields.size-1).toLong*separator.length<=4194304,"output line exceeds 4 MiB characters");fields.mkString(separator)
50 }
51 private def cancelled():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
52 private val delegate=new ByteStreamOperator {
53 override def outputFormat:String=if(operation=="condense_characters")"text"else "tsv"
54 def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit={
55 validate(p)
56 val reader=new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT)),8192)
57 val writer=new BufferedWriter(new OutputStreamWriter(output,StandardCharsets.UTF_8),8192)
58 val cols=if(Set("change_case","cut_columns","merge_columns")(operation))columns(p)else Vector.empty
59 val delimiter=delimiters(text(p,"delimiter","tab"))
60 val character=text(p,"character","tab")
61 val pattern=java.util.regex.Pattern.compile("(?:"+delimiters(character)+")+")
62 val replacement=character match {case "tab"=>"\t";case "whitespace"|"space"=>" ";case "dot"=>".";case "comma"=>",";case "dash"=>"-";case "underscore"=>"_";case "pipe"=>"|";case "colon"=>":"}
63 val value=text(p,"value","1");var counter=if(value.matches("[0-9]+"))BigInt(value)else BigInt(0)
64 val numeric=value.matches("[0-9]+")
65 def emit(line:String):Unit={
66 cancelled()
67 val result:Option[String]=operation match {
68 case "change_case"=>val selected=cols.toSet;Some(line.split(delimiter).zipWithIndex.map{case(v,i)=>if(selected(i)){if(text(p,"casing","upper")=="upper")v.toUpperCase(Locale.ROOT)else v.toLowerCase(Locale.ROOT)}else v}.mkString("\t"))
69 case "condense_characters"=>Some(pattern.matcher(line).replaceAll(java.util.regex.Matcher.quoteReplacement(replacement)))
70 case "convert_delimiters"=>Some(pattern.matcher(line.strip()).replaceAll("\t"))
71 case "cut_columns"=>if(line.startsWith("#"))None else {val fields=line.split(delimiter);Some(boundedJoin(cols.map(i=>fields.lift(i).getOrElse(".")),"\t"))}
72 case "merge_columns"=>if(line.isEmpty || line.startsWith("#"))Some(line)else {val fields=line.split("\t",-1);Some(boundedJoin(Vector(line,boundedJoin(cols.flatMap(fields.lift),"")),"\t"))}
73 case "add_column"=>val next=if(!p.path("iterate").asBoolean(false))value else if(numeric)counter.toString else value+"-"+counter;counter+=1;Some(line+"\t"+next)
74 }
75 result.foreach{s=>writer.write(s);writer.write('\n')}
76 }
77 // readLine() can allocate an unbounded line. Enforce the limit before append.
78 val line=new java.lang.StringBuilder();var ch=reader.read();var afterCR=false;var count=0
79 while(ch != -1){
80 if((count & 4095)==0)cancelled();count+=1
81 if(ch=='\n' && afterCR)afterCR=false
82 else if(ch=='\r'||ch=='\n'){emit(line.toString);line.setLength(0);afterCR=ch=='\r'}
83 else {afterCR=false;require(line.length<1048576,"line exceeds 1 MiB characters");line.append(ch.toChar)}
84 ch=reader.read()
85 }
86 if(line.length>0)emit(line.toString)
87 writer.flush()
88 }
89 }
90 def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit=delegate.transform(input,output,p)
91 def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(inputs,p)}
92 def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(inputs,p,limits)}
93 def process(ctx:OperatorContext):Unit={validate(ctx.params);delegate.process(ctx)}
94}
95object ChangeCase extends GalaxyOperator("change_case")
96object CondenseCharacters extends GalaxyOperator("condense_characters")
97object ConvertDelimiters extends GalaxyOperator("convert_delimiters")
98object CutColumns extends GalaxyOperator("cut_columns")
99object MergeColumns extends GalaxyOperator("merge_columns")
100object AddColumn extends GalaxyOperator("add_column")