src/Extended.scala
1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.Locale
6import java.util.concurrent.CancellationException
7import com.fasterxml.jackson.databind.{JsonNode,ObjectMapper}
8import scala.jdk.CollectionConverters._
9import xstream.operator.{Operator,ByteStreamOperator,OperatorContext}
10import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
11
12/** Common stream and resource contracts for independently documented Galaxy adapters. */
13abstract class ExtendedOperator(val operation:String) extends Operator {
14 protected def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
15 protected def spec:JsonNode=ExtendedContract.specs.path(operation)
16 protected def validate(p:JsonNode):Unit={
17 require(p!=null&&p.isObject,"parameters must be an object")
18 val props=spec.path("parameters").path("properties")
19 require(p.fieldNames.asScala.forall(props.has),"unknown parameter")
20 def check(v:JsonNode,s:JsonNode):Unit={s.path("type").asText match{
21 case "string"=>require(v.isTextual&&v.asText.length<=s.path("maxLength").asInt(4096)&& !v.asText.exists(c=>c=='\r'||c=='\n'||c==0),"invalid text parameter");if(s.has("enum"))require(s.path("enum").elements.asScala.exists(_==v),"unsupported parameter value")
22 case "boolean"=>require(v.isBoolean,"parameter must be boolean")
23 case "integer"=>require(v.isIntegralNumber&&v.canConvertToInt&&v.asInt>=s.path("minimum").asInt&&v.asInt<=s.path("maximum").asInt,"integer parameter outside allowed bounds")
24 case "array"=>require(v.isArray&&v.size<=s.path("maxItems").asInt(128),"invalid parameter array");v.elements.asScala.foreach(check(_,s.path("items")))
25 }}
26 p.fields.asScala.foreach(e=>check(e.getValue,props.path(e.getKey)))
27 def text(k:String)=param(p,k).asText
28 if(Set("replace_text_in_line","replace_text_in_column","find_and_replace")(operation))require(text("find").nonEmpty,"find must not be empty")
29 if(operation=="unfold_column")require(text("separator").nonEmpty,"separator must not be empty")
30 if(Set("arrange_columns","remove_columns_by_header")(operation)){val a=param(p,"names").elements.asScala.map(_.asText).toVector;require(a.nonEmpty&&a.distinct.size==a.size&&a.forall(x=>x.nonEmpty && !x.contains('\t')),"names must be nonempty and unique")}
31 if(operation=="fixed_width_to_tabular")require(param(p,"widths").size>0&¶m(p,"widths").elements.asScala.map(_.asInt.toLong).sum<=1048576,"widths must be nonempty and fit line budget")
32 if(operation=="find_subsequences")require(text("pattern").nonEmpty&&text("pattern").toUpperCase(Locale.ROOT).forall(Genetics.iupac.contains),"invalid IUPAC motif")
33 if(operation=="motif_flank_counts"){val a=param(p,"motifs").elements.asScala.map(_.asText).toVector;require(a.nonEmpty&&a.distinct.size==a.size,"motifs must be nonempty and unique");a.foreach{s=>val v=s.split("=",-1);require(v.length<=2&&v.forall(_.nonEmpty)&&v.last.length<=128&&v.last.toUpperCase(Locale.ROOT).forall("ACGTN".contains(_)),"motifs must be [name=]literal DNA")}}
34 if(operation=="recurring_lines")require((text("text").length.toLong+1)*param(p,"times").asInt<=16777216,"repeat output exceeds 16 MiB characters")
35 if(operation=="vcf_to_pgsnp")require(param(p,"sample_column").asInt==0||param(p,"sample_column").asInt>=10,"VCF sample_column must be zero or at least 10")
36 }
37 protected def param(p:JsonNode,k:String):JsonNode=Option(p.get(k)).getOrElse(spec.path("parameters").path("properties").path(k).path("default"))
38 protected def lines(input:InputStream)(f:String=>Unit):Unit={
39 val r=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder.onMalformedInput(CodingErrorAction.REPORT))),1)
40 val b=new java.lang.StringBuilder;var c=r.read();var ticks=0
41 while(c != -1){if((ticks&4095)==0)active();ticks+=1
42 if(c=='\n'||c=='\r'){if(c=='\r'){val n=r.read();if(n != -1&&n!='\n')r.unread(n)};f(b.toString);b.setLength(0)}
43 else{require(b.length<1048576,"line exceeds 1 MiB characters");b.append(c.toChar)};c=r.read()
44 };if(b.length>0)f(b.toString);active()
45 }
46 protected def run(input:InputStream,emit:String=>Unit,p:JsonNode):Unit
47 protected def format:String="text"
48 private val delegate=new ByteStreamOperator {
49 override def outputFormat:String=ExtendedOperator.this.format
50 def transform(i:InputStream,o:OutputStream,p:JsonNode):Unit={validate(p);active();val out=new BufferedWriter(new OutputStreamWriter(o,StandardCharsets.UTF_8));run(i,s=>{active();require(s.length<=16777216,"output exceeds row budget");out.write(s)},p);active();out.flush()}
51 }
52 def inputPorts:Vector[String]=Vector("source")
53 def transformInputs(inputs:Map[String,InputStream],o:OutputStream,p:JsonNode):Unit={require(inputs.keySet==Set("source"),"expected source input");transform(inputs("source"),o,p)}
54 def transform(i:InputStream,o:OutputStream,p:JsonNode):Unit=delegate.transform(i,o,p)
55 def inferOutputs(i:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(i,p)}
56 def previewOutputs(i:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(i,p,l)}
57 def process(c:OperatorContext):Unit={validate(c.params);delegate.process(c)}
58}
59
60abstract class ExtendedText(op:String) extends ExtendedOperator(op){
61 protected def run(input:InputStream,emit:String=>Unit,p:JsonNode):Unit={
62 def t(k:String)=param(p,k).asText;def n(k:String)=param(p,k).asInt;def b(k:String)=param(p,k).asBoolean
63 def row(s:String):Unit=emit(s+"\n")
64 def fields(s:String)=s.split("\t",-1)
65 var index=0L;var number=n("start").toLong
66 val stored=scala.collection.mutable.ArrayBuffer.empty[String];var chars=0L
67 def store(s:String):Unit={chars+=s.length.toLong+32;require(chars<=8388608&&stored.size<100000,"operation buffer exceeds 8 MiB characters or 100000 rows");stored+=s}
68 var previous:Option[String]=None;var count=0L
69 def fold(s:String)=if(b("ignore_case"))s.toUpperCase(Locale.ROOT)else s
70 def key(s:String):String={val col=n("column");fold(if(col==0)s else {val f=fields(s);require(col<=f.length,"selected column missing");f(col-1)})}
71 def compare(a:String,c:String):Int={active();val x=key(a);val y=key(c);if(b("numeric")){val u=BigDecimal(x);val v=BigDecimal(y);u.compare(v)}else x.compareTo(y)}
72 def flushGroup():Unit=previous.foreach{s=>if(t("mode")=="all"||t("mode")=="repeated"&&count>1||t("mode")=="single"&&count==1)row((if(b("count"))count+"\t"else"")+s)}
73 var columns=Vector.empty[Int];var width=0
74 def replace(s:String):String={
75 val needle=t("find");val replacement=t("replacement");val out=new java.lang.StringBuilder;var pos=0;var replacements=0
76 def locate(from:Int):Int={if(!b("ignore_case"))s.indexOf(needle,from)else{var j=from;while(j<=s.length-needle.length && !s.regionMatches(true,j,needle,0,needle.length)){if((j&4095)==0)active();j+=1};if(j<=s.length-needle.length)j else -1}}
77 var at=locate(pos)
78 while(at>=0){active();out.append(s,pos,at);require(out.length.toLong+replacement.length<=16777216,"replacement expansion exceeds output budget");out.append(replacement);pos=at+needle.length;replacements+=1;at=if(b("first_only")) -1 else locate(pos)}
79 require(out.length.toLong+s.length-pos<=16777216,"replacement output exceeds budget");out.append(s,pos,s.length);out.toString
80 }
81 if(operation=="add_line"&&t("position")=="header")row(t("text"))
82 lines(input){s=>
83 active();index+=1
84 operation match{
85 case "reverse_lines"=>store(s)
86 case "number_lines"=>if(s.isEmpty)row(" "*n("width"))else{val v=number.toString;val padding=" "*math.max(0,n("width")-v.length);val label=t("style")match{case "ln"=>v+padding;case "rz"=>("0"*math.max(0,n("width")-v.length))+v;case _=>padding+v};row(label+"\t"+s);number=Math.addExact(number,n("increment").toLong)}
87 case "sort_lines"|"unique_lines"=>if(index<=n("header"))row(s)else {if(b("numeric"))BigDecimal(key(s));store(s)}
88 case "sort_fields"=>row(s.strip().split("\t",-1).sorted.mkString("\t"))
89 case "unique_consecutive"=>if(previous.exists(x=>fold(x)==fold(s)))count+=1 else{flushGroup();previous=Some(s);count=1}
90 case "unfold_column"=>val f=fields(s);val col=n("column")-1;require(col<f.length,"selected column missing");val a=f(col).split(java.util.regex.Pattern.quote(t("separator")),-1);require(a.length<=10000&&a.length.toLong*(s.length+1)<=16777216,"unfold expansion exceeds budget");a.foreach{x=>active();f(col)=x;row(f.mkString("\t"))}
91 case "replace_text_in_column"=>if(b("skip_header")&&index==1)row(s)else{val f=fields(s);val col=n("column")-1;require(col<f.length,"selected column missing");f(col)=replace(f(col));row(f.mkString("\t"))}
92 case "replace_text_in_line"|"find_and_replace"=>row(if(b("skip_header")&&index==1)s else replace(s))
93 case "grep_lines"=>val found=if(b("ignore_case"))s.toUpperCase(Locale.ROOT).contains(t("pattern").toUpperCase(Locale.ROOT))else s.contains(t("pattern"));if(found != b("invert"))row(s)
94 case "arrange_columns"|"remove_columns_by_header"|"sort_columns_by_header"=>
95 val f=fields(s)
96 if(index==1){require(f.distinct.length==f.length,"header values must be unique");width=f.length
97 if(operation=="sort_columns_by_header"){val fixed=n("key_column")-1;require(fixed<f.length,"key column missing");columns=(if(fixed>=0)Vector(fixed)else Vector.empty)++f.indices.filter(_!=fixed).sortBy(f(_))}
98 else{val requested=param(p,"names").elements.asScala.map(_.asText).toVector;val names=f.zipWithIndex.map{case(v,i)=>if(i==0&&operation=="remove_columns_by_header")v.dropWhile(t("strip_prefix").contains(_))else v};require(requested.forall(names.contains),"unknown requested header")
99 if(operation=="arrange_columns"){val chosen=requested.map(x=>names.indexOf(x));columns=chosen++(if(b("discard"))Vector.empty else f.indices.filterNot(chosen.contains))}
100 else columns=f.indices.filter(i=>requested.contains(names(i))==b("keep")).toVector
101 }
102 };require(f.length==width,"ragged table row");row(columns.map(f(_)).mkString("\t"))
103 case "add_input_name"=>val value=if(index==1&&t("header").nonEmpty)t("header")else t("label");row(if(b("prepend"))value+"\t"+s else s+"\t"+value)
104 case "fixed_width_to_tabular"=>val sizes=param(p,"widths").elements.asScala.map(_.asInt).toVector;val total=sizes.sum;require(s.length<=total&&(s.length==total||b("pad_short")),"fixed-width row has wrong length");val padded=s.padTo(total,' ');var pos=0;row(sizes.map{len=>val value=padded.substring(pos,pos+len).strip();pos+=len;value}.mkString("\t"))
105 case "add_line"=>row(s)
106 case "recurring_lines"=>()
107 }
108 }
109 operation match{
110 case "reverse_lines"=>stored.reverseIterator.foreach(row)
111 case "sort_lines"=>val sorted=stored.sortWith((a,c)=>if(b("reverse"))compare(a,c)>0 else compare(a,c)<0);sorted.foreach(row)
112 case "unique_lines"=>var last:Option[String]=None;stored.sortWith((a,c)=>compare(a,c)<0).foreach{s=>if(last.forall(x=>compare(x,s)!=0)){row(s);last=Some(s)}}
113 case "unique_consecutive"=>flushGroup()
114 case "add_line"=>if(t("position")=="footer")row(t("text"))
115 case "recurring_lines"=>(0 until n("times")).foreach{_=>active();row(t("text"))}
116 case _=>()
117 }
118 }
119}