Source code

Download source archive ↓

src/Extended.scala

1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.Locale
6import java.util.concurrent.CancellationException
7import com.fasterxml.jackson.databind.{JsonNode,ObjectMapper}
8import scala.jdk.CollectionConverters._
9import xstream.operator.{TransformOperator,ByteStreamOperator,OperatorContext}
10import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
11
12/** Common stream and resource contracts for independently documented Galaxy adapters. */
13abstract class ExtendedOperator(val operation:String) extends TransformOperator {
14 protected def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
15 protected def spec:JsonNode=ExtendedContract.specs.path(operation)
16 protected def validate(p:JsonNode):Unit={
17  require(p!=null&&p.isObject,"parameters must be an object")
18  val props=spec.path("parameters").path("properties")
19  require(p.fieldNames.asScala.forall(props.has),"unknown parameter")
20  def check(v:JsonNode,s:JsonNode):Unit={s.path("type").asText match{
21   case "string"=>require(v.isTextual&&v.asText.length<=s.path("maxLength").asInt(4096)&& !v.asText.exists(c=>c=='\r'||c=='\n'||c==0),"invalid text parameter");if(s.has("enum"))require(s.path("enum").elements.asScala.exists(_==v),"unsupported parameter value")
22   case "boolean"=>require(v.isBoolean,"parameter must be boolean")
23   case "integer"=>require(v.isIntegralNumber&&v.canConvertToInt&&v.asInt>=s.path("minimum").asInt&&v.asInt<=s.path("maximum").asInt,"integer parameter outside allowed bounds")
24   case "array"=>require(v.isArray&&v.size<=s.path("maxItems").asInt(128),"invalid parameter array");v.elements.asScala.foreach(check(_,s.path("items")))
25  }}
26  p.fields.asScala.foreach(e=>check(e.getValue,props.path(e.getKey)))
27  def text(k:String)=param(p,k).asText
28  if(Set("replace_text_in_line","replace_text_in_column","find_and_replace")(operation))require(text("find").nonEmpty,"find must not be empty")
29  if(operation=="unfold_column")require(text("separator").nonEmpty,"separator must not be empty")
30  if(Set("arrange_columns","remove_columns_by_header")(operation)){val a=param(p,"names").elements.asScala.map(_.asText).toVector;require(a.nonEmpty&&a.distinct.size==a.size&&a.forall(x=>x.nonEmpty && !x.contains('\t')),"names must be nonempty and unique")}
31  if(operation=="fixed_width_to_tabular")require(param(p,"widths").size>0&&param(p,"widths").elements.asScala.map(_.asInt.toLong).sum<=1048576,"widths must be nonempty and fit line budget")
32  if(operation=="find_subsequences")require(text("pattern").nonEmpty&&text("pattern").toUpperCase(Locale.ROOT).forall(Genetics.iupac.contains),"invalid IUPAC motif")
33  if(operation=="motif_flank_counts"){val a=param(p,"motifs").elements.asScala.map(_.asText).toVector;require(a.nonEmpty&&a.distinct.size==a.size,"motifs must be nonempty and unique");a.foreach{s=>val v=s.split("=",-1);require(v.length<=2&&v.forall(_.nonEmpty)&&v.last.length<=128&&v.last.toUpperCase(Locale.ROOT).forall("ACGTN".contains(_)),"motifs must be [name=]literal DNA")}}
34  if(operation=="recurring_lines")require((text("text").length.toLong+1)*param(p,"times").asInt<=16777216,"repeat output exceeds 16 MiB characters")
35  if(operation=="vcf_to_pgsnp")require(param(p,"sample_column").asInt==0||param(p,"sample_column").asInt>=10,"VCF sample_column must be zero or at least 10")
36 }
37 protected def param(p:JsonNode,k:String):JsonNode=Option(p.get(k)).getOrElse(spec.path("parameters").path("properties").path(k).path("default"))
38 protected def lines(input:InputStream)(f:String=>Unit):Unit={
39  val r=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder.onMalformedInput(CodingErrorAction.REPORT))),1)
40  val b=new java.lang.StringBuilder;var c=r.read();var ticks=0
41  while(c != -1){if((ticks&4095)==0)active();ticks+=1
42   if(c=='\n'||c=='\r'){if(c=='\r'){val n=r.read();if(n != -1&&n!='\n')r.unread(n)};f(b.toString);b.setLength(0)}
43   else{require(b.length<1048576,"line exceeds 1 MiB characters");b.append(c.toChar)};c=r.read()
44  };if(b.length>0)f(b.toString);active()
45 }
46 protected def run(input:InputStream,emit:String=>Unit,p:JsonNode):Unit
47 protected def format:String="text"
48 private val delegate=new ByteStreamOperator {
49  override def outputFormat:String=ExtendedOperator.this.format
50  def transform(i:InputStream,o:OutputStream,p:JsonNode):Unit={validate(p);active();val out=new BufferedWriter(new OutputStreamWriter(o,StandardCharsets.UTF_8));run(i,s=>{active();require(s.length<=16777216,"output exceeds row budget");out.write(s)},p);active();out.flush()}
51 }
52 def inputPorts:Vector[String]=Vector("source")
53 def transformInputs(inputs:Map[String,InputStream],o:OutputStream,p:JsonNode):Unit={require(inputs.keySet==Set("source"),"expected source input");transform(inputs("source"),o,p)}
54 def transform(i:InputStream,o:OutputStream,p:JsonNode):Unit=delegate.transform(i,o,p)
55 def inferOutputs(i:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(i,p)}
56 def previewOutputs(i:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(i,p,l)}
57 def process(c:OperatorContext):Unit={validate(c.params);delegate.process(c)}
58}
59
60abstract class ExtendedText(op:String) extends ExtendedOperator(op){
61 protected def run(input:InputStream,emit:String=>Unit,p:JsonNode):Unit={
62  def t(k:String)=param(p,k).asText;def n(k:String)=param(p,k).asInt;def b(k:String)=param(p,k).asBoolean
63  def row(s:String):Unit=emit(s+"\n")
64  def fields(s:String)=s.split("\t",-1)
65  var index=0L;var number=n("start").toLong
66  val stored=scala.collection.mutable.ArrayBuffer.empty[String];var chars=0L
67  def store(s:String):Unit={chars+=s.length.toLong+32;require(chars<=8388608&&stored.size<100000,"operation buffer exceeds 8 MiB characters or 100000 rows");stored+=s}
68  var previous:Option[String]=None;var count=0L
69  def fold(s:String)=if(b("ignore_case"))s.toUpperCase(Locale.ROOT)else s
70  def key(s:String):String={val col=n("column");fold(if(col==0)s else {val f=fields(s);require(col<=f.length,"selected column missing");f(col-1)})}
71  def compare(a:String,c:String):Int={active();val x=key(a);val y=key(c);if(b("numeric")){val u=BigDecimal(x);val v=BigDecimal(y);u.compare(v)}else x.compareTo(y)}
72  def flushGroup():Unit=previous.foreach{s=>if(t("mode")=="all"||t("mode")=="repeated"&&count>1||t("mode")=="single"&&count==1)row((if(b("count"))count+"\t"else"")+s)}
73  var columns=Vector.empty[Int];var width=0
74  def replace(s:String):String={
75   val needle=t("find");val replacement=t("replacement");val out=new java.lang.StringBuilder;var pos=0;var replacements=0
76   def locate(from:Int):Int={if(!b("ignore_case"))s.indexOf(needle,from)else{var j=from;while(j<=s.length-needle.length && !s.regionMatches(true,j,needle,0,needle.length)){if((j&4095)==0)active();j+=1};if(j<=s.length-needle.length)j else -1}}
77   var at=locate(pos)
78   while(at>=0){active();out.append(s,pos,at);require(out.length.toLong+replacement.length<=16777216,"replacement expansion exceeds output budget");out.append(replacement);pos=at+needle.length;replacements+=1;at=if(b("first_only")) -1 else locate(pos)}
79   require(out.length.toLong+s.length-pos<=16777216,"replacement output exceeds budget");out.append(s,pos,s.length);out.toString
80  }
81  if(operation=="add_line"&&t("position")=="header")row(t("text"))
82  lines(input){s=>
83   active();index+=1
84   operation match{
85    case "reverse_lines"=>store(s)
86    case "number_lines"=>if(s.isEmpty)row(" "*n("width"))else{val v=number.toString;val padding=" "*math.max(0,n("width")-v.length);val label=t("style")match{case "ln"=>v+padding;case "rz"=>("0"*math.max(0,n("width")-v.length))+v;case _=>padding+v};row(label+"\t"+s);number=Math.addExact(number,n("increment").toLong)}
87    case "sort_lines"|"unique_lines"=>if(index<=n("header"))row(s)else {if(b("numeric"))BigDecimal(key(s));store(s)}
88    case "sort_fields"=>row(s.strip().split("\t",-1).sorted.mkString("\t"))
89    case "unique_consecutive"=>if(previous.exists(x=>fold(x)==fold(s)))count+=1 else{flushGroup();previous=Some(s);count=1}
90    case "unfold_column"=>val f=fields(s);val col=n("column")-1;require(col<f.length,"selected column missing");val a=f(col).split(java.util.regex.Pattern.quote(t("separator")),-1);require(a.length<=10000&&a.length.toLong*(s.length+1)<=16777216,"unfold expansion exceeds budget");a.foreach{x=>active();f(col)=x;row(f.mkString("\t"))}
91    case "replace_text_in_column"=>if(b("skip_header")&&index==1)row(s)else{val f=fields(s);val col=n("column")-1;require(col<f.length,"selected column missing");f(col)=replace(f(col));row(f.mkString("\t"))}
92    case "replace_text_in_line"|"find_and_replace"=>row(if(b("skip_header")&&index==1)s else replace(s))
93    case "grep_lines"=>val found=if(b("ignore_case"))s.toUpperCase(Locale.ROOT).contains(t("pattern").toUpperCase(Locale.ROOT))else s.contains(t("pattern"));if(found != b("invert"))row(s)
94    case "arrange_columns"|"remove_columns_by_header"|"sort_columns_by_header"=>
95     val f=fields(s)
96     if(index==1){require(f.distinct.length==f.length,"header values must be unique");width=f.length
97      if(operation=="sort_columns_by_header"){val fixed=n("key_column")-1;require(fixed<f.length,"key column missing");columns=(if(fixed>=0)Vector(fixed)else Vector.empty)++f.indices.filter(_!=fixed).sortBy(f(_))}
98      else{val requested=param(p,"names").elements.asScala.map(_.asText).toVector;val names=f.zipWithIndex.map{case(v,i)=>if(i==0&&operation=="remove_columns_by_header")v.dropWhile(t("strip_prefix").contains(_))else v};require(requested.forall(names.contains),"unknown requested header")
99       if(operation=="arrange_columns"){val chosen=requested.map(x=>names.indexOf(x));columns=chosen++(if(b("discard"))Vector.empty else f.indices.filterNot(chosen.contains))}
100       else columns=f.indices.filter(i=>requested.contains(names(i))==b("keep")).toVector
101      }
102     };require(f.length==width,"ragged table row");row(columns.map(f(_)).mkString("\t"))
103    case "add_input_name"=>val value=if(index==1&&t("header").nonEmpty)t("header")else t("label");row(if(b("prepend"))value+"\t"+s else s+"\t"+value)
104    case "fixed_width_to_tabular"=>val sizes=param(p,"widths").elements.asScala.map(_.asInt).toVector;val total=sizes.sum;require(s.length<=total&&(s.length==total||b("pad_short")),"fixed-width row has wrong length");val padded=s.padTo(total,' ');var pos=0;row(sizes.map{len=>val value=padded.substring(pos,pos+len).strip();pos+=len;value}.mkString("\t"))
105    case "add_line"=>row(s)
106    case "recurring_lines"=>()
107   }
108  }
109  operation match{
110   case "reverse_lines"=>stored.reverseIterator.foreach(row)
111   case "sort_lines"=>val sorted=stored.sortWith((a,c)=>if(b("reverse"))compare(a,c)>0 else compare(a,c)<0);sorted.foreach(row)
112   case "unique_lines"=>var last:Option[String]=None;stored.sortWith((a,c)=>compare(a,c)<0).foreach{s=>if(last.forall(x=>compare(x,s)!=0)){row(s);last=Some(s)}}
113   case "unique_consecutive"=>flushGroup()
114   case "add_line"=>if(t("position")=="footer")row(t("text"))
115   case "recurring_lines"=>(0 until n("times")).foreach{_=>active();row(t("text"))}
116   case _=>()
117  }
118 }
119}