src/Sequences.scala
1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.concurrent.CancellationException
6import com.fasterxml.jackson.databind.JsonNode
7import scala.jdk.CollectionConverters._
8import xstream.operator.{Operator,ByteStreamOperator,OperatorContext}
9import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
10
11/** Bounded FASTA and line selection tools, adapted from the pinned Galaxy scripts. */
12abstract class SequenceOperator(val operation:String) extends Operator {
13 private def number(p:JsonNode,k:String,d:Int,max:Int):Int={
14 val n=p.get(k);require(n==null || n.isIntegralNumber && n.canConvertToInt,k+" must be an integer")
15 val v=if(n==null)d else n.asInt;require(v>=0 && v<=max,k+" is outside the allowed range");v
16 }
17 private def validate(p:JsonNode):Unit={
18 require(p!=null && p.isObject,"parameters must be an object")
19 val keys=operation match {
20 case "fasta_length"=>Set("keep_first","id_only")
21 case "fasta_filter_length"=>Set("min_length","max_length")
22 case "fasta_to_tabular"=>Set("keep_first","description_columns")
23 case _=>Set("lines")
24 }
25 require(p.fieldNames.asScala.forall(keys),"unknown parameter")
26 if(operation=="fasta_length"||operation=="fasta_to_tabular")number(p,"keep_first",0,4096)
27 if(operation=="fasta_length")require(!p.has("id_only")||p.path("id_only").isBoolean,"id_only must be boolean")
28 if(operation=="fasta_filter_length"){
29 val min=number(p,"min_length",0,1000000000);val max=number(p,"max_length",0,1000000000)
30 require(max==0 || max>=min,"max_length must be zero or at least min_length")
31 }
32 if(operation=="fasta_to_tabular")require(number(p,"description_columns",1,32)>=1,"description_columns must be at least one")
33 if(!operation.startsWith("fasta_"))number(p,"lines",10,if(operation=="tail")10000 else 1000000)
34 }
35 private def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
36 private def lines(input:InputStream)(f:(String,String)=>Unit):Unit={
37 val r=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)),8192),1)
38 val b=new java.lang.StringBuilder();var ch=r.read();var count=0
39 while(ch != -1){
40 if((count & 4095)==0)active();count+=1
41 if(ch=='\r'||ch=='\n'){
42 var ending=ch.toChar.toString
43 if(ch=='\r'){val next=r.read();if(next=='\n')ending="\r\n"else if(next != -1)r.unread(next)}
44 f(b.toString,ending);b.setLength(0)
45 }else {require(b.length<1048576,"line exceeds 1 MiB characters");b.append(ch.toChar)}
46 ch=r.read()
47 }
48 if(b.length>0)f(b.toString,"")
49 active()
50 }
51 private def truncate(s:String,n:Int):String=if(n==0 || s.codePointCount(0,s.length)<=n)s else s.substring(0,s.offsetByCodePoints(0,n))
52 private val delegate=new ByteStreamOperator {
53 override def outputFormat:String=operation match {case "fasta_filter_length"=>"fasta";case "fasta_length"|"fasta_to_tabular"=>"tsv";case _=>"text"}
54 def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit={
55 validate(p);active();val out=new BufferedWriter(new OutputStreamWriter(output,StandardCharsets.UTF_8),8192)
56 def write(s:String):Unit={active();out.write(s)}
57 if(!operation.startsWith("fasta_")){
58 val limit=number(p,"lines",10,if(operation=="tail")10000 else 1000000);var index=0L
59 val tail=new java.util.ArrayDeque[String]();var chars=0L
60 lines(input){(line,ending)=>
61 index=Math.addExact(index,1L)
62 operation match {
63 case "remove_beginning"=>if(index>limit)write(line+ending)
64 case "head"=>if(index<=limit)write(line+ending)
65 case "tail"=>if(limit>0){val s=line+ending;tail.addLast(s);chars+=s.length;if(tail.size>limit)chars-=tail.removeFirst().length;require(chars<=8388608,"tail buffer exceeds 8 MiB characters")}
66 }
67 }
68 if(operation=="tail")tail.iterator.asScala.foreach(write)
69 }else {
70 var title:Option[String]=None;var length=0L;var haveOutput=false
71 val record=new java.lang.StringBuilder();var discarded=false
72 val min=number(p,"min_length",0,1000000000);val max=number(p,"max_length",0,1000000000)
73 val keep=number(p,"keep_first",0,4096)
74 def append(s:String):Unit={require(record.length.toLong+s.length<=8388608,"FASTA record exceeds 8 MiB characters");record.append(s)}
75 def flush(last:Boolean):Unit=title.foreach { header=>
76 operation match {
77 case "fasta_length"=>val id=if(p.path("id_only").asBoolean(false))header.split("(?U)\\s+",2).head else header;write(truncate(id,keep)+"\t"+length+"\n")
78 case "fasta_filter_length"=>if(!discarded && length>=min && (max==0 || length<=max)){
79 val value=record.toString;write(if(last)value.replaceAll("[\\r\\n]+$","")else value)
80 }
81 case "fasta_to_tabular"=>()
82 }
83 }
84 lines(input){(raw,_)=>
85 val line=if(operation=="fasta_length")raw.strip()else raw
86 if(line.nonEmpty && !line.startsWith("#")){
87 if(line.startsWith(">")){
88 flush(false);title=Some(line.drop(1));length=0;record.setLength(0);discarded=false
89 if(operation=="fasta_filter_length")append(line+"\n")
90 if(operation=="fasta_to_tabular"){
91 if(haveOutput)write("\n");haveOutput=true
92 val header=line.drop(1).replace('\t',' ');val columns=number(p,"description_columns",1,32)
93 val fields=if(columns==1)Vector(truncate(header,keep))else {
94 val words=header.strip().split("(?U)\\s+",columns).toVector
95 words.updated(0,truncate(words.head,keep)).padTo(columns,"")
96 }
97 write(fields.mkString("\t")+"\t")
98 }
99 }else {
100 require(title.nonEmpty,"FASTA sequence must follow a > header")
101 length=Math.addExact(length,line.codePointCount(0,line.length).toLong)
102 if(operation=="fasta_filter_length"){
103 if(max>0 && length>max){discarded=true;record.setLength(0)}
104 else if(!discarded)append(line+"\n")
105 }
106 if(operation=="fasta_to_tabular")write(line)
107 }
108 }
109 }
110 flush(true);if(operation=="fasta_to_tabular" && haveOutput)write("\n")
111 }
112 out.flush()
113 }
114 }
115 def transform(in:InputStream,out:OutputStream,p:JsonNode):Unit=delegate.transform(in,out,p)
116 def inferOutputs(in:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(in,p)}
117 def previewOutputs(in:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(in,p,l)}
118 def process(ctx:OperatorContext):Unit={validate(ctx.params);delegate.process(ctx)}
119}
120object FastaLength extends SequenceOperator("fasta_length")
121object FastaFilterLength extends SequenceOperator("fasta_filter_length")
122object FastaToTabular extends SequenceOperator("fasta_to_tabular")
123object RemoveBeginning extends SequenceOperator("remove_beginning")
124object Head extends SequenceOperator("head")
125object Tail extends SequenceOperator("tail")