Source code

Download source archive ↓

src/Sequences.scala

1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.concurrent.CancellationException
6import com.fasterxml.jackson.databind.JsonNode
7import scala.jdk.CollectionConverters._
8import xstream.operator.{Operator,ByteStreamOperator,OperatorContext}
9import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
10
11/** Bounded FASTA and line selection tools, adapted from the pinned Galaxy scripts. */
12abstract class SequenceOperator(val operation:String) extends Operator {
13  private def number(p:JsonNode,k:String,d:Int,max:Int):Int={
14    val n=p.get(k);require(n==null || n.isIntegralNumber && n.canConvertToInt,k+" must be an integer")
15    val v=if(n==null)d else n.asInt;require(v>=0 && v<=max,k+" is outside the allowed range");v
16  }
17  private def validate(p:JsonNode):Unit={
18    require(p!=null && p.isObject,"parameters must be an object")
19    val keys=operation match {
20      case "fasta_length"=>Set("keep_first","id_only")
21      case "fasta_filter_length"=>Set("min_length","max_length")
22      case "fasta_to_tabular"=>Set("keep_first","description_columns")
23      case _=>Set("lines")
24    }
25    require(p.fieldNames.asScala.forall(keys),"unknown parameter")
26    if(operation=="fasta_length"||operation=="fasta_to_tabular")number(p,"keep_first",0,4096)
27    if(operation=="fasta_length")require(!p.has("id_only")||p.path("id_only").isBoolean,"id_only must be boolean")
28    if(operation=="fasta_filter_length"){
29      val min=number(p,"min_length",0,1000000000);val max=number(p,"max_length",0,1000000000)
30      require(max==0 || max>=min,"max_length must be zero or at least min_length")
31    }
32    if(operation=="fasta_to_tabular")require(number(p,"description_columns",1,32)>=1,"description_columns must be at least one")
33    if(!operation.startsWith("fasta_"))number(p,"lines",10,if(operation=="tail")10000 else 1000000)
34  }
35  private def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
36  private def lines(input:InputStream)(f:(String,String)=>Unit):Unit={
37    val r=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)),8192),1)
38    val b=new java.lang.StringBuilder();var ch=r.read();var count=0
39    while(ch != -1){
40      if((count & 4095)==0)active();count+=1
41      if(ch=='\r'||ch=='\n'){
42        var ending=ch.toChar.toString
43        if(ch=='\r'){val next=r.read();if(next=='\n')ending="\r\n"else if(next != -1)r.unread(next)}
44        f(b.toString,ending);b.setLength(0)
45      }else {require(b.length<1048576,"line exceeds 1 MiB characters");b.append(ch.toChar)}
46      ch=r.read()
47    }
48    if(b.length>0)f(b.toString,"")
49    active()
50  }
51  private def truncate(s:String,n:Int):String=if(n==0 || s.codePointCount(0,s.length)<=n)s else s.substring(0,s.offsetByCodePoints(0,n))
52  private val delegate=new ByteStreamOperator {
53    override def outputFormat:String=operation match {case "fasta_filter_length"=>"fasta";case "fasta_length"|"fasta_to_tabular"=>"tsv";case _=>"text"}
54    def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit={
55      validate(p);active();val out=new BufferedWriter(new OutputStreamWriter(output,StandardCharsets.UTF_8),8192)
56      def write(s:String):Unit={active();out.write(s)}
57      if(!operation.startsWith("fasta_")){
58        val limit=number(p,"lines",10,if(operation=="tail")10000 else 1000000);var index=0L
59        val tail=new java.util.ArrayDeque[String]();var chars=0L
60        lines(input){(line,ending)=>
61          index=Math.addExact(index,1L)
62          operation match {
63            case "remove_beginning"=>if(index>limit)write(line+ending)
64            case "head"=>if(index<=limit)write(line+ending)
65            case "tail"=>if(limit>0){val s=line+ending;tail.addLast(s);chars+=s.length;if(tail.size>limit)chars-=tail.removeFirst().length;require(chars<=8388608,"tail buffer exceeds 8 MiB characters")}
66          }
67        }
68        if(operation=="tail")tail.iterator.asScala.foreach(write)
69      }else {
70        var title:Option[String]=None;var length=0L;var haveOutput=false
71        val record=new java.lang.StringBuilder();var discarded=false
72        val min=number(p,"min_length",0,1000000000);val max=number(p,"max_length",0,1000000000)
73        val keep=number(p,"keep_first",0,4096)
74        def append(s:String):Unit={require(record.length.toLong+s.length<=8388608,"FASTA record exceeds 8 MiB characters");record.append(s)}
75        def flush(last:Boolean):Unit=title.foreach { header=>
76          operation match {
77            case "fasta_length"=>val id=if(p.path("id_only").asBoolean(false))header.split("(?U)\\s+",2).head else header;write(truncate(id,keep)+"\t"+length+"\n")
78            case "fasta_filter_length"=>if(!discarded && length>=min && (max==0 || length<=max)){
79              val value=record.toString;write(if(last)value.replaceAll("[\\r\\n]+$","")else value)
80            }
81            case "fasta_to_tabular"=>()
82          }
83        }
84        lines(input){(raw,_)=>
85          val line=if(operation=="fasta_length")raw.strip()else raw
86          if(line.nonEmpty && !line.startsWith("#")){
87            if(line.startsWith(">")){
88              flush(false);title=Some(line.drop(1));length=0;record.setLength(0);discarded=false
89              if(operation=="fasta_filter_length")append(line+"\n")
90              if(operation=="fasta_to_tabular"){
91                if(haveOutput)write("\n");haveOutput=true
92                val header=line.drop(1).replace('\t',' ');val columns=number(p,"description_columns",1,32)
93                val fields=if(columns==1)Vector(truncate(header,keep))else {
94                  val words=header.strip().split("(?U)\\s+",columns).toVector
95                  words.updated(0,truncate(words.head,keep)).padTo(columns,"")
96                }
97                write(fields.mkString("\t")+"\t")
98              }
99            }else {
100              require(title.nonEmpty,"FASTA sequence must follow a > header")
101              length=Math.addExact(length,line.codePointCount(0,line.length).toLong)
102              if(operation=="fasta_filter_length"){
103                if(max>0 && length>max){discarded=true;record.setLength(0)}
104                else if(!discarded)append(line+"\n")
105              }
106              if(operation=="fasta_to_tabular")write(line)
107            }
108          }
109        }
110        flush(true);if(operation=="fasta_to_tabular" && haveOutput)write("\n")
111      }
112      out.flush()
113    }
114  }
115  def transform(in:InputStream,out:OutputStream,p:JsonNode):Unit=delegate.transform(in,out,p)
116  def inferOutputs(in:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(in,p)}
117  def previewOutputs(in:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(in,p,l)}
118  def process(ctx:OperatorContext):Unit={validate(ctx.params);delegate.process(ctx)}
119}
120object FastaLength extends SequenceOperator("fasta_length")
121object FastaFilterLength extends SequenceOperator("fasta_filter_length")
122object FastaToTabular extends SequenceOperator("fasta_to_tabular")
123object RemoveBeginning extends SequenceOperator("remove_beginning")
124object Head extends SequenceOperator("head")
125object Tail extends SequenceOperator("tail")