Source code

Download source archive ↓

src/Alignments.scala

1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.concurrent.CancellationException
6import com.fasterxml.jackson.databind.JsonNode
7import scala.jdk.CollectionConverters._
8import xstream.operator.{TransformOperator,ByteStreamOperator,OperatorContext}
9import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
10
11/** Streaming adaptations of Galaxy tabular-to-FASTA and SAM utilities. */
12abstract class AlignmentOperator(val operation:String) extends TransformOperator {
13 private def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
14 private def integer(p:JsonNode,key:String,default:Int,max:Int):Int={
15  val n=p.get(key);require(n==null||n.isIntegralNumber&&n.canConvertToInt,key+" must be an integer")
16  val value=if(n==null)default else n.asInt;require(value>=0&&value<=max,key+" outside allowed range");value
17 }
18 private def titleColumns(p:JsonNode):Vector[Int]={
19  require(!p.has("title_columns")||p.path("title_columns").isTextual,"title_columns must be text")
20  val text=p.path("title_columns").asText("1");require(text.length<=4096,"title_columns too long")
21  val cols=text.split(",",-1).toVector
22  require(cols.nonEmpty&&cols.size<=128&&cols.forall(_.matches("[1-9][0-9]{0,4}")),"use comma-separated one-based title columns")
23  cols.map{s=>val n=s.toInt;require(n<=65536,"title column exceeds 65536");n-1}
24 }
25 private def validate(p:JsonNode):Unit={
26  require(p!=null&&p.isObject,"parameters must be an object")
27  val keys=operation match{case "tabular_to_fasta"=>Set("title_columns","sequence_column");case "sam_filter_flags"=>Set("require_flags","exclude_flags");case _=>Set("print_all")}
28  require(p.fieldNames.asScala.forall(keys),"unknown parameter")
29  operation match{
30   case "tabular_to_fasta"=>titleColumns(p);require(integer(p,"sequence_column",2,65536)>0,"sequence_column must be positive")
31   case "sam_filter_flags"=>val yes=integer(p,"require_flags",0,65535);val no=integer(p,"exclude_flags",0,65535);require((yes & no)==0,"required and excluded flags overlap")
32   case _=>require(!p.has("print_all")||p.path("print_all").isBoolean,"print_all must be boolean")
33  }
34 }
35 private val delegate=new ByteStreamOperator {
36  override def outputFormat:String=operation match{case "tabular_to_fasta"=>"fasta";case "sam_filter_flags"=>"sam";case _=>"tsv"}
37  def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit={
38   validate(p);active()
39   val reader=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder.onMalformedInput(CodingErrorAction.REPORT)),8192),1)
40   val out=new BufferedWriter(new OutputStreamWriter(output,StandardCharsets.UTF_8),8192)
41   val titles=if(operation=="tabular_to_fasta")titleColumns(p)else Vector.empty
42   val sequence=integer(p,"sequence_column",2,65536)-1
43   val yes=integer(p,"require_flags",0,65535);val no=integer(p,"exclude_flags",0,65535)
44   def emit(text:String):Unit={active();require(text.length<=2097152,"output row exceeds 2 MiB characters");out.write(text);out.write("\n")}
45   if(operation=="sam_to_interval")emit("#chrom\tstart\tend\tstrand\tread_name")
46   def row(line:String):Unit={
47    active()
48    if(line.nonEmpty && !line.startsWith("#") && (operation=="tabular_to_fasta" || !line.startsWith("@"))){
49     val fields=line.split("\t",-1)
50     if(operation=="tabular_to_fasta"){
51      // Galaxy skips rows missing any selected column.
52      if(sequence<fields.length&&titles.forall(_<fields.length)){
53       val outputSize=titles.foldLeft(0L)((sum,i)=>sum+fields(i).length)+titles.size+fields(sequence).length+3
54       require(outputSize<=2097152,"output row exceeds 2 MiB characters")
55       val header=titles.map(fields(_)).updated(0,fields(titles.head).stripPrefix(">")).mkString("_")
56       emit(">"+header+"\n"+fields(sequence))
57      }
58     }else {
59      require(fields.length>=11,"SAM alignment requires at least 11 fields")
60      val flags=fields(1).toInt;require(flags>=0&&flags<=65535,"invalid SAM flag")
61      if(operation=="sam_filter_flags"){
62       if((flags & yes)==yes && (flags & no)==0)emit(line)
63      }else if(fields(2)!="*" && (flags & 4)==0){
64       val position=fields(3).toLong;require(position>0,"mapped SAM position must be positive")
65       val cigar=fields(5);val pattern="([0-9]+)([MIDNSHP=X])".r
66       val matches=pattern.findAllMatchIn(cigar).toVector
67       require(matches.nonEmpty&&matches.map(_.matched).mkString==cigar,"invalid mapped SAM CIGAR")
68       var span=0L
69       matches.foreach{m=>val length=m.group(1).toLong;require(length>0,"CIGAR lengths must be positive");if(Set("M","D","N","=","X")(m.group(2)))span=Math.addExact(span,length)}
70       require(span>0,"mapped CIGAR must consume reference bases")
71       val start=position-1;val end=Math.addExact(start,span);val strand=if((flags & 16)==0)"+"else"-"
72       emit(Vector(fields(2),start.toString,end.toString,strand,if(p.path("print_all").asBoolean(false))line else fields(0)).mkString("\t"))
73      }
74     }
75    }
76   }
77   val line=new java.lang.StringBuilder();var ch=reader.read();var count=0
78   while(ch != -1){
79    if((count & 4095)==0)active();count+=1
80    if(ch=='\r'||ch=='\n'){
81     if(ch=='\r'){val next=reader.read();if(next != -1&&next!='\n')reader.unread(next)}
82     row(line.toString);line.setLength(0)
83    }else{require(line.length<1048576,"line exceeds 1 MiB characters");line.append(ch.toChar)}
84    ch=reader.read()
85   }
86   if(line.length>0)row(line.toString);active();out.flush()
87  }
88 }
89 def transform(in:InputStream,out:OutputStream,p:JsonNode):Unit=delegate.transform(in,out,p)
90 def inferOutputs(in:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(in,p)}
91 def previewOutputs(in:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(in,p,l)}
92 def process(ctx:OperatorContext):Unit={validate(ctx.params);delegate.process(ctx)}
93}
94object TabularToFasta extends AlignmentOperator("tabular_to_fasta")
95object SamFilterFlags extends AlignmentOperator("sam_filter_flags")
96object SamToInterval extends AlignmentOperator("sam_to_interval")