src/Alignments.scala
1package xstream.galaxy
2
3import java.io._
4import java.nio.charset.{StandardCharsets,CodingErrorAction}
5import java.util.concurrent.CancellationException
6import com.fasterxml.jackson.databind.JsonNode
7import scala.jdk.CollectionConverters._
8import xstream.operator.{TransformOperator,ByteStreamOperator,OperatorContext}
9import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
10
11/** Streaming adaptations of Galaxy tabular-to-FASTA and SAM utilities. */
12abstract class AlignmentOperator(val operation:String) extends TransformOperator {
13 private def active():Unit=if(Thread.currentThread.isInterrupted)throw new CancellationException("operator cancelled")
14 private def integer(p:JsonNode,key:String,default:Int,max:Int):Int={
15 val n=p.get(key);require(n==null||n.isIntegralNumber&&n.canConvertToInt,key+" must be an integer")
16 val value=if(n==null)default else n.asInt;require(value>=0&&value<=max,key+" outside allowed range");value
17 }
18 private def titleColumns(p:JsonNode):Vector[Int]={
19 require(!p.has("title_columns")||p.path("title_columns").isTextual,"title_columns must be text")
20 val text=p.path("title_columns").asText("1");require(text.length<=4096,"title_columns too long")
21 val cols=text.split(",",-1).toVector
22 require(cols.nonEmpty&&cols.size<=128&&cols.forall(_.matches("[1-9][0-9]{0,4}")),"use comma-separated one-based title columns")
23 cols.map{s=>val n=s.toInt;require(n<=65536,"title column exceeds 65536");n-1}
24 }
25 private def validate(p:JsonNode):Unit={
26 require(p!=null&&p.isObject,"parameters must be an object")
27 val keys=operation match{case "tabular_to_fasta"=>Set("title_columns","sequence_column");case "sam_filter_flags"=>Set("require_flags","exclude_flags");case _=>Set("print_all")}
28 require(p.fieldNames.asScala.forall(keys),"unknown parameter")
29 operation match{
30 case "tabular_to_fasta"=>titleColumns(p);require(integer(p,"sequence_column",2,65536)>0,"sequence_column must be positive")
31 case "sam_filter_flags"=>val yes=integer(p,"require_flags",0,65535);val no=integer(p,"exclude_flags",0,65535);require((yes & no)==0,"required and excluded flags overlap")
32 case _=>require(!p.has("print_all")||p.path("print_all").isBoolean,"print_all must be boolean")
33 }
34 }
35 private val delegate=new ByteStreamOperator {
36 override def outputFormat:String=operation match{case "tabular_to_fasta"=>"fasta";case "sam_filter_flags"=>"sam";case _=>"tsv"}
37 def transform(input:InputStream,output:OutputStream,p:JsonNode):Unit={
38 validate(p);active()
39 val reader=new PushbackReader(new BufferedReader(new InputStreamReader(input,StandardCharsets.UTF_8.newDecoder.onMalformedInput(CodingErrorAction.REPORT)),8192),1)
40 val out=new BufferedWriter(new OutputStreamWriter(output,StandardCharsets.UTF_8),8192)
41 val titles=if(operation=="tabular_to_fasta")titleColumns(p)else Vector.empty
42 val sequence=integer(p,"sequence_column",2,65536)-1
43 val yes=integer(p,"require_flags",0,65535);val no=integer(p,"exclude_flags",0,65535)
44 def emit(text:String):Unit={active();require(text.length<=2097152,"output row exceeds 2 MiB characters");out.write(text);out.write("\n")}
45 if(operation=="sam_to_interval")emit("#chrom\tstart\tend\tstrand\tread_name")
46 def row(line:String):Unit={
47 active()
48 if(line.nonEmpty && !line.startsWith("#") && (operation=="tabular_to_fasta" || !line.startsWith("@"))){
49 val fields=line.split("\t",-1)
50 if(operation=="tabular_to_fasta"){
51 // Galaxy skips rows missing any selected column.
52 if(sequence<fields.length&&titles.forall(_<fields.length)){
53 val outputSize=titles.foldLeft(0L)((sum,i)=>sum+fields(i).length)+titles.size+fields(sequence).length+3
54 require(outputSize<=2097152,"output row exceeds 2 MiB characters")
55 val header=titles.map(fields(_)).updated(0,fields(titles.head).stripPrefix(">")).mkString("_")
56 emit(">"+header+"\n"+fields(sequence))
57 }
58 }else {
59 require(fields.length>=11,"SAM alignment requires at least 11 fields")
60 val flags=fields(1).toInt;require(flags>=0&&flags<=65535,"invalid SAM flag")
61 if(operation=="sam_filter_flags"){
62 if((flags & yes)==yes && (flags & no)==0)emit(line)
63 }else if(fields(2)!="*" && (flags & 4)==0){
64 val position=fields(3).toLong;require(position>0,"mapped SAM position must be positive")
65 val cigar=fields(5);val pattern="([0-9]+)([MIDNSHP=X])".r
66 val matches=pattern.findAllMatchIn(cigar).toVector
67 require(matches.nonEmpty&&matches.map(_.matched).mkString==cigar,"invalid mapped SAM CIGAR")
68 var span=0L
69 matches.foreach{m=>val length=m.group(1).toLong;require(length>0,"CIGAR lengths must be positive");if(Set("M","D","N","=","X")(m.group(2)))span=Math.addExact(span,length)}
70 require(span>0,"mapped CIGAR must consume reference bases")
71 val start=position-1;val end=Math.addExact(start,span);val strand=if((flags & 16)==0)"+"else"-"
72 emit(Vector(fields(2),start.toString,end.toString,strand,if(p.path("print_all").asBoolean(false))line else fields(0)).mkString("\t"))
73 }
74 }
75 }
76 }
77 val line=new java.lang.StringBuilder();var ch=reader.read();var count=0
78 while(ch != -1){
79 if((count & 4095)==0)active();count+=1
80 if(ch=='\r'||ch=='\n'){
81 if(ch=='\r'){val next=reader.read();if(next != -1&&next!='\n')reader.unread(next)}
82 row(line.toString);line.setLength(0)
83 }else{require(line.length<1048576,"line exceeds 1 MiB characters");line.append(ch.toChar)}
84 ch=reader.read()
85 }
86 if(line.length>0)row(line.toString);active();out.flush()
87 }
88 }
89 def transform(in:InputStream,out:OutputStream,p:JsonNode):Unit=delegate.transform(in,out,p)
90 def inferOutputs(in:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={validate(p);delegate.inferOutputs(in,p)}
91 def previewOutputs(in:Map[String,StreamDescriptor],p:JsonNode,l:PreviewLimits):Map[String,OutputSample]={validate(p);delegate.previewOutputs(in,p,l)}
92 def process(ctx:OperatorContext):Unit={validate(ctx.params);delegate.process(ctx)}
93}
94object TabularToFasta extends AlignmentOperator("tabular_to_fasta")
95object SamFilterFlags extends AlignmentOperator("sam_filter_flags")
96object SamToInterval extends AlignmentOperator("sam_to_interval")