Source code

Download source archive ↓

src/AlignmentStreams.scala

1package xstream.galaxy
2import java.io.InputStream
3import java.util.Locale
4import com.fasterxml.jackson.databind.JsonNode
5import scala.collection.mutable
6
7abstract class AlignmentStreams(op:String) extends ExtendedOperator(op){
8 override protected def format:String=if(operation=="sam_to_fastq")"fastq"else if(Set("sam_sort","sam_collate","sam_reheader","sam_calmd")(operation))"sam"else"tsv"
9 private def dictionary(header:Vector[String]):Vector[(String,Long)]={
10  val d=header.filter(_.startsWith("@SQ\t")).map{s=>val f=s.split("\t").drop(1);val sn=f.filter(_.startsWith("SN:"));val ln=f.filter(_.startsWith("LN:"));require(sn.length==1&&ln.length==1&&sn(0).length>3,"invalid @SQ");val l=ln(0).drop(3).toLong;require(l>0&&l<=2147483647L,"invalid reference length");sn(0).drop(3)->l}
11  require(d.map(_._1).distinct.size==d.size,"duplicate reference");d
12 }
13 override protected def validate(p:JsonNode):Unit={super.validate(p)
14  if(operation=="sam_reheader"){val h=param(p,"header").asText.split("\n").toVector;require(h.nonEmpty&&h.forall(_.matches("@(HD|SQ|RG|PG|CO)\\t.*")),"invalid replacement header");dictionary(h)}
15  if(operation=="sam_calmd")references(p)
16 }
17 private def references(p:JsonNode):Map[String,String]={val all=param(p,"reference").asText.split(",",-1).toVector.map{x=>val at=x.indexOf('=');require(at>0&&at<x.length-1,"reference must be name=DNA");val dna=x.drop(at+1).toUpperCase(Locale.ROOT);require(dna.forall("ACGTN".contains(_)),"reference must be ACGTN");x.take(at)->dna};require(all.map(_._1).distinct.size==all.size,"duplicate reference");all.toMap}
18 protected def run(input:InputStream,emit:String=>Unit,p:JsonNode):Unit={
19  def row(s:String):Unit=emit(s+"\n")
20  def fmt(x:Double)=String.format(Locale.ROOT,"%.6f",Double.box(x))
21  val header=mutable.ArrayBuffer.empty[String];var started=false;var dict=Vector.empty[(String,Long)];var lengths=Map.empty[String,Long];var ranks=Map.empty[String,Int];var totalChars=0L
22  val records=mutable.ArrayBuffer.empty[Vector[String]];val blocks=mutable.Map.empty[String,mutable.ArrayBuffer[(Long,Long)]];val referenceCounts=mutable.Map.empty[String,Array[Long]];var unknownUnmapped=0L
23  val flagNames=Vector("total","primary","secondary","supplementary","duplicates","mapped","paired","read1","read2","proper_pair","singletons");val flags=Array.fill(flagNames.size,2)(0L)
24  var recordCount=0L;var mappedCount=0L;var baseCount=0L;var alignedCount=0L;var mapqTotal=0L;var blockCount=0L
25  val refs=if(operation=="sam_calmd")references(p)else Map.empty[String,String]
26  def begin():Unit=if(!started){started=true;dict=dictionary(header.toVector);require(dict.size<=100000,"too many references");lengths=dict.toMap;ranks=dict.map(_._1).zipWithIndex.toMap;dict.foreach{case(n,_)=>referenceCounts(n)=Array(0L,0L)}
27   if(format=="sam"&& !Set("sam_sort","sam_reheader")(operation))header.foreach(row)
28   if(operation=="sam_reheader"){val h=param(p,"header").asText.split("\n").toVector;require(dictionary(h)==dict,"replacement dictionary differs");h.foreach(row)}
29  }
30  lines(input){s=>active()
31   if(s.startsWith("@")){
32    require(!started&&s.matches("@(HD|SQ|RG|PG|CO)\\t.*"),"invalid or misplaced SAM header");totalChars+=s.length+1;require(totalChars<=8388608,"header budget exceeded");header+=s
33   }else{
34    begin();val f=s.split("\t",-1).toVector;require(f.size>=11,"SAM requires eleven fields");require(f(0).nonEmpty && !f(0).exists(_.isWhitespace),"invalid read name")
35    val flag=f(1).toInt;val pos=f(3).toLong;val mapq=f(4).toInt;require(flag>=0&&flag<=65535&&pos>=0&&pos<=2147483647L&&mapq>=0&&mapq<=255,"invalid alignment fields")
36    val mapped=(flag&4)==0;require(f(2)=="*"||lengths.contains(f(2)),"unknown reference");require(!mapped||(f(2)!="*"&&pos>0),"mapped read requires reference position")
37    require(f(9)=="*"||f(9).matches("[A-Za-z=.]+"),"invalid SAM sequence");require(f(10)=="*"||f(9)!="*"&&f(10).length==f(9).length&&f(10).forall(c=>c>=33&&c<=126),"invalid SAM quality")
38    val cigar=if(f(5)=="*")Vector.empty else "([0-9]+)([MIDNSHP=X])".r.findAllMatchIn(f(5)).map{m=>require(m.group(1).length<=10,"CIGAR length overflow");val n=m.group(1).toLong;require(n>0&&n<=2147483647L,"invalid CIGAR length");n->m.group(2).head}.toVector
39    require(f(5)=="*"||cigar.map{case(n,c)=>s"$n$c"}.mkString==f(5),"invalid CIGAR")
40    val query=cigar.filter(x=>"MIS=X".contains(x._2)).map(_._1).sum;val reference=cigar.filter(x=>"MDN=X".contains(x._2)).map(_._1).sum
41    require(f(9)=="*"||f(5)=="*"||query==f(9).length,"CIGAR sequence length mismatch")
42    if(mapped)require(cigar.nonEmpty&&reference>0&&pos-1+reference<=lengths(f(2)),"alignment outside reference")
43    val aligned=cigar.filter(x=>"M=X".contains(x._2)).map(_._1).sum
44    operation match{
45     case "sam_flagstat"=>val primary=(flag&0x900)==0;val predicates=Vector(true,primary,(flag&0x100)!=0,(flag&0x800)!=0,(flag&0x400)!=0,mapped,primary&&(flag&1)!=0,primary&&(flag&65)==65,primary&&(flag&129)==129,primary&&(flag&3)==3&&mapped,primary&&(flag&9)==9&&mapped);val qc=if((flag&512)==0)0 else 1;predicates.indices.foreach(i=>if(predicates(i))flags(i)(qc)=Math.addExact(flags(i)(qc),1L))
46     case "sam_reference_counts"=>if(f(2)=="*")unknownUnmapped+=1 else referenceCounts(f(2))(if(mapped)0 else 1)+=1
47     case "sam_alignment_stats"=>if((flag&0x900)==0){recordCount+=1;if(mapped){mappedCount+=1;mapqTotal+=mapq;alignedCount+=aligned};if(f(9)!="*")baseCount+=f(9).length}
48     case "sam_depth"|"sam_coverage"=>if(mapped&&(flag&0x704)==0){referenceCounts(f(2))(0)+=1;totalChars+=s.length+1;require(totalChars<=8388608,"coverage input budget exceeded");var at=pos-1;cigar.foreach{case(n,c)=>if("M=X".contains(c)){blockCount+=1;require(blockCount<=1000000,"too many coverage blocks");blocks.getOrElseUpdate(f(2),mutable.ArrayBuffer.empty)+=((at,at+n))};if("MDN=X".contains(c))at+=n}}
49     case "sam_sort"|"sam_collate"=>totalChars+=s.length+1;require(totalChars<=8388608&&records.size<100000,"SAM sort buffer exceeded");records+=f
50     case "sam_to_fastq"=>if((flag&0x900)==0&&f(9)!="*"){val reverse=(flag&16)!=0;val sequence=if(reverse)f(9).reverse.map(Genetics.complement(_))else f(9);val q=if(f(10)=="*")(param(p,"missing_quality").asInt+33).toChar.toString*f(9).length else if(reverse)f(10).reverse else f(10);row("@"+f(0));row(sequence);row("+");row(q)}
51     case "sam_reheader"=>row(s)
52     case "sam_calmd"=>if(!mapped)row(s)else{
53      require(f(9)!="*"&&refs.contains(f(2))&&refs(f(2)).length.toLong==lengths(f(2)),"matching reference and SEQ required");require(!cigar.exists(_._2=='N'),"CIGAR N skips unsupported")
54      val dna=refs(f(2));val sequence=f(9).toUpperCase(Locale.ROOT);val md=new java.lang.StringBuilder;var matched=0L;var nm=0L;var q=0;var at=(pos-1).toInt
55      cigar.foreach{case(n,c)=>active();c match{
56       case 'M'|'='|'X'=>var i=0L;while(i<n){if((i&4095)==0)active();if(sequence(q)==dna(at)&&sequence(q)!='N')matched+=1 else{md.append(matched).append(dna(at));matched=0;nm+=1};q+=1;at+=1;i+=1}
57       case 'I'=>q+=n.toInt;nm+=n
58       case 'D'=>md.append(matched).append('^').append(dna.substring(at,at+n.toInt));matched=0;at+=n.toInt;nm+=n
59       case 'S'=>q+=n.toInt
60       case 'H'|'P'=>()
61       case _=>throw new IllegalArgumentException("unsupported CIGAR")
62      }};md.append(matched);row((f.take(11)++f.drop(11).filterNot(x=>x.startsWith("NM:")||x.startsWith("MD:"))++Vector("NM:i:"+nm,"MD:Z:"+md)).mkString("\t"))
63     }
64     case _=>throw new IllegalArgumentException("unknown SAM operation")
65    }
66   }
67  };begin()
68  if(operation=="sam_flagstat"){row("category\tqc_pass\tqc_fail");flagNames.indices.foreach(i=>row(s"${flagNames(i)}\t${flags(i)(0)}\t${flags(i)(1)}"))}
69  if(operation=="sam_reference_counts"){dict.foreach{case(n,l)=>row(s"$n\t$l\t${referenceCounts(n)(0)}\t${referenceCounts(n)(1)}")};row(s"*\t0\t0\t$unknownUnmapped")}
70  if(operation=="sam_alignment_stats"){row("records\tmapped\tunmapped\tbases\taligned_bases\tmean_mapping_quality");row(s"$recordCount\t$mappedCount\t${recordCount-mappedCount}\t$baseCount\t$alignedCount\t${fmt(if(mappedCount==0)0d else mapqTotal.toDouble/mappedCount)}")}
71  if(operation=="sam_collate"){val groups=mutable.LinkedHashMap.empty[String,mutable.ArrayBuffer[Vector[String]]];records.foreach{f=>active();groups.getOrElseUpdate(f(0),mutable.ArrayBuffer.empty)+=f};groups.values.foreach(_.foreach{f=>active();row(f.mkString("\t"))})}
72  if(operation=="sam_sort"){
73   val order=param(p,"order").asText;val hd=header.filter(_.startsWith("@HD\t"));require(hd.size<=1,"duplicate @HD");val fields=hd.headOption.map(_.split("\t").toVector.filterNot(_.startsWith("SO:"))).getOrElse(Vector("@HD","VN:1.6"));row((fields:+("SO:"+order)).mkString("\t"));header.filterNot(_.startsWith("@HD\t")).foreach(row)
74   val sorted=if(order=="queryname")records.sortBy(_(0))else records.sortBy(f=>(ranks.getOrElse(f(2),Int.MaxValue),f(3).toLong));sorted.foreach{f=>active();row(f.mkString("\t"))}
75  }
76  if(operation=="sam_depth"||operation=="sam_coverage"){
77   if(operation=="sam_coverage")row("reference\tlength\treads\tcovered_bases\tcoverage_percent\tmean_depth")
78   var outputPositions=0L
79   dict.foreach{case(name,length)=>active();val events=mutable.Map.empty[Long,Long];blocks.getOrElse(name,mutable.ArrayBuffer.empty).foreach{case(a,z)=>events(a)=events.getOrElse(a,0L)+1;events(z)=events.getOrElse(z,0L)-1};var depth=0L;var at=0L;var covered=0L;var sum=0L
80    events.toVector.sortBy(_._1).foreach{case(x,d)=>active();if(depth>0){covered+=x-at;sum=Math.addExact(sum,Math.multiplyExact(x-at,depth));if(operation=="sam_depth"){outputPositions+=x-at;require(outputPositions<=1000000,"depth output exceeds 1000000 positions");var i=at;while(i<x){active();row(s"$name\t${i+1}\t$depth");i+=1}}};depth+=d;at=x}
81    if(operation=="sam_coverage")row(s"$name\t$length\t${referenceCounts(name)(0)}\t$covered\t${fmt(covered*100d/length)}\t${fmt(sum.toDouble/length)}")
82   }
83  }
84 }
85}