src/TabularUtilities.scala
1package xstream.galaxy
2
3import java.io.InputStream
4import java.util.Locale
5import com.fasterxml.jackson.core.JsonPointer
6import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode, ObjectMapper}
7import scala.collection.mutable.ArrayBuffer
8import scala.jdk.CollectionConverters._
9
10abstract class TabularUtilities(op: String) extends ExtendedOperator(op) {
11 private val mapper = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
12
13 override protected def format: String = operation match {
14 case "table_to_geojson" => "geojson"
15 case "jq_path" => "json"
16 case "draw_stacked_barplot" => "svg"
17 case _ => "tsv"
18 }
19
20 override protected def validate(p: JsonNode): Unit = {
21 super.validate(p)
22 if (operation == "jq_path") {
23 val pointer = param(p, "pointer").asText
24 require(pointer.matches("(?:/(?:[^/~]|~[01])*)*"), "expected a JSON Pointer")
25 }
26 if (operation == "table_to_geojson")
27 require(param(p, "latitude_column") != param(p, "longitude_column"), "coordinate columns must differ")
28 }
29
30 protected def run(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
31 val line = (value: String) => emit(value + "\n")
32 operation match {
33 case "filter_tabular" => filter(input, line, p)
34 case "table_compute" => compute(input, line, p)
35 case "summary_statistics" => summary(input, line, p)
36 case "table_to_geojson" => geojson(input, emit, p)
37 case "jq_path" => project(input, line, p)
38 case "draw_stacked_barplot" => stacked(input, line)
39 }
40 }
41
42 private def fields(line: String): Array[String] = line.split("\t", -1)
43
44 private def number(value: String): Double = {
45 require(value.length <= 128, "numeric field is too long")
46 val n = value.toDouble
47 require(n.isFinite && math.abs(n) <= 1e100, "expected a bounded finite number")
48 n
49 }
50
51 private def fixed(value: Double): String = {
52 require(value.isFinite, "numeric result outside the supported domain")
53 String.format(Locale.ROOT, "%.6f", Double.box(if (value == 0) 0d else value))
54 }
55
56 private def filter(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
57 val skip = param(p, "skip").asInt
58 val dropComments = param(p, "drop_comments").asBoolean
59 val appendNumber = param(p, "append_number").asBoolean
60 val columns = param(p, "columns").elements.asScala.map(_.asInt - 1).toVector
61 var inputRow = 0L
62 var outputRow = 0L
63 lines(input) { row =>
64 inputRow += 1
65 if (inputRow > skip && !(dropComments && row.startsWith("#"))) {
66 outputRow += 1
67 val selected = if (columns.isEmpty) row else {
68 val parts = fields(row)
69 require(columns.forall(_ < parts.length), "selected column is missing")
70 require(columns.iterator.map(i => parts(i).length.toLong + 1).sum <= 1048576,
71 "selected fields exceed the output row budget")
72 columns.map(parts(_)).mkString("\t")
73 }
74 emit(if (appendNumber) selected + "\t" + outputRow else selected)
75 }
76 }
77 }
78
79 private def compute(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
80 val column = param(p, "column").asInt - 1
81 val header = param(p, "header").asBoolean
82 val evaluate: Double => Double = param(p, "function").asText match {
83 case "log" => math.log
84 case "log10" => math.log10
85 case "exp" => math.exp
86 case "floor" => math.floor
87 case "ceil" => math.ceil
88 case "sqrt" => math.sqrt
89 }
90 var first = true
91 lines(input) { row =>
92 if (first && header) emit(row)
93 else {
94 val parts = fields(row)
95 require(column < parts.length, "numeric column is missing")
96 parts(column) = fixed(evaluate(number(parts(column))))
97 emit(parts.mkString("\t"))
98 }
99 first = false
100 }
101 }
102
103 private def summary(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
104 val column = param(p, "column").asInt - 1
105 val values = ArrayBuffer.empty[Double]
106 lines(input) { row =>
107 if (row.nonEmpty && !row.startsWith("#")) {
108 val parts = fields(row)
109 require(column < parts.length, "summary column is missing")
110 val value = parts(column)
111 if (value.nonEmpty && value != "NA") {
112 require(values.size < 100000, "summary exceeds 100000 observations")
113 values += number(value)
114 }
115 }
116 }
117 require(values.nonEmpty, "summary requires at least one observation")
118 val sorted = values.sorted
119 def quantile(probability: Double): Double = {
120 val rank = (sorted.size - 1) * probability
121 val lower = rank.toInt
122 val upper = math.ceil(rank).toInt
123 sorted(lower) + (rank - lower) * (sorted(upper) - sorted(lower))
124 }
125 val total = values.sum
126 val mean = total / values.size
127 val deviation = if (values.size == 1) "NA"
128 else fixed(math.sqrt(values.iterator.map(v => (v - mean) * (v - mean)).sum / (values.size - 1)))
129 emit("#sum\tmean\tstdev\t0%\t25%\t50%\t75%\t100%")
130 val quantiles = Vector(0d, 0.25, 0.5, 0.75, 1d).map(q => fixed(quantile(q)))
131 emit((Vector(fixed(total), fixed(mean), deviation) ++ quantiles).mkString("\t"))
132 }
133
134 private def geojson(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
135 val latitude = param(p, "latitude_column").asInt - 1
136 val longitude = param(p, "longitude_column").asInt - 1
137 val header = param(p, "has_header").asBoolean
138 var names = Option.empty[Array[String]]
139 var written = false
140 emit("""{"type":"FeatureCollection","features":[""")
141 lines(input) { row =>
142 val parts = fields(row)
143 require(parts.length > math.max(latitude, longitude) && parts.length <= 128, "coordinate columns are missing")
144 val isHeader = header && names.isEmpty
145 if (names.isEmpty) {
146 val labels = if (header) parts else parts.indices.map(i => s"column_${i + 1}").toArray
147 require(labels.distinct.length == labels.length && labels.forall(_.nonEmpty), "column names must be unique and nonempty")
148 names = Some(labels)
149 }
150 if (!isHeader) {
151 require(parts.length == names.get.length, "table rows must have the same width")
152 val lat = number(parts(latitude))
153 val lon = number(parts(longitude))
154 require(math.abs(lat) <= 90 && math.abs(lon) <= 180, "coordinates must be WGS 84 longitude and latitude")
155 val feature = mapper.createObjectNode().put("type", "Feature")
156 feature.putObject("geometry").put("type", "Point").putArray("coordinates").add(lon).add(lat)
157 val properties = feature.putObject("properties")
158 parts.indices.filter(i => i != latitude && i != longitude).foreach(i => properties.put(names.get(i), parts(i)))
159 if (written) emit(",")
160 emit(mapper.writeValueAsString(feature))
161 written = true
162 }
163 }
164 emit("]}\n")
165 }
166
167 private def project(input: InputStream, emit: String => Unit, p: JsonNode): Unit = {
168 val pointer = JsonPointer.compile(param(p, "pointer").asText)
169 lines(input) { row =>
170 if (row.trim.nonEmpty) {
171 val value = mapper.readTree(row).at(pointer)
172 emit(if (value.isMissingNode) "null" else mapper.writeValueAsString(value))
173 }
174 }
175 }
176
177 private def stacked(input: InputStream, emit: String => Unit): Unit = {
178 def escape(s: String): String = {
179 require(s.length <= 80 && !s.exists(Character.isISOControl), "chart labels must be at most 80 printable characters")
180 s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """)
181 }
182 val palette = Vector("#2563eb", "#0f766e", "#b45309", "#7c3aed", "#be123c", "#475569")
183 val series = ArrayBuffer.empty[(String, Vector[Double])]
184 var labels = Vector.empty[String]
185 lines(input) { row =>
186 val parts = fields(row)
187 require(parts.length >= 2 && parts.length <= 25, "expected a category and up to 24 criteria")
188 if (labels.isEmpty) labels = parts.drop(1).map(escape).toVector
189 else {
190 require(series.size < 24 && parts.length == labels.size + 1, "expected up to 24 equal-width category rows")
191 val values = parts.drop(1).map(number).toVector
192 require(values.forall(_ >= 0), "stacked bars require nonnegative values")
193 series += escape(parts.head) -> values
194 }
195 }
196 require(labels.nonEmpty && series.nonEmpty, "chart needs a header and category rows")
197 val width = 100 + labels.size * 120
198 val height = 322 + series.size * 18
199 val maximum = labels.indices.map(i => series.iterator.map(_._2(i)).sum).max
200 val scale = if (maximum == 0) 0d else 230d / maximum
201 emit(s"""<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 $width $height" role="img">""")
202 emit("<title>Stacked counts by criterion</title>")
203 labels.zipWithIndex.foreach { case (label, index) =>
204 active()
205 val x = 42 + index * 120
206 var top = 256d
207 series.zipWithIndex.foreach { case ((category, values), color) =>
208 val size = values(index) * scale
209 top -= size
210 emit(s"""<rect x="$x" y="${fixed(top)}" width="66" height="${fixed(size)}" fill="${palette(color % palette.size)}"><title>$category: ${values(index)}</title></rect>""")
211 }
212 emit(s"""<text x="$x" y="276">$label</text>""")
213 }
214 series.zipWithIndex.foreach { case ((category, _), i) =>
215 emit(s"""<text x="42" y="${310 + i * 18}" fill="${palette(i % palette.size)}">$category</text>""")
216 }
217 emit("</svg>")
218 }
219}
220
221abstract class ConcatenatedText(op: String) extends PairedStreams(op) {
222 protected def runPair(reference: InputStream, source: InputStream, emit: String => Unit, p: JsonNode): Unit = {
223 lines(source)(value => emit(value + "\n"))
224 lines(reference)(value => emit(value + "\n"))
225 }
226}