Source code

Download source archive ↓

src/Endpoints.scala

1package xstream.examples.endpoints
2
3import com.fasterxml.jackson.databind.JsonNode
4import java.nio.charset.StandardCharsets.UTF_8
5import xstream.execution.{InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
6import xstream.operator.{Operator,OperatorContext}
7
8private[endpoints] object Parameters {
9  def fields(p:JsonNode,required:Set[String],optional:Set[String]=Set.empty):Unit={
10    import scala.jdk.CollectionConverters._
11    require(p!=null && p.isObject && required.subsetOf(p.fieldNames().asScala.toSet) && p.fieldNames().asScala.toSet.subsetOf(required++optional),"invalid parameters")
12  }
13  def text(p:JsonNode,key:String,max:Int):String={
14    val n=p.path(key);require(n.isTextual && n.asText.length<=max,"invalid "+key);n.asText
15  }
16  def number(p:JsonNode,key:String,min:Long,max:Long,default:Long):Long={
17    val n=p.path(key);if(n.isMissingNode)default else {require(n.isIntegralNumber && n.canConvertToLong && n.asLong>=min && n.asLong<=max,"invalid "+key);n.asLong}
18  }
19  def bytes(format:String)=InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set(format),certainty="known")))
20  def sample(format:String,raw:Array[Byte],limits:PreviewLimits,truncated:Boolean=false):Map[String,OutputSample]={
21    val take=raw.take(math.min(raw.length,limits.maxBytes/2))
22    Map("result"->OutputSample("bytes",format,data=take.toVector,truncated=truncated||take.length<raw.length).bounded(limits))
23  }
24}
25
26/** Emits bounded UTF-8 text without an input port. */
27object TextSource extends Operator {
28  import Parameters._
29  private def values(p:JsonNode):(Array[Byte],Int)={
30    fields(p,Set("text"),Set("repeat"));val bytes=text(p,"text",4096).getBytes(UTF_8)
31    val repeat=number(p,"repeat",0,10000,1).toInt
32    require(bytes.length.toLong*repeat<=8L*1024*1024,"generated text exceeds 8 MiB")
33    (bytes,repeat)
34  }
35  def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.isEmpty,"Source has no input ports");values(p);bytes("text")}
36  def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
37    val (value,repeat)=values(p);val out=new java.io.ByteArrayOutputStream()
38    var n=0;while(n<repeat && out.size()<limits.maxBytes/2){out.write(value);n+=1}
39    sample("text",out.toByteArray,limits,n<repeat)
40  }
41  def process(ctx:OperatorContext):Unit={
42    require(ctx.inputNames.isEmpty,"Source has no input ports")
43    val (value,repeat)=values(ctx.params);val out=ctx.bytesOutput("result","text")
44    var n=0;while(n<repeat){out.write(value);n+=1}
45  }
46}
47
48/** Generates a newline-delimited integer stream, without staging a file. */
49object RangeSource extends Operator {
50  import Parameters._
51  private def values(p:JsonNode):(Long,Int)={
52    fields(p,Set("start","count"))
53    val start=number(p,"start",Long.MinValue,Long.MaxValue,0)
54    val count=number(p,"count",0,100000,0).toInt
55    if(count>0)Math.addExact(start,count.toLong-1)
56    (start,count)
57  }
58  def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.isEmpty,"Source has no input ports");values(p);bytes("text")}
59  def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
60    val (start,count)=values(p);val out=new java.io.ByteArrayOutputStream();var n=0
61    while(n<count && n<100 && out.size()<limits.maxBytes/2){out.write((start+n).toString.getBytes(UTF_8));out.write('\n');n+=1}
62    sample("text",out.toByteArray,limits,n<count)
63  }
64  def process(ctx:OperatorContext):Unit={
65    require(ctx.inputNames.isEmpty,"Source has no input ports")
66    val (start,count)=values(ctx.params);val out=ctx.bytesOutput("result","text")
67    val chunk=new StringBuilder;var n=0
68    while(n<count){chunk.append(start+n).append('\n');if(chunk.length>=32768){out.write(chunk.toString.getBytes(UTF_8));chunk.clear()};n+=1}
69    if(chunk.nonEmpty)out.write(chunk.toString.getBytes(UTF_8))
70  }
71}
72
73/** Consumes all bytes and fails the Job unless the requested UTF-8 text occurs. */
74object ContainsTextSink extends Operator {
75  import Parameters._
76  private def needle(p:JsonNode):Array[Byte]={fields(p,Set("text"));val value=text(p,"text",256).getBytes(UTF_8);require(value.nonEmpty,"text must not be empty");value}
77  def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.keySet==Set("source") && inputs("source").representations("bytes"),"bytes source required");needle(p);InferenceResult(Map.empty)}
78  def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
79  def process(ctx:OperatorContext):Unit={
80    val pattern=needle(ctx.params);val prefix=new Array[Int](pattern.length)
81    var i=1;var j=0;while(i<pattern.length){while(j>0 && pattern(i)!=pattern(j))j=prefix(j-1);if(pattern(i)==pattern(j))j+=1;prefix(i)=j;i+=1}
82    val in=ctx.input("source").byteStream();var matched=false;j=0
83    try {var value=in.read();while(value>=0){
84      val b=value.toByte;while(j>0 && b!=pattern(j))j=prefix(j-1)
85      if(b==pattern(j))j+=1
86      if(j==pattern.length){matched=true;j=prefix(j-1)}
87      value=in.read()
88    }}finally in.close()
89    require(matched,"required text was not found")
90  }
91}
92
93/** Consumes all bytes and checks the minimum number of newline-delimited lines. */
94object MinLinesSink extends Operator {
95  import Parameters._
96  private def minimum(p:JsonNode):Long={fields(p,Set("minimum"));number(p,"minimum",0,100000000,0)}
97  def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.keySet==Set("source") && inputs("source").representations("bytes"),"bytes source required");minimum(p);InferenceResult(Map.empty)}
98  def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
99  def process(ctx:OperatorContext):Unit={
100    val min=minimum(ctx.params);val in=ctx.input("source").byteStream()
101    var lines=0L;var seen=false;var last=0
102    try {var value=in.read();while(value>=0){seen=true;if(value=='\n')lines+=1;last=value;value=in.read()}}finally in.close()
103    if(seen && last!='\n')lines+=1
104    require(lines>=min,s"expected at least $min lines; found $lines")
105  }
106}