src/Endpoints.scala
1package xstream.examples.endpoints
2
3import com.fasterxml.jackson.databind.JsonNode
4import java.nio.charset.StandardCharsets.UTF_8
5import xstream.execution.{InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
6import xstream.operator.{Operator,OperatorContext}
7
8private[endpoints] object Parameters {
9 def fields(p:JsonNode,required:Set[String],optional:Set[String]=Set.empty):Unit={
10 import scala.jdk.CollectionConverters._
11 require(p!=null && p.isObject && required.subsetOf(p.fieldNames().asScala.toSet) && p.fieldNames().asScala.toSet.subsetOf(required++optional),"invalid parameters")
12 }
13 def text(p:JsonNode,key:String,max:Int):String={
14 val n=p.path(key);require(n.isTextual && n.asText.length<=max,"invalid "+key);n.asText
15 }
16 def number(p:JsonNode,key:String,min:Long,max:Long,default:Long):Long={
17 val n=p.path(key);if(n.isMissingNode)default else {require(n.isIntegralNumber && n.canConvertToLong && n.asLong>=min && n.asLong<=max,"invalid "+key);n.asLong}
18 }
19 def bytes(format:String)=InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set(format),certainty="known")))
20 def sample(format:String,raw:Array[Byte],limits:PreviewLimits,truncated:Boolean=false):Map[String,OutputSample]={
21 val take=raw.take(math.min(raw.length,limits.maxBytes/2))
22 Map("result"->OutputSample("bytes",format,data=take.toVector,truncated=truncated||take.length<raw.length).bounded(limits))
23 }
24}
25
26/** Emits bounded UTF-8 text without an input port. */
27object TextSource extends Operator {
28 import Parameters._
29 private def values(p:JsonNode):(Array[Byte],Int)={
30 fields(p,Set("text"),Set("repeat"));val bytes=text(p,"text",4096).getBytes(UTF_8)
31 val repeat=number(p,"repeat",0,10000,1).toInt
32 require(bytes.length.toLong*repeat<=8L*1024*1024,"generated text exceeds 8 MiB")
33 (bytes,repeat)
34 }
35 def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.isEmpty,"Source has no input ports");values(p);bytes("text")}
36 def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
37 val (value,repeat)=values(p);val out=new java.io.ByteArrayOutputStream()
38 var n=0;while(n<repeat && out.size()<limits.maxBytes/2){out.write(value);n+=1}
39 sample("text",out.toByteArray,limits,n<repeat)
40 }
41 def process(ctx:OperatorContext):Unit={
42 require(ctx.inputNames.isEmpty,"Source has no input ports")
43 val (value,repeat)=values(ctx.params);val out=ctx.bytesOutput("result","text")
44 var n=0;while(n<repeat){out.write(value);n+=1}
45 }
46}
47
48/** Generates a newline-delimited integer stream, without staging a file. */
49object RangeSource extends Operator {
50 import Parameters._
51 private def values(p:JsonNode):(Long,Int)={
52 fields(p,Set("start","count"))
53 val start=number(p,"start",Long.MinValue,Long.MaxValue,0)
54 val count=number(p,"count",0,100000,0).toInt
55 if(count>0)Math.addExact(start,count.toLong-1)
56 (start,count)
57 }
58 def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.isEmpty,"Source has no input ports");values(p);bytes("text")}
59 def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
60 val (start,count)=values(p);val out=new java.io.ByteArrayOutputStream();var n=0
61 while(n<count && n<100 && out.size()<limits.maxBytes/2){out.write((start+n).toString.getBytes(UTF_8));out.write('\n');n+=1}
62 sample("text",out.toByteArray,limits,n<count)
63 }
64 def process(ctx:OperatorContext):Unit={
65 require(ctx.inputNames.isEmpty,"Source has no input ports")
66 val (start,count)=values(ctx.params);val out=ctx.bytesOutput("result","text")
67 val chunk=new StringBuilder;var n=0
68 while(n<count){chunk.append(start+n).append('\n');if(chunk.length>=32768){out.write(chunk.toString.getBytes(UTF_8));chunk.clear()};n+=1}
69 if(chunk.nonEmpty)out.write(chunk.toString.getBytes(UTF_8))
70 }
71}
72
73/** Consumes all bytes and fails the Job unless the requested UTF-8 text occurs. */
74object ContainsTextSink extends Operator {
75 import Parameters._
76 private def needle(p:JsonNode):Array[Byte]={fields(p,Set("text"));val value=text(p,"text",256).getBytes(UTF_8);require(value.nonEmpty,"text must not be empty");value}
77 def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.keySet==Set("source") && inputs("source").representations("bytes"),"bytes source required");needle(p);InferenceResult(Map.empty)}
78 def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
79 def process(ctx:OperatorContext):Unit={
80 val pattern=needle(ctx.params);val prefix=new Array[Int](pattern.length)
81 var i=1;var j=0;while(i<pattern.length){while(j>0 && pattern(i)!=pattern(j))j=prefix(j-1);if(pattern(i)==pattern(j))j+=1;prefix(i)=j;i+=1}
82 val in=ctx.input("source").byteStream();var matched=false;j=0
83 try {var value=in.read();while(value>=0){
84 val b=value.toByte;while(j>0 && b!=pattern(j))j=prefix(j-1)
85 if(b==pattern(j))j+=1
86 if(j==pattern.length){matched=true;j=prefix(j-1)}
87 value=in.read()
88 }}finally in.close()
89 require(matched,"required text was not found")
90 }
91}
92
93/** Consumes all bytes and checks the minimum number of newline-delimited lines. */
94object MinLinesSink extends Operator {
95 import Parameters._
96 private def minimum(p:JsonNode):Long={fields(p,Set("minimum"));number(p,"minimum",0,100000000,0)}
97 def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={require(inputs.keySet==Set("source") && inputs("source").representations("bytes"),"bytes source required");minimum(p);InferenceResult(Map.empty)}
98 def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
99 def process(ctx:OperatorContext):Unit={
100 val min=minimum(ctx.params);val in=ctx.input("source").byteStream()
101 var lines=0L;var seen=false;var last=0
102 try {var value=in.read();while(value>=0){seen=true;if(value=='\n')lines+=1;last=value;value=in.read()}}finally in.close()
103 if(seen && last!='\n')lines+=1
104 require(lines>=min,s"expected at least $min lines; found $lines")
105 }
106}