src/ItemSources.scala
1package xstream.examples.items
2
3import xstream.operator.{SourceOperator,TransformOperator,OperatorContext}
4import xstream.data._
5import xstream.execution.{InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
6import com.fasterxml.jackson.databind.JsonNode
7import java.nio.charset.StandardCharsets.UTF_8
8
9abstract class ItemCollectionSource extends SourceOperator {
10 protected def mode:String
11 protected def selection(params:JsonNode):JsonNode
12 final def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
13 require(inputs.isEmpty,"Source has no inputs")
14 require(params.path("uri").isTextual,"uri is required");ItemReferences.validate(params.path("uri").asText)
15 val s=selection(params);new ItemsViewSupport().selection(s)
16 InferenceResult(Map("result"->StreamDescriptor(Set("items"),Set("items"),certainty="known",structure=Some(new ItemsViewSupport().descriptor(ItemReferences.schema)))))
17 }
18 final def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
19 final def process(ctx:OperatorContext):Unit={
20 inferOutputs(Map.empty,ctx.params)
21 val output=ctx.itemsOutput("result")
22 ctx.openItem(ctx.params.path("uri").asText,"items",selection(ctx.params)){stream=>
23 ItemReferences.foreach(stream)(output.write)
24 }
25 }
26}
27object FolderItemsSource extends ItemCollectionSource {
28 protected val mode="folder"
29 protected def selection(params:JsonNode):JsonNode={
30 require(!params.has("recursive") || params.path("recursive").isBoolean,"recursive must be boolean")
31 Data.mapper.createObjectNode().put("mode",mode).put("recursive",params.path("recursive").asBoolean(false))
32 }
33}
34object CsvItemsSource extends ItemCollectionSource {
35 protected val mode="csv"
36 protected def selection(params:JsonNode):JsonNode={
37 require(!params.has("uri_column") || params.path("uri_column").isTextual && params.path("uri_column").asText.nonEmpty,"uri_column must be a nonempty string")
38 Data.mapper.createObjectNode().put("mode",mode).put("uri_column",params.path("uri_column").asText("uri"))
39 }
40}
41/** One invocation handles the whole collection; it chooses whether to open any member. */
42object ItemInventory extends TransformOperator {
43 def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
44 require(inputs.keySet==Set("source") && inputs("source").representations==Set("items"),"expected source items input")
45 InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set("text"),certainty="known")))
46 }
47 def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
48 def process(ctx:OperatorContext):Unit={
49 val output=ctx.bytesOutput("result","text")
50 ctx.input("source").foreachItem { item=>output.write(s"${item.uri}\t${item.kind}\t${item.format}\n".getBytes(UTF_8)) }
51 }
52}
53
54/** Demonstrates bounded lazy reads; directory-only references have no byte content. */
55object ItemByteCount extends TransformOperator {
56 def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult=
57 ItemInventory.inferOutputs(inputs,params)
58 def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
59 def process(ctx:OperatorContext):Unit={
60 val output=ctx.bytesOutput("result","text")
61 ctx.input("source").foreachItem { item=>
62 if(item.kind!="folder" || item.views.exists(_.contains("bytes"))) {
63 var count=0L
64 ctx.openItem(item,"bytes") { stream=>
65 ArrowBlocks.consume(stream,new DataConsumer[ByteChunk] {
66 def start(d:DataDescriptor[ByteChunk]):Unit=()
67 def accept(chunk:ByteChunk):Unit={count=Math.addExact(count,chunk.length.toLong)}
68 })
69 }
70 output.write(s"${item.uri}\t$count\n".getBytes(UTF_8))
71 }
72 }
73 }
74}