Source code

Download source archive ↓

src/ItemSources.scala

1package xstream.examples.items
2
3import xstream.operator.{SourceOperator,TransformOperator,OperatorContext}
4import xstream.data._
5import xstream.execution.{InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
6import com.fasterxml.jackson.databind.JsonNode
7import java.nio.charset.StandardCharsets.UTF_8
8
9abstract class ItemCollectionSource extends SourceOperator {
10  protected def mode:String
11  protected def selection(params:JsonNode):JsonNode
12  final def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
13    require(inputs.isEmpty,"Source has no inputs")
14    require(params.path("uri").isTextual,"uri is required");ItemReferences.validate(params.path("uri").asText)
15    val s=selection(params);new ItemsViewSupport().selection(s)
16    InferenceResult(Map("result"->StreamDescriptor(Set("items"),Set("items"),certainty="known",structure=Some(new ItemsViewSupport().descriptor(ItemReferences.schema)))))
17  }
18  final def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
19  final def process(ctx:OperatorContext):Unit={
20    inferOutputs(Map.empty,ctx.params)
21    val output=ctx.itemsOutput("result")
22    ctx.openItem(ctx.params.path("uri").asText,"items",selection(ctx.params)){stream=>
23      ItemReferences.foreach(stream)(output.write)
24    }
25  }
26}
27object FolderItemsSource extends ItemCollectionSource {
28  protected val mode="folder"
29  protected def selection(params:JsonNode):JsonNode={
30    require(!params.has("recursive") || params.path("recursive").isBoolean,"recursive must be boolean")
31    Data.mapper.createObjectNode().put("mode",mode).put("recursive",params.path("recursive").asBoolean(false))
32  }
33}
34object CsvItemsSource extends ItemCollectionSource {
35  protected val mode="csv"
36  protected def selection(params:JsonNode):JsonNode={
37    require(!params.has("uri_column") || params.path("uri_column").isTextual && params.path("uri_column").asText.nonEmpty,"uri_column must be a nonempty string")
38    Data.mapper.createObjectNode().put("mode",mode).put("uri_column",params.path("uri_column").asText("uri"))
39  }
40}
41/** One invocation handles the whole collection; it chooses whether to open any member. */
42object ItemInventory extends TransformOperator {
43  def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
44    require(inputs.keySet==Set("source") && inputs("source").representations==Set("items"),"expected source items input")
45    InferenceResult(Map("result"->StreamDescriptor(Set("bytes"),Set("text"),certainty="known")))
46  }
47  def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
48  def process(ctx:OperatorContext):Unit={
49    val output=ctx.bytesOutput("result","text")
50    ctx.input("source").foreachItem { item=>output.write(s"${item.uri}\t${item.kind}\t${item.format}\n".getBytes(UTF_8)) }
51  }
52}
53
54/** Demonstrates bounded lazy reads; directory-only references have no byte content. */
55object ItemByteCount extends TransformOperator {
56  def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult=
57    ItemInventory.inferOutputs(inputs,params)
58  def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=Map.empty
59  def process(ctx:OperatorContext):Unit={
60    val output=ctx.bytesOutput("result","text")
61    ctx.input("source").foreachItem { item=>
62      if(item.kind!="folder" || item.views.exists(_.contains("bytes"))) {
63        var count=0L
64        ctx.openItem(item,"bytes") { stream=>
65          ArrowBlocks.consume(stream,new DataConsumer[ByteChunk] {
66            def start(d:DataDescriptor[ByteChunk]):Unit=()
67            def accept(chunk:ByteChunk):Unit={count=Math.addExact(count,chunk.length.toLong)}
68          })
69        }
70        output.write(s"${item.uri}\t$count\n".getBytes(UTF_8))
71      }
72    }
73  }
74}