Source code

Download source archive ↓

spec.json

1{
2  "class": "BatchFieldThresholdClean",
3  "description": "EcoFlow inclusive threshold cleaning. left: records to clean; right: field,min_value,max_value,flag_value threshold records. No params. Fixes upstream Row-to-string field matching; field names must match exactly. Spark may coerce affected column types.",
4  "params": {},
5  "inputs": {
6    "left": {
7      "representations": [
8        "records"
9      ],
10      "formats": []
11    },
12    "right": {
13      "representations": [
14        "records"
15      ],
16      "formats": []
17    }
18  },
19  "outputs": {
20    "result": {
21      "representations": [
22        "records"
23      ],
24      "formats": [
25        "arrow"
26      ]
27    }
28  },
29  "mode": "spark_batch",
30  "test": "batch_field_threshold_clean",
31  "upstream_ports": {
32    "left": "MasterTablePort",
33    "right": "ThresholdTablePort"
34  },
35  "executor": "jvm",
36  "parameters": {
37    "type": "object",
38    "properties": {},
39    "required": [],
40    "additionalProperties": false
41  },
42  "icon": {
43    "path": "M3 5h5l5 7h8M3 19h5l5-7m4-4 4 4-4 4"
44  }
45}