spec.json
1{
2 "class": "BatchFieldThresholdClean",
3 "description": "EcoFlow inclusive threshold cleaning. left: records to clean; right: field,min_value,max_value,flag_value threshold records. No params. Fixes upstream Row-to-string field matching; field names must match exactly. Spark may coerce affected column types.",
4 "params": {},
5 "inputs": {
6 "left": {
7 "representations": [
8 "records"
9 ],
10 "formats": []
11 },
12 "right": {
13 "representations": [
14 "records"
15 ],
16 "formats": []
17 }
18 },
19 "outputs": {
20 "result": {
21 "representations": [
22 "records"
23 ],
24 "formats": [
25 "arrow"
26 ]
27 }
28 },
29 "mode": "spark_batch",
30 "test": "batch_field_threshold_clean",
31 "upstream_ports": {
32 "left": "MasterTablePort",
33 "right": "ThresholdTablePort"
34 },
35 "executor": "jvm",
36 "parameters": {
37 "type": "object",
38 "properties": {},
39 "required": [],
40 "additionalProperties": false
41 },
42 "icon": {
43 "path": "M3 5h5l5 7h8M3 19h5l5-7m4-4 4 4-4 4"
44 }
45}