src/rag.py
1"""CONET RAG stages using XStream streams and a per-run lexical index."""
2
3from collections import Counter
4from html.parser import HTMLParser
5import json
6import math
7import re
8
9import pyarrow as pa
10from xstream_operator import descriptor, preview_process, result, run_operator
11
12
13DOCUMENT = pa.schema([(key, pa.string()) for key in ("id", "source", "format", "text")])
14CHUNK = pa.schema([
15 ("id", pa.string()), ("source", pa.string()), ("chunk", pa.string()),
16 ("start", pa.int64()), ("end", pa.int64()), ("text", pa.string()),
17])
18INDEXED = CHUNK.append(pa.field("terms", pa.list_(pa.string())))
19HIT = CHUNK.append(pa.field("score", pa.float64()))
20SCHEMAS = {
21 "parse": DOCUMENT, "clean": DOCUMENT, "chunk": CHUNK, "index": INDEXED, "retrieve": HIT,
22 "context": None, "generate": None, "cite": None,
23}
24BYTES_INPUTS = {"parse", "generate", "cite"}
25MAX_INPUT_BYTES = 2 * 1024 * 1024
26MAX_DOCUMENTS = 128
27MAX_CHUNKS = 4096
28
29
30class HTMLText(HTMLParser):
31 """Extract visible text from the demo's HTML subset without fetching anything."""
32
33 def __init__(self):
34 super().__init__()
35 self.parts = []
36 self.hidden = None
37
38 def handle_starttag(self, tag, _attrs):
39 if self.hidden is None:
40 if tag in {"script", "style"}:
41 self.hidden = tag
42 elif tag in {"p", "br", "div", "li", "h1", "h2", "h3"}:
43 self.parts.append(" ")
44
45 def handle_endtag(self, tag):
46 if tag == self.hidden:
47 self.hidden = None
48 elif self.hidden is None and tag in {"p", "div", "li", "h1", "h2", "h3"}:
49 self.parts.append(" ")
50
51 def handle_data(self, data):
52 if self.hidden is None:
53 self.parts.append(data)
54
55
56def read_bytes(source, limit=MAX_INPUT_BYTES):
57 raw = bytearray()
58 for block in source.bytes():
59 if len(raw) + len(block) > limit:
60 raise ValueError("RAG demo input exceeds its byte limit")
61 raw.extend(block)
62 return raw
63
64
65def jsonl_lines(source):
66 pending = bytearray()
67 total = 0
68 for block in source.bytes():
69 total += len(block)
70 if total > MAX_INPUT_BYTES:
71 raise ValueError("RAG demo input exceeds its byte limit")
72 lines = block.split(b"\n")
73 pending.extend(lines[0])
74 if len(lines) > 1:
75 yield bytes(pending)
76 yield from lines[1:-1]
77 pending = bytearray(lines[-1])
78 if pending:
79 yield bytes(pending)
80
81
82def parse_documents(source):
83 seen = set()
84 for number, line in enumerate(jsonl_lines(source), 1):
85 if not line.strip():
86 continue
87 row = json.loads(line.decode("utf-8"))
88 if (not isinstance(row, dict) or set(row) != {"id", "source", "format", "text"}
89 or not all(isinstance(value, str) for value in row.values())
90 or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", row["id"])
91 or not row["source"].strip() or len(row["source"]) > 512
92 or row["format"] not in {"text", "html"}):
93 raise ValueError(f"Invalid corpus record at line {number}")
94 if row["id"] in seen or len(seen) >= MAX_DOCUMENTS:
95 raise ValueError(f"Duplicate document ID or too many documents at line {number}")
96 seen.add(row["id"])
97 if row["format"] == "html":
98 parser = HTMLText()
99 parser.feed(row["text"])
100 parser.close()
101 row["text"] = "".join(parser.parts)
102 yield row
103
104
105def rows(source):
106 for batch in source.records():
107 yield from batch.to_pylist()
108
109
110def clean_documents(documents):
111 for row in documents:
112 text = " ".join(row["text"].split())
113 if text:
114 yield {**row, "text": text}
115
116
117def chunks(documents, size, overlap):
118 if not 0 <= overlap < size:
119 raise ValueError("overlap must be smaller than chunk_size")
120 count = 0
121 for row in documents:
122 text = row["text"]
123 start = 0
124 part = 1
125 while start < len(text):
126 end = min(start + size, len(text))
127 if end < len(text):
128 boundary = text.rfind(" ", start + size // 2, end)
129 if boundary > start + overlap:
130 end = boundary
131 count += 1
132 if count > MAX_CHUNKS:
133 raise ValueError(f"RAG demo supports at most {MAX_CHUNKS} chunks")
134 yield {
135 "id": row["id"], "source": row["source"], "chunk": f"{row['id']}:{part}",
136 "start": start, "end": end, "text": text[start:end],
137 }
138 if end == len(text):
139 break
140 start = end - overlap
141 part += 1
142
143
144def terms(text):
145 for token in re.findall(r"[\u3400-\u9fff]+|[^\W\d_]+|\d+", text.casefold()):
146 if re.fullmatch(r"[\u3400-\u9fff]+", token):
147 yield from token
148 yield from (token[i:i + 2] for i in range(len(token) - 1))
149 elif token not in {"a", "an", "and", "are", "is", "of", "the", "to", "what", "why", "can"}:
150 yield token
151
152
153def retrieve(indexed_chunks, question, top_k):
154 query = set(terms(question))
155 corpus = []
156 document_frequency = Counter()
157 total_length = 0
158 for row in indexed_chunks:
159 length = len(row["terms"])
160 counts = Counter(term for term in row.pop("terms") if term in query)
161 document_frequency.update(counts.keys())
162 total_length += length
163 corpus.append((row, counts, length))
164 if not corpus or not query:
165 return []
166 average = max(1, total_length / len(corpus))
167 candidates = []
168 for row, counts, length in corpus:
169 if not counts:
170 continue
171 length_weight = 1.2 * (0.25 + 0.75 * length / average)
172 bm25 = sum(
173 math.log1p((len(corpus) - document_frequency[t] + 0.5) / (document_frequency[t] + 0.5))
174 * (2.2 * counts[t]) / (counts[t] + length_weight)
175 for t in sorted(counts)
176 )
177 candidates.append((bm25, len(counts), row))
178 candidates.sort(key=lambda value: (-value[0], value[2]["chunk"]))
179 # Rerank the BM25 shortlist by query coverage, then retain its score order.
180 shortlist = candidates[: max(top_k * 4, 12)]
181 shortlist.sort(key=lambda value: (-value[1], -value[0], value[2]["chunk"]))
182 return [{**row, "score": round(score, 6)} for score, _, row in shortlist[:top_k]]
183
184
185def make_context(hits, question):
186 return {
187 "question": question,
188 "evidence": [{"citation": number, **row} for number, row in enumerate(hits, 1)],
189 }
190
191
192def generate(context):
193 query = set(terms(context["question"]))
194 excerpts, citations = [], []
195 for row in context["evidence"]:
196 sentences = re.split(r"(?<=[.!?]) +|(?<=[。!?])", row["text"])
197 quote = max((s for s in sentences if s.strip()), key=lambda s: len(set(terms(s)) & query))
198 if set(terms(quote)) & query:
199 # Brackets already present in the source are not answer citations.
200 excerpt = re.sub(r"\[(\d+)\]", r"[\1]", quote.strip())
201 excerpts.append(f"{excerpt} [{row['citation']}]")
202 citations.append(row["citation"])
203 generated = {
204 "answer": "\n".join(excerpts) or "未检索到与问题匹配的证据。",
205 "citations": citations,
206 }
207 return {**context, "mode": "extractive", "generated": generated}
208
209
210def cite(envelope):
211 generated = envelope["generated"]
212 if (not isinstance(generated, dict) or set(generated) != {"answer", "citations"}
213 or not isinstance(generated["answer"], str) or not generated["answer"].strip()
214 or not isinstance(generated["citations"], list)):
215 raise ValueError("Generator must return a nonempty answer and a citations list")
216 by_id = {row["citation"]: row for row in envelope["evidence"]}
217 ids = generated["citations"]
218 if any(type(number) is not int or number not in by_id for number in ids):
219 raise ValueError("Generator cited evidence outside the retrieved context")
220 markers = {int(number) for number in re.findall(r"\[(\d+)\]", generated["answer"])}
221 if markers != set(ids):
222 raise ValueError("Inline citation markers must match the citations list")
223 return {
224 "question": envelope["question"], "mode": envelope["mode"],
225 "answer": generated["answer"],
226 "citations": [by_id[number] for number in dict.fromkeys(ids)],
227 }
228
229
230def process(ctx):
231 stage = ctx.params["stage"]
232 source = ctx.input("source")
233 schema = SCHEMAS[stage]
234 if schema is None:
235 output = ctx.bytes_output("result", "json")
236 if stage == "context":
237 value = make_context(rows(source), ctx.params["question"])
238 elif stage == "generate":
239 value = generate(json.loads(read_bytes(source, 256 * 1024)))
240 else:
241 value = cite(json.loads(read_bytes(source, 256 * 1024)))
242 output.write((json.dumps(value, ensure_ascii=False) + "\n").encode("utf-8"))
243 return
244
245 output = ctx.records_output("result", schema)
246 if stage == "parse":
247 records = parse_documents(source)
248 elif stage == "clean":
249 records = clean_documents(rows(source))
250 elif stage == "chunk":
251 records = chunks(rows(source), ctx.params["chunk_size"], ctx.params["overlap"])
252 elif stage == "index":
253 records = ({**row, "terms": list(terms(row["text"]))} for row in rows(source))
254 else:
255 records = retrieve(rows(source), ctx.params["question"], ctx.params["top_k"])
256 for row in records:
257 output.write(pa.RecordBatch.from_pylist([row], schema=schema))
258
259
260def infer_outputs(inputs, params):
261 stage = params["stage"]
262 if set(inputs) != {"source"}:
263 raise ValueError("Expected a single source input")
264 expected = "bytes" if stage in BYTES_INPUTS else "records"
265 views = inputs["source"].get("representations", [])
266 if views and views != [expected]:
267 raise ValueError(f"{stage} expects a {expected} input")
268 schema = SCHEMAS[stage]
269 view, fmt = ("bytes", "json") if schema is None else ("records", "arrow")
270 return result({"result": descriptor([view], [fmt], schema, certainty="known")})
271
272
273def preview_outputs(inputs, params, limits):
274 return preview_process(process, inputs, params, limits)
275
276
277if __name__ == "__main__":
278 run_operator(process, infer_outputs, preview_outputs=preview_outputs)