Source code

Download source archive ↓

src/rag.py

1"""CONET RAG stages using XStream streams and a per-run lexical index."""
2
3from collections import Counter
4from html.parser import HTMLParser
5import json
6import math
7import re
8
9import pyarrow as pa
10from xstream_operator import descriptor, preview_process, result, run_operator
11
12
13DOCUMENT = pa.schema([(key, pa.string()) for key in ("id", "source", "format", "text")])
14CHUNK = pa.schema([
15    ("id", pa.string()), ("source", pa.string()), ("chunk", pa.string()),
16    ("start", pa.int64()), ("end", pa.int64()), ("text", pa.string()),
17])
18INDEXED = CHUNK.append(pa.field("terms", pa.list_(pa.string())))
19HIT = CHUNK.append(pa.field("score", pa.float64()))
20SCHEMAS = {
21    "parse": DOCUMENT, "clean": DOCUMENT, "chunk": CHUNK, "index": INDEXED, "retrieve": HIT,
22    "context": None, "generate": None, "cite": None,
23}
24BYTES_INPUTS = {"parse", "generate", "cite"}
25MAX_INPUT_BYTES = 2 * 1024 * 1024
26MAX_DOCUMENTS = 128
27MAX_CHUNKS = 4096
28
29
30class HTMLText(HTMLParser):
31    """Extract visible text from the demo's HTML subset without fetching anything."""
32
33    def __init__(self):
34        super().__init__()
35        self.parts = []
36        self.hidden = None
37
38    def handle_starttag(self, tag, _attrs):
39        if self.hidden is None:
40            if tag in {"script", "style"}:
41                self.hidden = tag
42            elif tag in {"p", "br", "div", "li", "h1", "h2", "h3"}:
43                self.parts.append(" ")
44
45    def handle_endtag(self, tag):
46        if tag == self.hidden:
47            self.hidden = None
48        elif self.hidden is None and tag in {"p", "div", "li", "h1", "h2", "h3"}:
49            self.parts.append(" ")
50
51    def handle_data(self, data):
52        if self.hidden is None:
53            self.parts.append(data)
54
55
56def read_bytes(source, limit=MAX_INPUT_BYTES):
57    raw = bytearray()
58    for block in source.bytes():
59        if len(raw) + len(block) > limit:
60            raise ValueError("RAG demo input exceeds its byte limit")
61        raw.extend(block)
62    return raw
63
64
65def jsonl_lines(source):
66    pending = bytearray()
67    total = 0
68    for block in source.bytes():
69        total += len(block)
70        if total > MAX_INPUT_BYTES:
71            raise ValueError("RAG demo input exceeds its byte limit")
72        lines = block.split(b"\n")
73        pending.extend(lines[0])
74        if len(lines) > 1:
75            yield bytes(pending)
76            yield from lines[1:-1]
77            pending = bytearray(lines[-1])
78    if pending:
79        yield bytes(pending)
80
81
82def parse_documents(source):
83    seen = set()
84    for number, line in enumerate(jsonl_lines(source), 1):
85        if not line.strip():
86            continue
87        row = json.loads(line.decode("utf-8"))
88        if (not isinstance(row, dict) or set(row) != {"id", "source", "format", "text"}
89                or not all(isinstance(value, str) for value in row.values())
90                or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", row["id"])
91                or not row["source"].strip() or len(row["source"]) > 512
92                or row["format"] not in {"text", "html"}):
93            raise ValueError(f"Invalid corpus record at line {number}")
94        if row["id"] in seen or len(seen) >= MAX_DOCUMENTS:
95            raise ValueError(f"Duplicate document ID or too many documents at line {number}")
96        seen.add(row["id"])
97        if row["format"] == "html":
98            parser = HTMLText()
99            parser.feed(row["text"])
100            parser.close()
101            row["text"] = "".join(parser.parts)
102        yield row
103
104
105def rows(source):
106    for batch in source.records():
107        yield from batch.to_pylist()
108
109
110def clean_documents(documents):
111    for row in documents:
112        text = " ".join(row["text"].split())
113        if text:
114            yield {**row, "text": text}
115
116
117def chunks(documents, size, overlap):
118    if not 0 <= overlap < size:
119        raise ValueError("overlap must be smaller than chunk_size")
120    count = 0
121    for row in documents:
122        text = row["text"]
123        start = 0
124        part = 1
125        while start < len(text):
126            end = min(start + size, len(text))
127            if end < len(text):
128                boundary = text.rfind(" ", start + size // 2, end)
129                if boundary > start + overlap:
130                    end = boundary
131            count += 1
132            if count > MAX_CHUNKS:
133                raise ValueError(f"RAG demo supports at most {MAX_CHUNKS} chunks")
134            yield {
135                "id": row["id"], "source": row["source"], "chunk": f"{row['id']}:{part}",
136                "start": start, "end": end, "text": text[start:end],
137            }
138            if end == len(text):
139                break
140            start = end - overlap
141            part += 1
142
143
144def terms(text):
145    for token in re.findall(r"[\u3400-\u9fff]+|[^\W\d_]+|\d+", text.casefold()):
146        if re.fullmatch(r"[\u3400-\u9fff]+", token):
147            yield from token
148            yield from (token[i:i + 2] for i in range(len(token) - 1))
149        elif token not in {"a", "an", "and", "are", "is", "of", "the", "to", "what", "why", "can"}:
150            yield token
151
152
153def retrieve(indexed_chunks, question, top_k):
154    query = set(terms(question))
155    corpus = []
156    document_frequency = Counter()
157    total_length = 0
158    for row in indexed_chunks:
159        length = len(row["terms"])
160        counts = Counter(term for term in row.pop("terms") if term in query)
161        document_frequency.update(counts.keys())
162        total_length += length
163        corpus.append((row, counts, length))
164    if not corpus or not query:
165        return []
166    average = max(1, total_length / len(corpus))
167    candidates = []
168    for row, counts, length in corpus:
169        if not counts:
170            continue
171        length_weight = 1.2 * (0.25 + 0.75 * length / average)
172        bm25 = sum(
173            math.log1p((len(corpus) - document_frequency[t] + 0.5) / (document_frequency[t] + 0.5))
174            * (2.2 * counts[t]) / (counts[t] + length_weight)
175            for t in sorted(counts)
176        )
177        candidates.append((bm25, len(counts), row))
178    candidates.sort(key=lambda value: (-value[0], value[2]["chunk"]))
179    # Rerank the BM25 shortlist by query coverage, then retain its score order.
180    shortlist = candidates[: max(top_k * 4, 12)]
181    shortlist.sort(key=lambda value: (-value[1], -value[0], value[2]["chunk"]))
182    return [{**row, "score": round(score, 6)} for score, _, row in shortlist[:top_k]]
183
184
185def make_context(hits, question):
186    return {
187        "question": question,
188        "evidence": [{"citation": number, **row} for number, row in enumerate(hits, 1)],
189    }
190
191
192def generate(context):
193    query = set(terms(context["question"]))
194    excerpts, citations = [], []
195    for row in context["evidence"]:
196        sentences = re.split(r"(?<=[.!?]) +|(?<=[。!?])", row["text"])
197        quote = max((s for s in sentences if s.strip()), key=lambda s: len(set(terms(s)) & query))
198        if set(terms(quote)) & query:
199            # Brackets already present in the source are not answer citations.
200            excerpt = re.sub(r"\[(\d+)\]", r"[\1]", quote.strip())
201            excerpts.append(f"{excerpt} [{row['citation']}]")
202            citations.append(row["citation"])
203    generated = {
204        "answer": "\n".join(excerpts) or "未检索到与问题匹配的证据。",
205        "citations": citations,
206    }
207    return {**context, "mode": "extractive", "generated": generated}
208
209
210def cite(envelope):
211    generated = envelope["generated"]
212    if (not isinstance(generated, dict) or set(generated) != {"answer", "citations"}
213            or not isinstance(generated["answer"], str) or not generated["answer"].strip()
214            or not isinstance(generated["citations"], list)):
215        raise ValueError("Generator must return a nonempty answer and a citations list")
216    by_id = {row["citation"]: row for row in envelope["evidence"]}
217    ids = generated["citations"]
218    if any(type(number) is not int or number not in by_id for number in ids):
219        raise ValueError("Generator cited evidence outside the retrieved context")
220    markers = {int(number) for number in re.findall(r"\[(\d+)\]", generated["answer"])}
221    if markers != set(ids):
222        raise ValueError("Inline citation markers must match the citations list")
223    return {
224        "question": envelope["question"], "mode": envelope["mode"],
225        "answer": generated["answer"],
226        "citations": [by_id[number] for number in dict.fromkeys(ids)],
227    }
228
229
230def process(ctx):
231    stage = ctx.params["stage"]
232    source = ctx.input("source")
233    schema = SCHEMAS[stage]
234    if schema is None:
235        output = ctx.bytes_output("result", "json")
236        if stage == "context":
237            value = make_context(rows(source), ctx.params["question"])
238        elif stage == "generate":
239            value = generate(json.loads(read_bytes(source, 256 * 1024)))
240        else:
241            value = cite(json.loads(read_bytes(source, 256 * 1024)))
242        output.write((json.dumps(value, ensure_ascii=False) + "\n").encode("utf-8"))
243        return
244
245    output = ctx.records_output("result", schema)
246    if stage == "parse":
247        records = parse_documents(source)
248    elif stage == "clean":
249        records = clean_documents(rows(source))
250    elif stage == "chunk":
251        records = chunks(rows(source), ctx.params["chunk_size"], ctx.params["overlap"])
252    elif stage == "index":
253        records = ({**row, "terms": list(terms(row["text"]))} for row in rows(source))
254    else:
255        records = retrieve(rows(source), ctx.params["question"], ctx.params["top_k"])
256    for row in records:
257        output.write(pa.RecordBatch.from_pylist([row], schema=schema))
258
259
260def infer_outputs(inputs, params):
261    stage = params["stage"]
262    if set(inputs) != {"source"}:
263        raise ValueError("Expected a single source input")
264    expected = "bytes" if stage in BYTES_INPUTS else "records"
265    views = inputs["source"].get("representations", [])
266    if views and views != [expected]:
267        raise ValueError(f"{stage} expects a {expected} input")
268    schema = SCHEMAS[stage]
269    view, fmt = ("bytes", "json") if schema is None else ("records", "arrow")
270    return result({"result": descriptor([view], [fmt], schema, certainty="known")})
271
272
273def preview_outputs(inputs, params, limits):
274    return preview_process(process, inputs, params, limits)
275
276
277if __name__ == "__main__":
278    run_operator(process, infer_outputs, preview_outputs=preview_outputs)