"""TrendsWhat reproducible editorial labs. Synthetic fixtures only; no network. Run: python3 run-labs.py --output ./results.json License: MIT (code and synthetic fixtures), copyright 2026 TrendsWhat. These are deterministic workflow tests, not language-model benchmarks. """ import argparse import csv import io import json import platform import re from datetime import datetime, timezone from pathlib import Path def csv_lab(): cases = [ ('plain', 'name,note\nAda,hello\n', [['name', 'note'], ['Ada', 'hello']]), ('quoted-comma', 'name,note\nAda,"hello, team"\n', [['name', 'note'], ['Ada', 'hello, team']]), ('quoted-newline', 'name,note\nAda,"hello\nteam"\n', [['name', 'note'], ['Ada', 'hello\nteam']]), ('escaped-quote', 'name,note\nAda,"say ""hello"""\n', [['name', 'note'], ['Ada', 'say "hello"']]), ('empty-field', 'name,note\nAda,\n', [['name', 'note'], ['Ada', '']]), ('leading-zero', 'id,note\n0012,keep\n', [['id', 'note'], ['0012', 'keep']]), ('unicode', 'name,note\n민지,café\n', [['name', 'note'], ['민지', 'café']]), ('crlf', 'name,note\r\nAda,hello\r\n', [['name', 'note'], ['Ada', 'hello']]), ] results = [] for name, source, expected in cases: naive = [line.split(',') for line in source.splitlines()] parsed = list(csv.reader(io.StringIO(source, newline=''))) results.append(dict(case=name, input=source, expected=expected, split_output=naive, parser_output=parsed, split_pass=naive == expected, parser_pass=parsed == expected)) return dict(cases=results, total=len(results), split_pass=sum(r['split_pass'] for r in results), parser_pass=sum(r['parser_pass'] for r in results)) def evidence_lab(): sources = { 'S1': 'The synthetic pilot included 12 tickets.', 'S2': 'The median draft time was 4 minutes in the synthetic pilot.', 'S3': 'The synthetic pilot did not measure customer satisfaction.', } # expected is editorial judgment, deliberately not computed by the checker. cases = [ ('supported-count', 'The pilot included 12 tickets.', 'S1', sources['S1'], True), ('missing-source', 'The pilot included 12 tickets.', 'S9', sources['S1'], False), ('invented-quote', 'The pilot included 120 tickets.', 'S1', 'The pilot included 120 tickets.', False), ('wrong-claim', 'The pilot included 120 tickets.', 'S1', sources['S1'], False), ('supported-time', 'Median draft time was 4 minutes.', 'S2', sources['S2'], True), ('unsupported-causality', 'The tool caused faster work.', 'S2', sources['S2'], False), ('supported-limit', 'Satisfaction was not measured.', 'S3', sources['S3'], True), ('empty-quote', 'Customers loved it.', 'S3', '', False), ] results = [] for name, claim, source_id, quote, supported in cases: exists = source_id in sources provenance_pass = exists and bool(quote.strip()) and quote in sources[source_id] results.append(dict(case=name, claim=claim, source_id=source_id, quote=quote, source_exists=exists, provenance_pass=provenance_pass, editorially_supported=supported)) return dict(sources=sources, cases=results, total=len(results), provenance_pass=sum(r['provenance_pass'] for r in results), supported=sum(r['editorially_supported'] for r in results), false_accepts=sum(r['provenance_pass'] and not r['editorially_supported'] for r in results)) def redaction_lab(): pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b') # All identities and addresses are fictional. expected_spans defines this fixture's scope. cases = [ ('simple', 'Email ada@example.com.', ['ada@example.com']), ('plus', 'Email ada+lab@example.com.', ['ada+lab@example.com']), ('subdomain', 'Email ada@team.example.com.', ['ada@team.example.com']), ('uppercase', 'Email ADA@EXAMPLE.COM.', ['ADA@EXAMPLE.COM']), ('two', 'ada@example.com and ben@example.org', ['ada@example.com', 'ben@example.org']), ('obfuscated', 'ada [at] example [dot] com', ['ada [at] example [dot] com']), ('unicode-local', '민지@example.com', ['민지@example.com']), ('phone', 'Call +1 202 555 0147.', ['+1 202 555 0147']), ('identifier', 'Customer CUST-0482 asked for help.', ['CUST-0482']), ('ordinary', 'The build finished normally.', []), ] results = [] for name, source, spans in cases: output = pattern.sub('[EMAIL]', source) remaining = [span for span in spans if span in output] results.append(dict(case=name, input=source, expected_spans=spans, output=output, remaining_spans=remaining, pass_case=not remaining)) return dict(pattern=pattern.pattern, cases=results, total=len(results), pass_cases=sum(r['pass_case'] for r in results), sensitive_cases=sum(bool(r['expected_spans']) for r in results), completely_redacted_sensitive_cases=sum(bool(r['expected_spans']) and r['pass_case'] for r in results)) def retrieval_lab(): paragraphs = [ 'Trial workspace: exports are available for seven days. After that window, contact support for recovery options.', 'Paid workspace: exports are available for thirty days. Administrators can disable exports for a workspace.', 'Guest access: guests can read shared notes. Guests cannot export a workspace.', 'Deletion: deleting a workspace starts a fourteen day recovery period. Export availability does not extend this recovery period.', ] queries = [ ('trial', 'trial exports available days', 'Trial workspace: exports are available for seven days.'), ('paid', 'paid exports available days', 'Paid workspace: exports are available for thirty days.'), ('guest', 'guest export workspace', 'Guests cannot export a workspace.'), ('deletion', 'deleting workspace recovery period', 'deleting a workspace starts a fourteen day recovery period.'), ] document = '\n\n'.join(paragraphs) def windows(size, overlap): return [document[i:i + size] for i in range(0, len(document), size - overlap)] strategies = {'fixed-80': windows(80, 0), 'overlap-80-30': windows(80, 30), 'paragraph': paragraphs} def tokens(value): return set(re.findall(r'[a-z]+', value.lower())) results = [] for strategy, chunks in strategies.items(): for name, query, target in queries: scored = sorted(enumerate(chunks), key=lambda pair: (-len(tokens(query) & tokens(pair[1])), pair[0])) chosen = scored[0][1] results.append(dict(strategy=strategy, case=name, query=query, target=target, selected_chunk=chosen, pass_case=target.lower() in chosen.lower())) return dict(document=document, strategies=strategies, cases=results, scores={s: dict(pass_cases=sum(r['pass_case'] for r in results if r['strategy'] == s), total=len(queries)) for s in strategies}) def main(): parser = argparse.ArgumentParser() parser.add_argument('--output', default='results.json') args = parser.parse_args() report = dict(executed_at=datetime.now(timezone.utc).isoformat(), python=platform.python_version(), method='Deterministic synthetic-fixture tests; no external model, private data, or network calls.', csv=csv_lab(), evidence=evidence_lab(), redaction=redaction_lab(), retrieval=retrieval_lab()) Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps({k: {x:y for x,y in v.items() if x in ['total','split_pass','parser_pass','provenance_pass','supported','false_accepts','pass_cases','sensitive_cases','completely_redacted_sensitive_cases','scores']} for k,v in report.items() if isinstance(v,dict)}, indent=2)) if __name__ == '__main__': main()