A literate fictional measurement study
Explain the program. Rebuild the result.

<<validation-setup>>
<<validate-input>>

<<classify-measurements>>

<<select-usable-measurements>>

<<summarize-by-group>>
<<ordered-groups>>

<<plot-results>>

<<analysis-contract>>

<<report-template>>

<<report-style>>

<<workflow>>

<<pipeline-utilities>>
<<execute-stages>>
<<evaluate-explicitly>>
<<render-native-figure>>
<<propagate-provenance>>
<<weave-explanation>>
<<verify-handover>>
<<pipeline-entry>>

print "group","n_total","n_used","n_missing","n_rejected","mean"

if (total[g]!=used[g]+missing[g]+rejected[g]) {
 print "handover error: count conservation failed for " g > "/dev/stderr"; exit 3
}
printf "%s\t%d\t%d\t%d\t%d\t%s\n",g,total[g],used[g],missing[g],rejected[g],(used[g] ? sprintf("%.2f",sum[g]/used[g]) : "NA")

BEGIN { FS=OFS="\t" }
NR==1 { print "sample_id","group","value","category"; next }
{
 category=($4=="invalid" ? "rejected" : ($3=="NA" ? "missing" : "used"))
 print $1,$2,$3,category
}

BEGIN { FS=OFS="\t" }
NR==1 { print "sample_id","group","value"; next }
$4=="used" { print $1,$2,$3 }

BEGIN { FS=OFS="\t" }
FILENAME==ARGV[1] {
 if (FNR==1) next
 g=$2
 if (!(g in total)) groups[++ng]=g
 total[g]++
 if ($4=="missing") missing[g]++
 if ($4=="rejected") rejected[g]++
 next
}
FILENAME==ARGV[2] {
 if (FNR==1) next
 g=$2
 used[g]++; sum[g]+=$3
}

END {
 <<summary-header>>
 for (i=1;i<=ng;i++) for (j=i+1;j<=ng;j++) if (groups[j]<groups[i]) { t=groups[i];groups[i]=groups[j];groups[j]=t }
 for (i=1;i<=ng;i++) {
  g=groups[i]
  <<report-exclusions>>
 }
}

BEGIN { FS=OFS="\t"; number="^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$" }
function fail(message) { print "input error: " message > "/dev/stderr"; failed=1; exit 2 }

NR==1 {
 if ($0!="sample_id\tgroup\tvalue\tstatus") fail("line 1: expected sample_id, group, value, status TSV header")
 print; next
}
{
 if (NF!=4) fail("line " NR ": expected 4 tab-separated fields, got " NF)
 if ($1 !~ /^[A-Za-z][A-Za-z0-9_-]*$/) fail("line " NR ": invalid sample_id")
 if (seen[$1]++) fail("line " NR ": duplicate sample_id " $1)
 if ($2 !~ /^[A-Z][A-Z0-9_]*$/) fail("line " NR ": group must be an uppercase ASCII identifier")
 if ($4!="valid" && $4!="invalid") fail("line " NR ": unexpected status " $4)
 if ($3!="NA" && $3 !~ number) fail("line " NR ": value must be a finite decimal number or NA")
 if ($3!="NA" && (($3+0)>1000000 || ($3+0)<-1000000)) fail("line " NR ": absolute value exceeds 1000000 a.u.")
 print
}
END { if (!failed && NR==0) fail("empty input: expected TSV header") }

{
  "units": "a.u.",
  "missing": "NA",
  "status": [
    "valid",
    "invalid"
  ],
  "exclusion_priority": [
    "invalid",
    "NA"
  ],
  "locale": "C",
  "output_order": "ASCII group ascending",
  "number_format": "finite decimal/scientific; abs(value)<=1000000",
  "mean_format": "two decimal places; NA when n_used=0",
  "empty_input": "zero bytes: error 2; header only: no groups"
}

set datafile separator "\t"
set datafile missing "NA"
set terminal svg size 740,380 font "DejaVu Sans,12" enhanced
set output SVG
set title "Group means — fictional teaching data"
set xlabel "Measurement group"
set ylabel "Mean (a.u.)"
set boxwidth 0.55
set style fill solid 0.8 border rgb "#00549f"
set grid ytics lc rgb "#d8e3ec"
set key off
if (GROUPS>0) {
 stats SUMMARY using (column("n_used")>0 ? column("mean") : 0) nooutput
 low=(STATS_min<0 ? STATS_min*1.25 : 0)
 high=(STATS_max>0 ? STATS_max*1.3 : 1)
 if (low==high) { low=low-1; high=high+1 }
 set yrange [low:high]
 set xrange [-0.6:GROUPS-0.4]
 plot SUMMARY using 0:(column("mean")):xticlabels(1) with boxes lc rgb "#00549f", \
      SUMMARY using 0:(column("n_used")>0 ? column("mean") : 0):(column("n_used")>0 ? sprintf("n=%d",column("n_used")) : "n=0; mean=NA") with labels offset 0,1.1 tc rgb "#087f88"
} else {
 set xrange [0:1]; set yrange [0:1]; unset xtics; unset ytics
 set label "No groups: header-only input" at graph 0.5,0.5 center
 plot NaN
}
set terminal pdfcairo size 16cm,8.5cm font "DejaVu Sans,11"
set output PDF
replot
unset output

.DEFAULT_GOAL := all
.RECIPEPREFIX := >
.DELETE_ON_ERROR:
.SUFFIXES:
PYTHON ?= python3
AWK ?= awk
RUN = $(PYTHON) build/tools/pipeline.py
.PHONY: all results check figure provenance report
all: report
results: build/results/summary.tsv
check: build/checks.json
figure: build/results/figure.svg build/results/figure.pdf
provenance: build/provenance.json
report: build/site/index.html build/site/report.pdf
build/results/validated.tsv: data/measurements.tsv build/raw.sha256 build/environment.json build/programs/validate.awk build/tools/pipeline.py
>@$(RUN) stage validate $(AWK)
build/results/classified.tsv: build/results/validated.tsv build/programs/classify.awk build/tools/pipeline.py
>@$(RUN) stage classify $(AWK)
build/results/usable.tsv: build/results/classified.tsv build/programs/select.awk build/tools/pipeline.py
>@$(RUN) stage select $(AWK)
build/results/summary.tsv: build/results/classified.tsv build/results/usable.tsv build/programs/summarize.awk build/tools/pipeline.py
>@$(RUN) stage summarize $(AWK)
build/checks.json: build/results/summary.tsv tests/expected-summary.tsv tests/cases.json $(wildcard tests/fixtures/*.tsv) $(wildcard build/programs/*.awk) build/tools/pipeline.py
>@$(RUN) check $(AWK)
build/results/figure.svg build/results/figure.pdf &: build/results/summary.tsv build/checks.json build/programs/plot.gp build/tools/pipeline.py
>@$(RUN) plot
build/provenance.json: build/results/figure.svg build/results/figure.pdf build/checks.json build/source.json build/environment.json build/invocation.json build/tangle.json $(wildcard build/programs/*) build/workflow.mk build/tools/pipeline.py
>@$(RUN) provenance
build/site/index.html build/site/report.pdf &: build/provenance.json build/programs/report-template.org build/programs/report.css build/tools/pipeline.py bootstrap.el
>@$(RUN) report

import csv,hashlib,html,json,os,re,shutil,subprocess,sys,tempfile
from urllib.parse import unquote,urlsplit
from pathlib import Path
ROOT=Path.cwd()
KEYS=['group','n_total','n_used','n_missing','n_rejected','mean']
def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def read(path):
 with open(path,newline='') as f:return list(csv.DictReader(f,delimiter='\t'))
def write(path,data):
 path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
 data=data.encode() if isinstance(data,str) else data
 tmp=path.with_name(path.name+'.tmp');tmp.write_bytes(data);os.replace(tmp,path)
def js(path,value):write(path,json.dumps(value,sort_keys=True,indent=2)+'\n')
def event(name):
 with open('build/events.log','a') as f:f.write(name+'\n')
def run(argv,expected=0):
 p=subprocess.run(argv,capture_output=True,text=True)
 if p.returncode!=expected:raise RuntimeError(p.stderr.strip() or 'command failed: '+' '.join(argv))
 return p
def receipt(name,commands,outputs,operations=None):
 js('build/receipts/'+name+'.json',{'commands':commands,'operations':operations or [],'exit':0,'outputs':{p:sha(p) for p in outputs}})
 event(name)
def awk_pipeline(raw,directory,awk,commands=None):
 paths=[]
 for step,program in [('validated','validate'),('classified','classify'),('usable','select'),('summary','summarize')]:
  inputs=[str(raw)] if not paths else [str(paths[0])] if program=='classify' else [str(paths[1])] if program=='select' else [str(paths[1]),str(paths[2])]
  argv=[awk,'-f','build/programs/'+program+'.awk',*inputs]
  if commands is not None:commands.append(argv)
  p=subprocess.run(argv,capture_output=True,text=True)
  if p.returncode:return p,paths
  target=Path(directory)/(step+'.tsv');target.write_text(p.stdout);paths.append(target)
 return p,paths

def stage(name,awk):
 plans={'validate':('validate',['data/measurements.tsv'],'validated'),
        'classify':('classify',['build/results/validated.tsv'],'classified'),
        'select':('select',['build/results/classified.tsv'],'usable'),
        'summarize':('summarize',['build/results/classified.tsv','build/results/usable.tsv'],'summary')}
 program,inputs,output=plans[name];argv=[awk,'-f','build/programs/'+program+'.awk',*inputs]
 p=run(argv);target='build/results/'+output+'.tsv';write(target,p.stdout)
 receipt(name,[argv],[target])

def check(awk):
 expected=Path('tests/expected-summary.tsv').read_bytes()
 if Path('build/results/summary.tsv').read_bytes()!=expected:raise RuntimeError('reference summary differs from authored tests/expected-summary.tsv')
 cases=json.loads(Path('tests/cases.json').read_text());outcomes=[];commands=[]
 for case in cases:
  if not re.fullmatch('[a-z0-9-]+',case['name']):raise RuntimeError('invalid test case name')
  directory=Path('build/tests')/case['name'];directory.mkdir(parents=True,exist_ok=True)
  p,paths=awk_pipeline(case['file'],directory,awk,commands)
  if p.returncode!=case['exit']:raise RuntimeError(case['name']+': wrong error status')
  if case['error'] and p.stderr.strip()!='input error: '+case['error']:raise RuntimeError(case['name']+': wrong diagnostic')
  if not case['exit']:
   actual=[{k:r[k] for k in KEYS} for r in read(paths[-1])]
   if actual!=case['rows']:raise RuntimeError(case['name']+': incorrect scientific result')
   for row in actual:
    if int(row['n_total'])!=sum(int(row[k]) for k in ('n_used','n_missing','n_rejected')):raise RuntimeError('count conservation failed')
  outcomes.append({'case':case['name'],'status':p.returncode,'passed':True})
 js('build/checks.json',{'reference':'passed','cases':outcomes,'count_conservation':'passed','scope':'native AWK reference and specified edge cases; no cross-platform byte claim'})
 receipt('check',commands,['build/checks.json'],[{'api':'Python bytes equality','expected':'tests/expected-summary.tsv','actual':'build/results/summary.tsv'}])

def plot():
 groups=len(read('build/results/summary.tsv'))
 arguments="SUMMARY='build/results/summary.tsv';SVG='build/results/figure.svg';PDF='build/results/figure.pdf';GROUPS="+str(groups)
 argv=['gnuplot','-e',arguments,'build/programs/plot.gp'];run(argv)
 if not Path('build/results/figure.svg').is_file() or not Path('build/results/figure.pdf').is_file():raise RuntimeError('native figure missing')
 receipt('plot',[argv],['build/results/figure.svg','build/results/figure.pdf'])

def provenance():
 tangle=json.loads(Path('build/tangle.json').read_text());source=json.loads(Path('build/source.json').read_text())
 inputs=['data/measurements.tsv','tests/expected-summary.tsv','tests/cases.json']+sorted(str(p) for p in Path('tests/fixtures').glob('*.tsv'))
 programs=['build/'+p for p in sorted(tangle['files'])]
 results=sorted(str(p) for p in Path('build/results').glob('*'))+['build/checks.json']
 receipts={p.stem:json.loads(p.read_text()) for p in sorted(Path('build/receipts').glob('*.json')) if p.stem in {'validate','classify','select','summarize','check','plot'}}
 record={'schema':1,'question':'Group mean of usable fictional measurements','parameters':json.loads(Path('build/programs/contract.json').read_text()),
  'inputs':{p:sha(p) for p in inputs},'sources':{p:sha(p) for p in ['research.org','build.py','bootstrap.el']},
  'programs':{p:sha(p) for p in programs},'results':{p:sha(p) for p in results},
  'versions':json.loads(Path('build/environment.json').read_text()),'git':source['git'],
  'execution':receipts,'tangle':tangle['command'],'invocation':json.loads(Path('build/invocation.json').read_text()),
  'scope':'Same scientific tables for the specified input and recorded environment. Manifest excludes itself, receipts generated after it, and the downstream reports.'}
 js('build/provenance.json',record);receipt('provenance',[],['build/provenance.json'])

#+TITLE: Fictional measurement study
#+SUBTITLE: Explain the program. Rebuild the result.
#+LANGUAGE: en
#+OPTIONS: toc:nil num:nil author:nil date:nil creator:nil html-postamble:nil ^:nil
#+HTML_HEAD: <link rel="stylesheet" href="report.css">

* Question and decision
What is the mean of each group's usable measurements, in arbitrary units?
These are fictional teaching data, not observations supporting a scientific claim.
A recorded zero is usable. NA is missing. Invalid status takes precedence over
missingness. No usable observations means NA, never an invented zero mean.

* Results
{{TABLE}}

#+ATTR_HTML: :alt Group means from fictional data with used sample counts :width 740
[[file:results/figure.svg]]

The n labels count used measurements. A group without usable values has no bar
and is explicitly labeled n=0; mean=NA. Exclusion categories do not overlap.

* Evidence and handover
:PROPERTIES:
:CUSTOM_ID: evidence
:END:
ROOT the tools. SHAPE the workflow.
The reference and edge cases passed: {{CHECKS}}. Data conservation is checked
for every group: n_total = n_used + n_missing + n_rejected.
The source revision is {{REVISION}}; its recorded dirty status is {{DIRTY}}.

- [[file:research.org][Read the complete literate source]]
- [[file:data/measurements.tsv][Read the original TSV input]]
- [[file:results/summary.tsv][Download the summary TSV]]
- [[file:provenance.json][Inspect the machine-readable provenance]]
- [[file:programs/validate.awk][Reuse input validation]]
- [[file:programs/summarize.awk][Reuse the group statistic program]]

The manifest hashes inputs, source, generated programs and checked scientific
results. It does not hash itself or this report: manifest precedes report.
Identical scientific tables are tested in this recorded environment. PDF bytes
can differ with fonts, tool versions and timestamps; no cross-platform byte
identity is claimed. Source, raw TSV and manifest are attached to the PDF.

:root{--publisher-blue:#00549f;--publisher-teal:#087f88;--publisher-ink:#173147;--publisher-line:#d8e3ec}
body{font:15px/1.5 "DejaVu Sans",sans-serif;color:var(--publisher-ink);max-width:980px;margin:35px auto;padding:0 20px}
h1,h2{color:var(--publisher-blue)}a{color:var(--publisher-teal)}table{border-collapse:collapse;width:100%}td,th{border-bottom:1px solid var(--publisher-line);padding:6px 10px;text-align:right}td:first-child,th:first-child{text-align:left}img{max-width:100%;height:auto}
@page{size:A4;margin:16mm;@bottom-right{content:counter(page);font:9pt sans-serif}}
@media print{#outline-container-evidence{break-before:page}body{font-size:10pt;margin:0;padding:0}h1{font-size:23pt}h2{font-size:15pt;margin:4mm 0 2mm}p{margin:2mm 0}table{font-size:9pt}img{max-height:78mm}a{overflow-wrap:anywhere}.figure{margin:3mm 0}#postamble{display:none}}

def report():
 import pymupdf
 from weasyprint import HTML
 site=Path('build/site');site.mkdir(exist_ok=True)
 for source in ['research.org','data/measurements.tsv','build/provenance.json','build/programs/report.css']:
  target=site/('report.css' if source.endswith('.css') else source.removeprefix('build/'))
  target.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(source,target)
 for folder in ['results','programs']:
  shutil.copytree(Path('build')/folder,site/folder,dirs_exist_ok=True)
 rows=list(csv.reader(open('build/results/summary.tsv'),delimiter='\t'))
 table='\n'.join('| '+' | '.join(row)+' |' for row in rows[:1])+'\n|'+ '+'.join('-'*max(3,len(v)) for v in rows[0])+'|\n'+'\n'.join('| '+' | '.join(row)+' |' for row in rows[1:])
 checks=json.loads(Path('build/checks.json').read_text());proof=json.loads(Path('build/provenance.json').read_text())
 text=Path('build/programs/report-template.org').read_text().replace('{{TABLE}}',table).replace('{{CHECKS}}',str(1+len(checks['cases']))+' native checks').replace('{{REVISION}}',proof['git']['commit'] or 'not in Git').replace('{{DIRTY}}',str(proof['git']['dirty']))
 write(site/'report.org',text)
 argv=['emacs','--batch','-Q','--script','bootstrap.el','weave','build/site/report.org','build/site/index.html'];run(argv)
 HTML(filename=str(site/'index.html')).write_pdf(site/'report.pdf')
 with pymupdf.open(site/'report.pdf') as pdf:
  for file in ['research.org','data/measurements.tsv','provenance.json']:pdf.embfile_add(file,(site/file).read_bytes(),filename=file)
  for page in pdf:
   for link in page.get_links():
    filename=link.get('file','') or (urlsplit(link.get('uri','')).path if link.get('uri','').startswith('file:') else '')
    filename=unquote(filename)
    if filename and Path(filename).is_absolute() and Path(filename).is_relative_to(site.resolve()):page.update_link({'kind':pymupdf.LINK_LAUNCH,'xref':link['xref'],'from':link['from'],'file':str(Path(filename).relative_to(site.resolve()))})
  pdf.save(site/'attached.pdf',garbage=4,deflate=True)
 (site/'attached.pdf').replace(site/'report.pdf')
 receipt('report',[argv],['build/site/index.html','build/site/report.pdf'],[{'api':'weasyprint.HTML.write_pdf','input':'build/site/index.html','output':'build/site/report.pdf'}])

def verify():
 record=json.loads(Path('build/provenance.json').read_text())
 for role in ['inputs','sources','programs','results']:
  for file,expected in record[role].items():
   if not Path(file).is_file() or sha(file)!=expected:raise RuntimeError('provenance mismatch: '+file)
 if 'build/provenance.json' in record['results']:raise RuntimeError('manifest self-hash is forbidden')
 for file,expected in json.loads(Path('build/receipts/report.json').read_text())['outputs'].items():
  if sha(file)!=expected:raise RuntimeError('report receipt mismatch: '+file)
 print('verify: input, source, program, result and report hashes passed')

if __name__=='__main__':
 try:
  command=sys.argv[1]
  if command=='stage':stage(sys.argv[2],sys.argv[3])
  elif command=='check':check(sys.argv[2])
  elif command=='plot':plot()
  elif command=='provenance':provenance()
  elif command=='report':report()
  elif command=='verify':verify()
  else:raise RuntimeError('unknown pipeline command')
 except (OSError,RuntimeError,ValueError,KeyError) as error:raise SystemExit('study: '+str(error))