import csv,hashlib,html,json,os,re,shutil,subprocess,sys,tempfile
from urllib.parse import unquote,urlsplit
from pathlib import Path
ROOT=Path.cwd()
KEYS=['group','n_total','n_used','n_missing','n_rejected','mean']
def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def read(path):
 with open(path,newline='') as f:return list(csv.DictReader(f,delimiter='\t'))
def write(path,data):
 path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
 data=data.encode() if isinstance(data,str) else data
 tmp=path.with_name(path.name+'.tmp');tmp.write_bytes(data);os.replace(tmp,path)
def js(path,value):write(path,json.dumps(value,sort_keys=True,indent=2)+'\n')
def event(name):
 with open('build/events.log','a') as f:f.write(name+'\n')
def run(argv,expected=0):
 p=subprocess.run(argv,capture_output=True,text=True)
 if p.returncode!=expected:raise RuntimeError(p.stderr.strip() or 'command failed: '+' '.join(argv))
 return p
def receipt(name,commands,outputs,operations=None):
 js('build/receipts/'+name+'.json',{'commands':commands,'operations':operations or [],'exit':0,'outputs':{p:sha(p) for p in outputs}})
 event(name)
def awk_pipeline(raw,directory,awk,commands=None):
 paths=[]
 for step,program in [('validated','validate'),('classified','classify'),('usable','select'),('summary','summarize')]:
  inputs=[str(raw)] if not paths else [str(paths[0])] if program=='classify' else [str(paths[1])] if program=='select' else [str(paths[1]),str(paths[2])]
  argv=[awk,'-f','build/programs/'+program+'.awk',*inputs]
  if commands is not None:commands.append(argv)
  p=subprocess.run(argv,capture_output=True,text=True)
  if p.returncode:return p,paths
  target=Path(directory)/(step+'.tsv');target.write_text(p.stdout);paths.append(target)
 return p,paths
def stage(name,awk):
 plans={'validate':('validate',['data/measurements.tsv'],'validated'),
        'classify':('classify',['build/results/validated.tsv'],'classified'),
        'select':('select',['build/results/classified.tsv'],'usable'),
        'summarize':('summarize',['build/results/classified.tsv','build/results/usable.tsv'],'summary')}
 program,inputs,output=plans[name];argv=[awk,'-f','build/programs/'+program+'.awk',*inputs]
 p=run(argv);target='build/results/'+output+'.tsv';write(target,p.stdout)
 receipt(name,[argv],[target])
def check(awk):
 expected=Path('tests/expected-summary.tsv').read_bytes()
 if Path('build/results/summary.tsv').read_bytes()!=expected:raise RuntimeError('reference summary differs from authored tests/expected-summary.tsv')
 cases=json.loads(Path('tests/cases.json').read_text());outcomes=[];commands=[]
 for case in cases:
  if not re.fullmatch('[a-z0-9-]+',case['name']):raise RuntimeError('invalid test case name')
  directory=Path('build/tests')/case['name'];directory.mkdir(parents=True,exist_ok=True)
  p,paths=awk_pipeline(case['file'],directory,awk,commands)
  if p.returncode!=case['exit']:raise RuntimeError(case['name']+': wrong error status')
  if case['error'] and p.stderr.strip()!='input error: '+case['error']:raise RuntimeError(case['name']+': wrong diagnostic')
  if not case['exit']:
   actual=[{k:r[k] for k in KEYS} for r in read(paths[-1])]
   if actual!=case['rows']:raise RuntimeError(case['name']+': incorrect scientific result')
   for row in actual:
    if int(row['n_total'])!=sum(int(row[k]) for k in ('n_used','n_missing','n_rejected')):raise RuntimeError('count conservation failed')
  outcomes.append({'case':case['name'],'status':p.returncode,'passed':True})
 js('build/checks.json',{'reference':'passed','cases':outcomes,'count_conservation':'passed','scope':'native AWK reference and specified edge cases; no cross-platform byte claim'})
 receipt('check',commands,['build/checks.json'],[{'api':'Python bytes equality','expected':'tests/expected-summary.tsv','actual':'build/results/summary.tsv'}])
def plot():
 groups=len(read('build/results/summary.tsv'))
 arguments="SUMMARY='build/results/summary.tsv';SVG='build/results/figure.svg';PDF='build/results/figure.pdf';GROUPS="+str(groups)
 argv=['gnuplot','-e',arguments,'build/programs/plot.gp'];run(argv)
 if not Path('build/results/figure.svg').is_file() or not Path('build/results/figure.pdf').is_file():raise RuntimeError('native figure missing')
 receipt('plot',[argv],['build/results/figure.svg','build/results/figure.pdf'])
def provenance():
 tangle=json.loads(Path('build/tangle.json').read_text());source=json.loads(Path('build/source.json').read_text())
 inputs=['data/measurements.tsv','tests/expected-summary.tsv','tests/cases.json']+sorted(str(p) for p in Path('tests/fixtures').glob('*.tsv'))
 programs=['build/'+p for p in sorted(tangle['files'])]
 results=sorted(str(p) for p in Path('build/results').glob('*'))+['build/checks.json']
 receipts={p.stem:json.loads(p.read_text()) for p in sorted(Path('build/receipts').glob('*.json')) if p.stem in {'validate','classify','select','summarize','check','plot'}}
 record={'schema':1,'question':'Group mean of usable fictional measurements','parameters':json.loads(Path('build/programs/contract.json').read_text()),
  'inputs':{p:sha(p) for p in inputs},'sources':{p:sha(p) for p in ['research.org','build.py','bootstrap.el']},
  'programs':{p:sha(p) for p in programs},'results':{p:sha(p) for p in results},
  'versions':json.loads(Path('build/environment.json').read_text()),'git':source['git'],
  'execution':receipts,'tangle':tangle['command'],'invocation':json.loads(Path('build/invocation.json').read_text()),
  'scope':'Same scientific tables for the specified input and recorded environment. Manifest excludes itself, receipts generated after it, and the downstream reports.'}
 js('build/provenance.json',record);receipt('provenance',[],['build/provenance.json'])
def report():
 import pymupdf
 from weasyprint import HTML
 site=Path('build/site');site.mkdir(exist_ok=True)
 for source in ['research.org','data/measurements.tsv','build/provenance.json','build/programs/report.css']:
  target=site/('report.css' if source.endswith('.css') else source.removeprefix('build/'))
  target.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(source,target)
 for folder in ['results','programs']:
  shutil.copytree(Path('build')/folder,site/folder,dirs_exist_ok=True)
 rows=list(csv.reader(open('build/results/summary.tsv'),delimiter='\t'))
 table='\n'.join('| '+' | '.join(row)+' |' for row in rows[:1])+'\n|'+ '+'.join('-'*max(3,len(v)) for v in rows[0])+'|\n'+'\n'.join('| '+' | '.join(row)+' |' for row in rows[1:])
 checks=json.loads(Path('build/checks.json').read_text());proof=json.loads(Path('build/provenance.json').read_text())
 text=Path('build/programs/report-template.org').read_text().replace('{{TABLE}}',table).replace('{{CHECKS}}',str(1+len(checks['cases']))+' native checks').replace('{{REVISION}}',proof['git']['commit'] or 'not in Git').replace('{{DIRTY}}',str(proof['git']['dirty']))
 write(site/'report.org',text)
 argv=['emacs','--batch','-Q','--script','bootstrap.el','weave','build/site/report.org','build/site/index.html'];run(argv)
 HTML(filename=str(site/'index.html')).write_pdf(site/'report.pdf')
 with pymupdf.open(site/'report.pdf') as pdf:
  for file in ['research.org','data/measurements.tsv','provenance.json']:pdf.embfile_add(file,(site/file).read_bytes(),filename=file)
  for page in pdf:
   for link in page.get_links():
    filename=link.get('file','') or (urlsplit(link.get('uri','')).path if link.get('uri','').startswith('file:') else '')
    filename=unquote(filename)
    if filename and Path(filename).is_absolute() and Path(filename).is_relative_to(site.resolve()):page.update_link({'kind':pymupdf.LINK_LAUNCH,'xref':link['xref'],'from':link['from'],'file':str(Path(filename).relative_to(site.resolve()))})
  pdf.save(site/'attached.pdf',garbage=4,deflate=True)
 (site/'attached.pdf').replace(site/'report.pdf')
 receipt('report',[argv],['build/site/index.html','build/site/report.pdf'],[{'api':'weasyprint.HTML.write_pdf','input':'build/site/index.html','output':'build/site/report.pdf'}])
def verify():
 record=json.loads(Path('build/provenance.json').read_text())
 for role in ['inputs','sources','programs','results']:
  for file,expected in record[role].items():
   if not Path(file).is_file() or sha(file)!=expected:raise RuntimeError('provenance mismatch: '+file)
 if 'build/provenance.json' in record['results']:raise RuntimeError('manifest self-hash is forbidden')
 for file,expected in json.loads(Path('build/receipts/report.json').read_text())['outputs'].items():
  if sha(file)!=expected:raise RuntimeError('report receipt mismatch: '+file)
 print('verify: input, source, program, result and report hashes passed')
if __name__=='__main__':
 try:
  command=sys.argv[1]
  if command=='stage':stage(sys.argv[2],sys.argv[3])
  elif command=='check':check(sys.argv[2])
  elif command=='plot':plot()
  elif command=='provenance':provenance()
  elif command=='report':report()
  elif command=='verify':verify()
  else:raise RuntimeError('unknown pipeline command')
 except (OSError,RuntimeError,ValueError,KeyError) as error:raise SystemExit('study: '+str(error))