"""Verify archive integrity, global uniqueness and correspondence to source records."""
import argparse,gzip,hashlib,json,re,subprocess,unicodedata
from pathlib import Path
parser=argparse.ArgumentParser();parser.add_argument('--source-cache',type=Path,required=True);args=parser.parse_args()
root=json.loads(Path('data/kaomoji-manifest.json').read_text());archive=json.loads(Path(root['archive']['index']).read_text())
def norm(s):return ''.join(unicodedata.normalize('NFC',s).replace('\ufeff','').replace('\u200b','').replace('\u2060','').split())
def digest(s):return hashlib.sha256(norm(s).encode()).digest()
curated=json.loads(subprocess.check_output(['node','--input-type=module','-e',"import {curated} from './catalog.js';console.log(JSON.stringify(curated))"]))
seen={digest(x[0]) for x in curated+json.loads(Path('data/emoji.json').read_text())+json.loads(Path('data/unicode.json').read_text())}
for file in root['files']:
 for x in json.loads(Path(file).read_text()):
  key=digest(x[0]);assert key not in seen,'Existing-source duplicate';seen.add(key)
count=0;family_ids=set();part_files=set()
for family in archive['families']:
 assert family['id'] not in family_ids;family_ids.add(family['id']);raw=(args.source_cache/(family['id']+'.json.gz')).read_bytes()
 assert hashlib.md5(raw).hexdigest()==family['compressed_md5'];assert hashlib.sha256(raw).hexdigest()==family['source_sha256']
 source=json.loads(gzip.decompress(raw) if raw[:2]==b'\x1f\x8b' else raw);assert len(source['aa'])==family['source_rows'];local=0
 for part in family['parts']:
  assert part['file'] not in part_files;part_files.add(part['file']);assert part['start']==local
  encoded=Path(part['file']).read_bytes();assert len(encoded)==part['bytes'];assert hashlib.sha256(encoded).hexdigest()==part['sha256'];assert len(encoded)<25*1024*1024
  rows=json.loads(encoded);assert len(rows)==part['count']
  for row in rows:
   assert row[2]=='art' and row[5]=='sjis';match=re.fullmatch('aahub_'+family['id']+r'_(\d+)',row[4]);assert match
   source_row=source['aa'][int(match[1])]['value'];assert norm(source_row)==norm(row[0]),'Artwork does not match original source'
   key=digest(row[0]);assert key not in seen,'Archive duplicate';seen.add(key);local+=1;count+=1
 assert local==family['count']
assert count==archive['count']==root['archive']['count']==73079
assert root['base_count']+root['accepted']+count==250000
assert root['total']==root['collection_target']==250000+root.get('reference',{}).get('new_count',0)
assert root['collection_stopped'] is True
print(json.dumps({'verified_total':root['total'],'new_archive_artworks':count,'themes':len(family_ids),'verified_parts':len(part_files),'global_uniqueness':'passed','source_record_correspondence':'passed','baseline_count':250000,'collection':'bulk collection remains stopped; targeted reference extension accounted separately'}))