# SPDX-License-Identifier: AGPL-3.0-only
"""Add inspected traditional sources to the catalogue, with cross-source deduplication.
Photo-to-ASCII datasets and generated permutations are not accepted inputs.
"""
import argparse,collections,hashlib,json,re,subprocess,unicodedata
from pathlib import Path
parser=argparse.ArgumentParser();parser.add_argument('--tagged',type=Path);parser.add_argument('--traditional',type=Path);parser.add_argument('--captions',type=Path);args=parser.parse_args()
manifest_path=Path('data/kaomoji-manifest.json');manifest=json.loads(manifest_path.read_text());
if manifest.get('collection_stopped'):raise SystemExit('Collection is stopped at its target; no further imports are enabled.')
pending=json.loads(Path('docs/research/pending-sources.json').read_text())
records=[x for f in manifest['files'] for x in json.loads(Path(f).read_text())]
curated=json.loads(subprocess.check_output(['node','--input-type=module','-e',"import {curated} from './catalog.js';console.log(JSON.stringify(curated))"]))
base=curated+json.loads(Path('data/emoji.json').read_text())+json.loads(Path('data/unicode.json').read_text())
def normalize(text):return ''.join(unicodedata.normalize('NFC',text).replace('\ufeff','').replace('\u200b','').replace('\u2060','').split())
seen={normalize(x[0]) for x in base+records};sources=manifest.setdefault('supplemental_sources',[])
translations={'happy':'开心','smiling':'微笑 开心','sad':'伤心','crying':'哭泣 伤心','angry':'生气','love':'喜欢','heart':'爱心','kissing':'亲吻 喜欢','blushing':'害羞 喜欢','sleepy':'睡觉','sleeping':'睡觉','tired':'困倦','waving':'问候','bear':'小动物 熊','cat':'小动物 猫','dog':'小动物 狗','rabbit':'小动物 兔','surprised':'惊讶 困惑','confused':'困惑','shrug':'无奈','excited':'兴奋 加油','table_flip':'翻桌 无奈','flower':'花朵','hug':'拥抱'}
def verify(path,expected):
 raw=path.read_bytes()
 if len(raw)!=expected['size'] or hashlib.sha256(raw).hexdigest()!=expected['sha256']:raise ValueError('Unverified source file: '+str(path))
 return raw
if args.tagged:
 source=pending['sources'][0];raw=verify(args.tagged,source['files'][0]);identity=source['revision']
 if not any(x['source']==source['name'] and x['revision']==identity for x in sources):
  rows=[json.loads(line) for line in raw.decode().splitlines()];stats=collections.Counter();rows.sort(key=lambda x:x.get('quality_score',0),reverse=True)
  for row in rows:
   text=row['kaomoji'].strip();key=normalize(text)
   if key in seen:stats['duplicates']+=1;continue
   if not key or any(unicodedata.category(c)=='Cc' for c in text):stats['invalid']+=1;continue
   seen.add(key);tags=row.get('tags',[]);zh=list(dict.fromkeys(translations[t] for t in tags if t in translations));label=' · '.join(zh[:3] or tags[:3]) or '颜文字'
   records.append([text,label,'kaomoji',' '.join([*zh,*tags]),'tagged_'+row['id'],'']);stats['accepted']+=1
  sources.append({'source':source['name'],'revision':identity,'license':'MIT (Kare-Udon); upstream ekohrt attribution retained','source_rows':len(rows),'sha256':source['files'][0]['sha256'],**stats,'changes':'Preserved kaomoji text; deduplicated NFC/whitespace; compacted and translated tags; labels are upstream LLM-assisted annotations, not generated kaomoji.'})
if args.traditional:
 import pyarrow.parquet as pq
 source=pending['sources'][1];identity=source['revision'];stats=collections.Counter()
 labels={'Animals':'动物','Art and design':'艺术设计','Books':'书籍','Buildings and places':'建筑','Cartoons':'卡通','Clothing and accessories':'服饰','Comics':'漫画','Computers':'电脑','Electronics':'电子','Food and drinks':'食物','Holiday and events':'节日','Logos':'标志','Miscellaneous':'其他','Movies':'电影','Music':'音乐','Mythology':'神话','Nature':'自然','People':'人物','Plants':'植物','Religion':'宗教','Space':'太空','Sports and outdoors':'运动','Television':'电视','Toys':'玩具','Vehicles':'交通','Video games':'游戏','Weapons':'武器'}
 if not any(x['source']==source['name'] and x['revision']==identity for x in sources):
  for file in source['files']:
   path=args.traditional/file['path'];verify(path,file)
   for i,row in enumerate(pq.read_table(path).to_pylist()):
    stats['source_rows']+=1
    if re.search(r'\b(sexual|porn|nsfw|erotic|nudes?)\b',row['labels'],re.I):stats['excluded_adult_tag']+=1;continue
    text=row['content'].replace('\r\n','\n').replace('\r','\n').expandtabs(8);text='\n'.join(line.rstrip() for line in text.split('\n')).strip('\n');key=normalize(text)
    if key in seen:stats['duplicates']+=1;continue
    if not key or any(unicodedata.category(c)=='Cc' and c!='\n' for c in text):stats['invalid']+=1;continue
    seen.add(key);category=row['labels'].split(',')[0];title=labels.get(category,category)+' · '+(row['caption'] or row['labels'].replace(',', ' / '))
    records.append([text,title,'art',row['labels']+' '+labels.get(category,''),'apehex_'+path.stem+'_'+str(i),'monospace']);stats['accepted']+=1
  sources.append({'source':source['name'],'revision':identity,'config':'asciiart/train only','license':'Dataset card AGPL-3.0; original artist signatures retained','files_sha256':{f['path']:f['sha256'] for f in source['files']},**stats,'changes':'Preserved artwork and signatures; CRLF normalized; tabs expanded to 8 columns; trailing whitespace trimmed; NFC/whitespace deduplication; adult-tagged records excluded. No image conversion or font-generated graffiti configurations.'})
if args.captions:
 import pyarrow as pa
 import pyarrow.ipc as ipc
 source=next(x for x in pending['sources'] if x['name']=='mrzjy/kaomoji_caption');identity=source['revision'];expected=source['files'][0];verify(args.captions,expected)
 if not any(x['source']==source['name'] and x['revision']==identity for x in sources):
  rows=ipc.open_stream(pa.memory_map(str(args.captions),'r')).read_all().to_pylist();stats=collections.Counter()
  for i,row in enumerate(rows):
   text=row['kaomoji'].strip();key=normalize(text)
   if key in seen:stats['duplicates']+=1;continue
   if not key or any(unicodedata.category(c)=='Cc' and c!='\n' for c in text):stats['invalid']+=1;continue
   seen.add(key);caption=row['caption'].strip();zh=[v for k,v in translations.items() if re.search(r'\b'+re.escape(k)+r'\b',caption,re.I)];meta=json.loads(row.get('meta') or '{}')
   records.append([text,caption or '颜文字','art' if '\n' in text else 'kaomoji',' '.join([*zh,caption]),'captions_'+str(i),'',meta.get('source','')]);stats['accepted']+=1
  sources.append({'source':source['name'],'revision':identity,'license':'Apache-2.0 (dataset card)','source_rows':len(rows),'sha256':expected['sha256'],**stats,'changes':'Preserved kaomoji and short captions, retained original source URLs, omitted long explanatory prose; NFC/whitespace cross-source deduplication; no generated records.'})
files=[]
for i in range(0,len(records),4000):
 path=f'data/kaomoji/{i//4000:02d}.json';Path(path).write_text(json.dumps(records[i:i+4000],ensure_ascii=False,separators=(',',':')));files.append('./'+path)
manifest.update(files=files,accepted=len(records),total=manifest['base_count']+len(records),category_counts=dict(collections.Counter(x[2] for x in records)))
manifest_path.write_text(json.dumps(manifest,indent=2,ensure_ascii=False)+'\n')
print(json.dumps({'supplemental':sources,'total':manifest['total'],'category_counts':manifest['category_counts']},ensure_ascii=False))