lab/landing_page
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: resolve I1 date parsing regression and implement strict volume cross-checking gate

Browse Source
This commit is contained in:
Godopu
2026-07-30 10:49:33 +09:00
parent 7943b9390f
commit c3c18422a4
5 changed files with 109 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
refer_landing_page/scripts/extract/generate_all.py
+28 -22
View File
@@ -27,44 +27,50 @@ def to_iso(raw):
return None
raw = raw.strip()
# 1) YYYY-MM-DD or YYYY.MM.DD
m = re.search(r'(\d{4})[. /\\-]?(\d{1,2})[. /\\-]?(\d{1,2})', raw)
if m:
yr = int(m.group(1))
mo = int(m.group(2))
day = int(m.group(3))
if mo <= 12 and day <= 31:
return f"{yr:04d}-{mo:02d}-{day:02d}"
# 2) "May 2026", "Feb. 2008", "August 2018"
m_txt = re.search(r'([A-Za-z]+)\.?\s*(\d{4})', raw)
# 1) English month + 19XX/20XX year: "April 2026", "Feb. 2008", "August, 2018"
m_txt = re.search(r'([A-Za-z]+)\.?\s*,?\s*((?:19|20)\d{2})', raw)
if m_txt:
mo_str = m_txt.group(1).lower().rstrip('.')
yr_str = m_txt.group(2)
if mo_str in MONTH_MAP:
return f"{yr_str}-{MONTH_MAP[mo_str]}"
# 3) "2019년 12월", "9월, 1999"
m_kor = re.search(r'(\d{4})년\s*(\d{1,2})월', raw)
# 2) Korean "2026년 6월", "9월, 1999"
m_kor = re.search(r'((?:19|20)\d{2})년\s*(\d{1,2})월', raw)
if m_kor:
return f"{int(m_kor.group(1)):04d}-{int(m_kor.group(2)):02d}"
yr = int(m_kor.group(1))
mo = int(m_kor.group(2))
if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}"
m_kor2 = re.search(r'(\d{1,2})월.*?,?\s*(\d{4})', raw)
m_kor2 = re.search(r'(\d{1,2})월\s*,?\s*((?:19|20)\d{2})', raw)
if m_kor2:
return f"{int(m_kor2.group(2)):04d}-{int(m_kor2.group(1)):02d}"
mo = int(m_kor2.group(1))
yr = int(m_kor2.group(2))
if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}"
# 4) YYYY-MM or YYYY.MM
m_ym = re.search(r'(\d{4})[. /\\-]?(\d{1,2})', raw)
# 3) ISO date: YYYY-MM-DD, YYYY.MM.DD (where YYYY is 19XX or 20XX)
m = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})[. /\\-]?(\d{1,2})', raw)
if m:
yr = int(m.group(1))
mo = int(m.group(2))
day = int(m.group(3))
if 1 <= mo <= 12 and 1 <= day <= 31:
return f"{yr:04d}-{mo:02d}-{day:02d}"
# 4) ISO month: YYYY-MM or YYYY.MM
m_ym = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})', raw)
if m_ym:
yr = int(m_ym.group(1))
mo = int(m_ym.group(2))
if mo <= 12:
if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}"
# 5) YYYY only
m_y = re.search(r'(\d{4})', raw)
# 5) 19XX or 20XX year only
m_y = re.search(r'((?:19|20)\d{2})', raw)
if m_y:
return f"{int(m_y.group(1)):04d}"
return f"{m_y.group(1)}"
return raw
refer_landing_page/scripts/extract/verify_counts.py
+43 -8
View File
@@ -6,6 +6,13 @@ import re
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, os.path.join(BASE_DIR, "refer_landing_page", "scripts", "extract"))
MONTH_MAP = {
"january": "01", "jan": "01", "february": "02", "feb": "02", "march": "03", "mar": "03",
"april": "04", "apr": "04", "may": "05", "june": "06", "jun": "06", "july": "07", "jul": "07",
"august": "08", "aug": "08", "september": "09", "sep": "09", "sept": "09", "october": "10",
"oct": "10", "november": "11", "nov": "11", "december": "12", "dec": "12"
}
def verify():
from generate_all import parse_members, parse_publications, parse_lectures, parse_standards
@@ -17,7 +24,7 @@ def verify():
total_pubs = len(intl_p) + len(dom_p) + len(pat_p)
std_docs_count = sum(len(p["documents"]) for b in bodies for p in b["projects"])
print("=== DATA VERIFICATION GATE (AC-07 / AC-08 / AC-10 / AC-11) ===")
print("=== DATA VERIFICATION GATE (AC-07 / AC-08 / AC-10 / AC-11 / VOLUME CROSS-CHECK) ===")
print(f"Active Members: {len(active_m)} (Expected: 16)")
print(f"Alumni: {len(alumni_m)} (Expected: 60)")
print(f"International Papers: {len(intl_p)} (Expected: 68)")
@@ -29,6 +36,8 @@ def verify():
print(f"Standard Documents: {std_docs_count} (Expected: 44)")
errors = []
# 1. Count checks
if len(active_m) != 16:
errors.append(f"Active Members count mismatch: got {len(active_m)}, expected 16")
if len(alumni_m) != 60:
@@ -48,8 +57,10 @@ def verify():
if std_docs_count != 44:
errors.append(f"Standard Documents count mismatch: got {std_docs_count}, expected 44")
# Date ISO 8601 validation (AC-11)
# 2. File checks (BOM, Strict ISO 8601, and Semantic Date Ranges)
content_dir = os.path.join(BASE_DIR, "content")
date_regex = re.compile(r'^(19|20)\d{2}(-(0[1-9]|1[0-2]))?(-(0[1-9]|[12]\d|3[01]))?$')
for fname in ["members.ts", "publications.ts", "lectures.ts", "standards.ts"]:
fpath = os.path.join(content_dir, fname)
if not os.path.exists(fpath):
@@ -58,18 +69,42 @@ def verify():
with open(fpath, "r", encoding="utf-8") as f:
text = f.read()
# Check BOM (AC-10)
if text.startswith('\ufeff'):
errors.append(f"BOM found in {fname}")
errors.append(f"BOM (U+FEFF) found in {fname}")
# Check dates
dates = re.findall(r'"(publishedAt|applicationAt|registrationAt|graduatedAt)": "([^"]+)"', text)
for k, d in dates:
if not re.match(r'^\d{4}(-\d{2})?(-\d{2})?$', d):
errors.append(f"Non-ISO 8601 date in {fname}: {k}='{d}'")
if not date_regex.match(d):
errors.append(f"Invalid ISO 8601 or out-of-bounds date in {fname}: {k}='{d}'")
# 3. Volume Cross-Check for Publications (Ensures volume year matches publishedAt year)
for pub in intl_p + dom_p:
vol = pub.get("volume", "")
pub_date = pub.get("publishedAt", "")
if not pub_date:
errors.append(f"Missing publishedAt for paper: {pub.get('title')}")
continue
# Look for English month + 19XX/20XX year or Korean YYYY년 MM월
m_txt = re.search(r'([A-Za-z]+)\.?\s*,?\s*((?:19|20)\d{2})', vol)
m_kor = re.search(r'((?:19|20)\d{2})년\s*(\d{1,2})월', vol)
m_kor2 = re.search(r'(\d{1,2})월\s*,?\s*((?:19|20)\d{2})', vol)
expected_yr = None
if m_txt:
mo_str = m_txt.group(1).lower().rstrip('.')
if mo_str in MONTH_MAP:
expected_yr = m_txt.group(2)
elif m_kor:
expected_yr = m_kor.group(1)
elif m_kor2:
expected_yr = m_kor2.group(2)
if expected_yr and not pub_date.startswith(expected_yr):
errors.append(f"Volume date mismatch for '{pub.get('title')}': volume has {expected_yr}, publishedAt is {pub_date}")
if errors:
print("\n❌ DATA VERIFICATION FAILED:")
print(f"\n❌ DATA VERIFICATION FAILED ({len(errors)} errors):")
for err in errors:
print(f" - {err}")
sys.exit(1)