fix: resolve edge case date parsing for volume strings and update volume date verification gate
This commit is contained in:
@@ -77,7 +77,9 @@ def verify():
|
||||
if not date_regex.match(d):
|
||||
errors.append(f"Invalid ISO 8601 or out-of-bounds date in {fname}: {k}='{d}'")
|
||||
|
||||
# 3. Volume Cross-Check for Publications (Ensures volume year matches publishedAt year)
|
||||
# 3. Volume Cross-Check for Publications (Ensures volume date candidates match publishedAt)
|
||||
MON_PAT = r'(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
|
||||
|
||||
for pub in intl_p + dom_p:
|
||||
vol = pub.get("volume", "")
|
||||
pub_date = pub.get("publishedAt", "")
|
||||
@@ -85,23 +87,17 @@ def verify():
|
||||
errors.append(f"Missing publishedAt for paper: {pub.get('title')}")
|
||||
continue
|
||||
|
||||
# Look for English month + 19XX/20XX year or Korean YYYY년 MM월
|
||||
m_txt = re.search(r'([A-Za-z]+)\.?\s*,?\s*((?:19|20)\d{2})', vol)
|
||||
m_kor = re.search(r'((?:19|20)\d{2})년\s*(\d{1,2})월', vol)
|
||||
m_kor2 = re.search(r'(\d{1,2})월\s*,?\s*((?:19|20)\d{2})', vol)
|
||||
|
||||
expected_yr = None
|
||||
if m_txt:
|
||||
mo_str = m_txt.group(1).lower().rstrip('.')
|
||||
cands = []
|
||||
for m in re.finditer(rf'\b({MON_PAT})\.?\s*,?\s*((?:19|20)\d{{2}})', vol, re.I):
|
||||
mo_str = m.group(1).lower().rstrip('.')
|
||||
if mo_str in MONTH_MAP:
|
||||
expected_yr = m_txt.group(2)
|
||||
elif m_kor:
|
||||
expected_yr = m_kor.group(1)
|
||||
elif m_kor2:
|
||||
expected_yr = m_kor2.group(2)
|
||||
cands.append(m.group(2))
|
||||
for m in re.finditer(r'((?:19|20)\d{2})\s*년\s*(\d{1,2})\s*월', vol):
|
||||
cands.append(m.group(1))
|
||||
|
||||
if expected_yr and not pub_date.startswith(expected_yr):
|
||||
errors.append(f"Volume date mismatch for '{pub.get('title')}': volume has {expected_yr}, publishedAt is {pub_date}")
|
||||
if cands:
|
||||
if not any(pub_date.startswith(yr) for yr in cands):
|
||||
errors.append(f"Volume date mismatch for '{pub.get('title')}': volume candidates={cands}, publishedAt={pub_date}")
|
||||
|
||||
if errors:
|
||||
print(f"\n❌ DATA VERIFICATION FAILED ({len(errors)} errors):")
|
||||
|
||||
Reference in New Issue
Block a user