fix: resolve edge case date parsing for volume strings and update volume date verification gate

This commit is contained in:
2026-07-30 10:54:49 +09:00
parent c3c18422a4
commit 67fca110bd
3 changed files with 38 additions and 32 deletions
+4 -4
View File
@@ -154,7 +154,7 @@ export const publications: Publication[] = [
"title": "IoT-Based Resource Control for In-Vehicle Infotainment Services: Design and Experimentation", "title": "IoT-Based Resource Control for In-Vehicle Infotainment Services: Design and Experimentation",
"venue": "Sensors", "venue": "Sensors",
"volume": "Vol. 19, Article No. s19030620, pp. 1~19, March 2019", "volume": "Vol. 19, Article No. s19030620, pp. 1~19, March 2019",
"publishedAt": "1903-06-20" "publishedAt": "2019-03"
}, },
{ {
"category": "intl-journal-conf", "category": "intl-journal-conf",
@@ -168,14 +168,14 @@ export const publications: Publication[] = [
"title": "Domain-based Distributed Identifier-Locator Mapping Management in Internet-of-Things Networks", "title": "Domain-based Distributed Identifier-Locator Mapping Management in Internet-of-Things Networks",
"venue": "International Journal of Network Management", "venue": "International Journal of Network Management",
"volume": "Vol. 28, Issue 5 (e2035), pp. 1~13, October 2018", "volume": "Vol. 28, Issue 5 (e2035), pp. 1~13, October 2018",
"publishedAt": "2035" "publishedAt": "2018-10"
}, },
{ {
"category": "intl-journal-conf", "category": "intl-journal-conf",
"title": "Cluster-Based Device Mobility Management in Named Data Networking for Vehicular Networks", "title": "Cluster-Based Device Mobility Management in Named Data Networking for Vehicular Networks",
"venue": "Mobile Information Systems", "venue": "Mobile Information Systems",
"volume": "Vol. 2018, pp. 1~7, Open Access Article (Article ID 1710591), August 2018", "volume": "Vol. 2018, pp. 1~7, Open Access Article (Article ID 1710591), August 2018",
"publishedAt": "2018" "publishedAt": "2018-08"
}, },
{ {
"category": "intl-journal-conf", "category": "intl-journal-conf",
@@ -252,7 +252,7 @@ export const publications: Publication[] = [
"title": "An ID/Locator Separation Based Group Mobility Management in Wireless Body Area Network", "title": "An ID/Locator Separation Based Group Mobility Management in Wireless Body Area Network",
"venue": "Journal of Sensors", "venue": "Journal of Sensors",
"volume": "Vol. 2015, Open Access Article (Article ID 537205), 12 pages, July 2015", "volume": "Vol. 2015, Open Access Article (Article ID 537205), 12 pages, July 2015",
"publishedAt": "2015" "publishedAt": "2015-07"
}, },
{ {
"category": "intl-journal-conf", "category": "intl-journal-conf",
@@ -27,30 +27,40 @@ def to_iso(raw):
return None return None
raw = raw.strip() raw = raw.strip()
# 1) English month + 19XX/20XX year: "April 2026", "Feb. 2008", "August, 2018" MON_PAT = r'(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
m_txt = re.search(r'([A-Za-z]+)\.?\s*,?\s*((?:19|20)\d{2})', raw)
if m_txt: # 1) End-anchored English month + 19XX/20XX year: "March 2019.", "October 2018"
mo_str = m_txt.group(1).lower().rstrip('.') m_end = re.search(rf'\b({MON_PAT})\.?\s*,?\s*((?:19|20)\d{{2}})\s*\.?\s*$', raw, re.I)
yr_str = m_txt.group(2) if m_end:
mo_str = m_end.group(1).lower().rstrip('.')
if mo_str in MONTH_MAP: if mo_str in MONTH_MAP:
return f"{yr_str}-{MONTH_MAP[mo_str]}" return f"{m_end.group(2)}-{MONTH_MAP[mo_str]}"
# 2) Korean "2026년 6월", "9월, 1999" # 2) Korean "2026년 6월", "9월, 1999"
m_kor = re.search(r'((?:19|20)\d{2})년\s*(\d{1,2})월', raw) m_kor = re.search(r'((?:19|20)\d{2})\s*\s*(\d{1,2})\s*', raw)
if m_kor: if m_kor:
yr = int(m_kor.group(1)) yr = int(m_kor.group(1))
mo = int(m_kor.group(2)) mo = int(m_kor.group(2))
if 1 <= mo <= 12: if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}" return f"{yr:04d}-{mo:02d}"
m_kor2 = re.search(r'(\d{1,2})월\s*,?\s*((?:19|20)\d{2})', raw) m_kor2 = re.search(r'(\d{1,2})\s*\s*,?\s*((?:19|20)\d{2})', raw)
if m_kor2: if m_kor2:
mo = int(m_kor2.group(1)) mo = int(m_kor2.group(1))
yr = int(m_kor2.group(2)) yr = int(m_kor2.group(2))
if 1 <= mo <= 12: if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}" return f"{yr:04d}-{mo:02d}"
# 3) ISO date: YYYY-MM-DD, YYYY.MM.DD (where YYYY is 19XX or 20XX) # 3) Last match of English Month + Year anywhere in string (bypasses preceding article/issue numbers like s19030620 or e2035)
ms = list(re.finditer(rf'\b({MON_PAT})\.?\s*,?\s*((?:19|20)\d{{2}})', raw, re.I))
if ms:
for m_txt in reversed(ms):
mo_str = m_txt.group(1).lower().rstrip('.')
yr_str = m_txt.group(2)
if mo_str in MONTH_MAP:
return f"{yr_str}-{MONTH_MAP[mo_str]}"
# 4) ISO date: YYYY-MM-DD, YYYY.MM.DD (where YYYY is 19XX or 20XX)
m = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})[. /\\-]?(\d{1,2})', raw) m = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})[. /\\-]?(\d{1,2})', raw)
if m: if m:
yr = int(m.group(1)) yr = int(m.group(1))
@@ -59,7 +69,7 @@ def to_iso(raw):
if 1 <= mo <= 12 and 1 <= day <= 31: if 1 <= mo <= 12 and 1 <= day <= 31:
return f"{yr:04d}-{mo:02d}-{day:02d}" return f"{yr:04d}-{mo:02d}-{day:02d}"
# 4) ISO month: YYYY-MM or YYYY.MM # 5) ISO month: YYYY-MM or YYYY.MM
m_ym = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})', raw) m_ym = re.search(r'((?:19|20)\d{2})[. /\\-]?(\d{1,2})', raw)
if m_ym: if m_ym:
yr = int(m_ym.group(1)) yr = int(m_ym.group(1))
@@ -67,8 +77,8 @@ def to_iso(raw):
if 1 <= mo <= 12: if 1 <= mo <= 12:
return f"{yr:04d}-{mo:02d}" return f"{yr:04d}-{mo:02d}"
# 5) 19XX or 20XX year only # 6) 19XX or 20XX year only
m_y = re.search(r'((?:19|20)\d{2})', raw) m_y = re.search(r'\b((?:19|20)\d{2})\b', raw)
if m_y: if m_y:
return f"{m_y.group(1)}" return f"{m_y.group(1)}"
@@ -77,7 +77,9 @@ def verify():
if not date_regex.match(d): if not date_regex.match(d):
errors.append(f"Invalid ISO 8601 or out-of-bounds date in {fname}: {k}='{d}'") errors.append(f"Invalid ISO 8601 or out-of-bounds date in {fname}: {k}='{d}'")
# 3. Volume Cross-Check for Publications (Ensures volume year matches publishedAt year) # 3. Volume Cross-Check for Publications (Ensures volume date candidates match publishedAt)
MON_PAT = r'(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
for pub in intl_p + dom_p: for pub in intl_p + dom_p:
vol = pub.get("volume", "") vol = pub.get("volume", "")
pub_date = pub.get("publishedAt", "") pub_date = pub.get("publishedAt", "")
@@ -85,23 +87,17 @@ def verify():
errors.append(f"Missing publishedAt for paper: {pub.get('title')}") errors.append(f"Missing publishedAt for paper: {pub.get('title')}")
continue continue
# Look for English month + 19XX/20XX year or Korean YYYY년 MM월 cands = []
m_txt = re.search(r'([A-Za-z]+)\.?\s*,?\s*((?:19|20)\d{2})', vol) for m in re.finditer(rf'\b({MON_PAT})\.?\s*,?\s*((?:19|20)\d{{2}})', vol, re.I):
m_kor = re.search(r'((?:19|20)\d{2})년\s*(\d{1,2})월', vol) mo_str = m.group(1).lower().rstrip('.')
m_kor2 = re.search(r'(\d{1,2})월\s*,?\s*((?:19|20)\d{2})', vol)
expected_yr = None
if m_txt:
mo_str = m_txt.group(1).lower().rstrip('.')
if mo_str in MONTH_MAP: if mo_str in MONTH_MAP:
expected_yr = m_txt.group(2) cands.append(m.group(2))
elif m_kor: for m in re.finditer(r'((?:19|20)\d{2})\s*년\s*(\d{1,2})\s*월', vol):
expected_yr = m_kor.group(1) cands.append(m.group(1))
elif m_kor2:
expected_yr = m_kor2.group(2)
if expected_yr and not pub_date.startswith(expected_yr): if cands:
errors.append(f"Volume date mismatch for '{pub.get('title')}': volume has {expected_yr}, publishedAt is {pub_date}") if not any(pub_date.startswith(yr) for yr in cands):
errors.append(f"Volume date mismatch for '{pub.get('title')}': volume candidates={cands}, publishedAt={pub_date}")
if errors: if errors:
print(f"\n❌ DATA VERIFICATION FAILED ({len(errors)} errors):") print(f"\n❌ DATA VERIFICATION FAILED ({len(errors)} errors):")