diff --git a/backend/.gitignore b/backend/.gitignore index fb277ef..3be7a30 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,5 +1,8 @@ # Binaries and build outputs /bin/ +/export-content +/api +/seed # SQLite databases and WAL journal files *.db diff --git a/backend/Makefile b/backend/Makefile index bc4cdb3..525c166 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -21,6 +21,8 @@ seed: build ./$(BIN_DIR)/seed --db $(DB_PATH) --seed-dir seed/data export-content: build + @echo "⚠️ Notice: Frontend has migrated to real-time dynamic runtime fetching (force-dynamic)." + @echo "export-content is deprecated to prevent re-introducing static content data files." ./$(BIN_DIR)/export-content --db $(DB_PATH) --out-dir ../refer_landing_page/content fmt: diff --git a/backend/cmd/export-content/main.go b/backend/cmd/export-content/main.go index 3d4588b..0ef07b3 100644 --- a/backend/cmd/export-content/main.go +++ b/backend/cmd/export-content/main.go @@ -50,9 +50,17 @@ func main() { dbPath := flag.String("db", defaultDB, "Path to SQLite database file") targetDir := flag.String("out-dir", "../refer_landing_page/content", "Target content directory") + force := flag.Bool("force", false, "Force export content files despite dynamic runtime architecture") flag.Parse() - log.Printf("Exporting content from SQLite (%s) to TypeScript files in %s...", *dbPath, *targetDir) + if !*force { + log.Printf("⚠️ NOTICE: Frontend architecture has migrated to real-time dynamic runtime fetching (force-dynamic).") + log.Printf("Exporting static TypeScript files to %s is DEPRECATED to prevent re-introducing static fallback leaks.", *targetDir) + log.Printf("If you really need to export fallback files, pass the --force flag.") + return + } + + log.Printf("Exporting content from SQLite (%s) to TypeScript files in %s (forced)...", *dbPath, *targetDir) database, err := db.Connect(*dbPath) if err != nil { log.Fatalf("Failed to connect to database: %v", err) diff --git a/refer_landing_page/scripts/extract/verify_counts.py b/refer_landing_page/scripts/extract/verify_counts.py index 799ae80..ac93271 100644 --- a/refer_landing_page/scripts/extract/verify_counts.py +++ b/refer_landing_page/scripts/extract/verify_counts.py @@ -1,11 +1,31 @@ #!/usr/bin/env python3 +""" +Data Verification Gate (AC-07 / AC-08 / AC-10 / AC-11 / AC-30 / Live Database & API Pipeline Integrity) +Verifies: +1. Full data extraction from legacy HTML sources (16 members, 60 alumni, 68 intl, 80 dom, 75 pat, 45 sem, 6 bodies, 44 docs, 3 projects). +2. SQLite Backend Database (backend/anl.db) tables & row counts matching baseline. +3. Live Go REST API endpoints (http://localhost:8080/api/v1) data losslessness & schema consistency if server is active. +4. ISO 8601 strict date formatting across all entities. +5. Publication volume cross-check and keyword substring assertions. +""" + import sys import os import re +import sqlite3 +import json +import urllib.request BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.join(BASE_DIR, "refer_landing_page", "scripts", "extract")) +ROOT_PROJECT_DIR = os.path.dirname(BASE_DIR) +DB_PATH = os.path.join(ROOT_PROJECT_DIR, "backend", "anl.db") +if not os.path.exists(DB_PATH): + DB_PATH = os.path.join(BASE_DIR, "backend", "anl.db") + +API_BASE_URL = os.environ.get("API_BASE_URL", "http://localhost:8080/api/v1") + MONTH_MAP = { "january": "01", "jan": "01", "february": "02", "feb": "02", "march": "03", "mar": "03", "april": "04", "apr": "04", "may": "05", "june": "06", "jun": "06", "july": "07", "jul": "07", @@ -25,42 +45,22 @@ def verify(): total_pubs = len(intl_p) + len(dom_p) + len(pat_p) std_docs_count = sum(len(p["documents"]) for b in bodies for p in b["projects"]) - print("=== DATA VERIFICATION GATE (AC-07 / AC-08 / AC-10 / AC-11 / AC-30 / VOLUME CROSS-CHECK) ===") - print(f"Active Members: {len(active_m)} (Expected: 16)") - print(f"Alumni: {len(alumni_m)} (Expected: 60)") - print(f"International Papers: {len(intl_p)} (Expected: 68)") - print(f"Domestic Papers: {len(dom_p)} (Expected: 80)") - print(f"Patents: {len(pat_p)} (Expected: 75)") - print(f"Total Publications: {total_pubs} (Expected: 223)") - print(f"Lectures Semesters: {len(semesters)} (Expected: 45)") - print(f"Standard Bodies: {len(bodies)} (Expected: 6)") - print(f"Standard Documents: {std_docs_count} (Expected: 44)") - print(f"Research Projects: {len(projects)} (Expected: 3)") + print("=== DATA VERIFICATION GATE (AC-07 / AC-08 / AC-10 / AC-11 / AC-30 / LIVE PIPELINE INTEGRITY) ===") + print(f"[HTML Source Extraction Baseline]") + print(f" Active Members: {len(active_m)} (Expected: 16)") + print(f" Alumni: {len(alumni_m)} (Expected: 60)") + print(f" International Papers: {len(intl_p)} (Expected: 68)") + print(f" Domestic Papers: {len(dom_p)} (Expected: 80)") + print(f" Patents: {len(pat_p)} (Expected: 75)") + print(f" Total Publications: {total_pubs} (Expected: 223)") + print(f" Lectures Semesters: {len(semesters)} (Expected: 45)") + print(f" Standard Bodies: {len(bodies)} (Expected: 6)") + print(f" Standard Documents: {std_docs_count} (Expected: 44)") + print(f" Research Projects: {len(projects)} (Expected: 3)") errors = [] - if len(projects) != 3: - errors.append(f"Research Projects count mismatch: got {len(projects)}, expected 3") - - # AC-30: Keyword Substring Verification & AC-32 Deliverable WG Attribution - for proj in projects: - abstract = proj.get("abstract", "") - for kw in proj.get("keywords", []): - if kw not in abstract: - errors.append(f"AC-30 Keyword Violation in project '{proj.get('slug')}': keyword '{kw}' not in abstract") - - if proj["slug"] == "ccis": - for d in proj.get("deliverables", []): - if "63246" not in d["ref"]: - errors.append(f"AC-32 Violation: CCIS project deliverable '{d['ref']}' does not belong to CCIS WG (IEC 63246)") - - # N1: Title artifact check (No trailing quotes or commas in publication titles) - for p in intl_p + dom_p + pat_p: - t = p.get("title", "") - if re.search(r'["“”]|,$', t): - errors.append(f"N1 Title Artifact Violation in publication '{p.get('id')}': title '{t}' contains quotes or trailing comma") - - # 1. Count checks + # 1. Source Extraction Integrity Checks if len(active_m) != 16: errors.append(f"Active Members count mismatch: got {len(active_m)}, expected 16") if len(alumni_m) != 60: @@ -79,28 +79,175 @@ def verify(): errors.append(f"Standard Bodies count mismatch: got {len(bodies)}, expected 6") if std_docs_count != 44: errors.append(f"Standard Documents count mismatch: got {std_docs_count}, expected 44") + if len(projects) != 3: + errors.append(f"Research Projects count mismatch: got {len(projects)}, expected 3") - # 2. File checks (BOM, Strict ISO 8601, and Semantic Date Ranges) + # 2. SQLite Backend Database Verification + if os.path.exists(DB_PATH): + print(f"\n[SQLite Database Verification ({DB_PATH})]") + try: + conn = sqlite3.connect(DB_PATH) + cur = conn.cursor() + + def get_count(table, where=""): + query = f"SELECT count(*) FROM {table} {where}" + cur.execute(query) + return cur.fetchone()[0] + + db_counts = { + "members": get_count("members"), + "alumni": get_count("alumni"), + "intl_pubs": get_count("publications", "WHERE category='intl-journal-conf'"), + "dom_pubs": get_count("publications", "WHERE category='domestic-journal-conf'"), + "patents": get_count("patents"), + "semesters": get_count("semesters"), + "standards_bodies": get_count("standards_bodies"), + "standard_documents": get_count("standard_documents"), + "research_projects": get_count("research_projects"), + } + + for k, v in db_counts.items(): + print(f" DB {k}: {v}") + + if db_counts["members"] != 16: + errors.append(f"SQLite members count: {db_counts['members']}, expected 16") + if db_counts["alumni"] != 60: + errors.append(f"SQLite alumni count: {db_counts['alumni']}, expected 60") + if db_counts["intl_pubs"] != 68: + errors.append(f"SQLite intl publications count: {db_counts['intl_pubs']}, expected 68") + if db_counts["dom_pubs"] != 80: + errors.append(f"SQLite dom publications count: {db_counts['dom_pubs']}, expected 80") + if db_counts["patents"] != 75: + errors.append(f"SQLite patents count: {db_counts['patents']}, expected 75") + if db_counts["semesters"] != 45: + errors.append(f"SQLite semesters count: {db_counts['semesters']}, expected 45") + if db_counts["standards_bodies"] != 6: + errors.append(f"SQLite standards bodies count: {db_counts['standards_bodies']}, expected 6") + if db_counts["standard_documents"] != 44: + errors.append(f"SQLite standard documents count: {db_counts['standard_documents']}, expected 44") + if db_counts["research_projects"] != 3: + errors.append(f"SQLite research projects count: {db_counts['research_projects']}, expected 3") + + conn.close() + print(" ✅ SQLite Database records match expected baseline exactly") + except Exception as e: + errors.append(f"SQLite DB verification failed: {e}") + else: + print(f"\n⚠️ SQLite database not found at {DB_PATH}, skipping direct DB query") + + # 3. Live Go REST API Endpoint Verification (if backend is active) + is_api_live = False + try: + req = urllib.request.Request(f"{API_BASE_URL}/health", headers={"User-Agent": "DataVerifier"}) + with urllib.request.urlopen(req, timeout=1) as resp: + if resp.status == 200: + is_api_live = True + except Exception: + is_api_live = False + + if is_api_live: + print(f"\n[Live Go REST API Verification ({API_BASE_URL})]") + try: + def fetch_json(endpoint): + req = urllib.request.Request(f"{API_BASE_URL}{endpoint}", headers={"User-Agent": "DataVerifier"}) + with urllib.request.urlopen(req, timeout=3) as resp: + data = json.loads(resp.read().decode("utf-8")) + return data.get("data", []) + + stats = fetch_json("/stats/summary") + api_members = fetch_json("/members") + api_alumni = fetch_json("/alumni") + api_intl = fetch_json("/publications?category=intl-journal-conf") + api_dom = fetch_json("/publications?category=domestic-journal-conf") + api_patents = fetch_json("/patents") + api_semesters = fetch_json("/semesters") + api_standards = fetch_json("/standards-bodies") + api_projects = fetch_json("/research-projects") + + api_std_docs_count = sum(len(p.get("documents", [])) for b in api_standards for p in b.get("projects", [])) + + print(f" API stats summary: {stats}") + print(f" API members count: {len(api_members)}") + print(f" API alumni count: {len(api_alumni)}") + print(f" API publications (intl): {len(api_intl)}, (dom): {len(api_dom)}") + print(f" API patents count: {len(api_patents)}") + print(f" API semesters count: {len(api_semesters)}") + print(f" API standards bodies: {len(api_standards)} (docs: {api_std_docs_count})") + print(f" API projects count: {len(api_projects)}") + + if stats.get("intl_publications") != 68: + errors.append(f"API stats intl_publications mismatch: {stats.get('intl_publications')}, expected 68") + if stats.get("standardization_docs") != 44: + errors.append(f"API stats standardization_docs mismatch: {stats.get('standardization_docs')}, expected 44") + if stats.get("patents") != 75: + errors.append(f"API stats patents mismatch: {stats.get('patents')}, expected 75") + if len(api_members) != 16: + errors.append(f"API members count: {len(api_members)}, expected 16") + if len(api_alumni) != 60: + errors.append(f"API alumni count: {len(api_alumni)}, expected 60") + if len(api_intl) != 68: + errors.append(f"API intl publications count: {len(api_intl)}, expected 68") + if len(api_dom) != 80: + errors.append(f"API dom publications count: {len(api_dom)}, expected 80") + if len(api_patents) != 75: + errors.append(f"API patents count: {len(api_patents)}, expected 75") + if len(api_semesters) != 45: + errors.append(f"API semesters count: {len(api_semesters)}, expected 45") + if len(api_standards) != 6: + errors.append(f"API standards bodies count: {len(api_standards)}, expected 6") + if api_std_docs_count != 44: + errors.append(f"API standard documents count: {api_std_docs_count}, expected 44") + if len(api_projects) != 3: + errors.append(f"API research projects count: {len(api_projects)}, expected 3") + + print(" ✅ Live REST API endpoints serve 100% losslessly verified data") + except Exception as e: + errors.append(f"Live API verification error: {e}") + else: + print(f"\nℹ️ Live Go API server not responding at {API_BASE_URL} (tested offline DB mode)") + + # 4. AC-30 Keyword Substring Verification & AC-32 Deliverable WG Attribution + for proj in projects: + abstract = proj.get("abstract", "") + for kw in proj.get("keywords", []): + if kw not in abstract: + errors.append(f"AC-30 Keyword Violation in project '{proj.get('slug')}': keyword '{kw}' not in abstract") + + if proj["slug"] == "ccis": + for d in proj.get("deliverables", []): + if "63246" not in d["ref"]: + errors.append(f"AC-32 Violation: CCIS project deliverable '{d['ref']}' does not belong to CCIS WG (IEC 63246)") + + # 5. Title artifact check (No trailing quotes or commas in publication titles) + for p in intl_p + dom_p + pat_p: + t = p.get("title", "") + if re.search(r'["“”]|,$', t): + errors.append(f"N1 Title Artifact Violation in publication '{p.get('id')}': title '{t}' contains quotes or trailing comma") + + # 6. Strict ISO 8601 Date and BOM Verification content_dir = os.path.join(BASE_DIR, "content") date_regex = re.compile(r'^(19|20)\d{2}(-(0[1-9]|1[0-2]))?(-(0[1-9]|[12]\d|3[01]))?$') - for fname in ["members.ts", "publications.ts", "lectures.ts", "standards.ts"]: - fpath = os.path.join(content_dir, fname) - if not os.path.exists(fpath): - errors.append(f"Missing content file: {fname}") - continue - with open(fpath, "r", encoding="utf-8") as f: - text = f.read() + for fname in os.listdir(content_dir) if os.path.exists(content_dir) else []: + if fname.endswith(".ts"): + fpath = os.path.join(content_dir, fname) + with open(fpath, "r", encoding="utf-8") as f: + text = f.read() + if text.startswith('\ufeff'): + errors.append(f"BOM (U+FEFF) found in {fname}") - if text.startswith('\ufeff'): - errors.append(f"BOM (U+FEFF) found in {fname}") + for p in intl_p + dom_p + pat_p: + for field in ["publishedAt", "applicationAt", "registrationAt"]: + val = p.get(field) + if val and not date_regex.match(val): + errors.append(f"Invalid ISO 8601 date in publication '{p.get('title')}': {field}='{val}'") - dates = re.findall(r'"(publishedAt|applicationAt|registrationAt|graduatedAt)": "([^"]+)"', text) - for k, d in dates: - if not date_regex.match(d): - errors.append(f"Invalid ISO 8601 or out-of-bounds date in {fname}: {k}='{d}'") + for m in alumni_m: + val = m.get("graduatedAt") + if val and not date_regex.match(val): + errors.append(f"Invalid ISO 8601 date in alumnus '{m.get('ko')}': graduatedAt='{val}'") - # 3. Volume Cross-Check for Publications (Ensures volume date candidates match publishedAt) + # 7. Volume Cross-Check for Publications MON_PAT = r'(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' for pub in intl_p + dom_p: diff --git a/refer_landing_page/scripts/run_e2e_tests.py b/refer_landing_page/scripts/run_e2e_tests.py index 90c3872..060b3d7 100755 --- a/refer_landing_page/scripts/run_e2e_tests.py +++ b/refer_landing_page/scripts/run_e2e_tests.py @@ -86,31 +86,38 @@ def test_build(): print(f"❌ Build failed:\n{res.stdout}\n{res.stderr}") return False - lines = res.stdout.splitlines() - - # Verify exact dynamic routes + # Parse exact Next.js route table output lines (e.g. "┌ ƒ /", "├ ○ /_not-found", "├ ƒ /publications", "├ ƒ /publications/[category]") + route_table_map = {} + for line in res.stdout.splitlines(): + m = re.match(r'^[┌├└]\s+([ƒ○●])\s+(\S+)', line.strip()) + if m: + mode, route = m.group(1), m.group(2) + route_table_map[route] = mode + + print(f"Parsed route modes from build table ({len(route_table_map)} routes): {route_table_map}") + + # Check exact match for each dynamic route for d_route in EXPECTED_DYNAMIC_ROUTES: - found = False - for line in lines: - if d_route in line and ("ƒ" in line or "Dynamic" in line): - found = True - break - if not found: - print(f"❌ Gate 2 Violation: Expected dynamic route '{d_route}' not marked with 'ƒ Dynamic' in build report") + mode = route_table_map.get(d_route) + if mode != "ƒ": + print(f"❌ Gate 2 Violation: Expected dynamic route '{d_route}' has mode '{mode}', expected 'ƒ'") return False - # Verify exact static routes + # Check exact match for each static route for s_route in EXPECTED_STATIC_ROUTES: - found = False - for line in lines: - if s_route in line and ("○" in line or "Static" in line): - found = True - break - if not found: - print(f"❌ Gate 2 Violation: Expected static route '{s_route}' not marked with '○ Static' in build report") + mode = route_table_map.get(s_route) + if mode not in ("○", "●"): + print(f"❌ Gate 2 Violation: Expected static route '{s_route}' has mode '{mode}', expected '○' or '●'") return False - print("✅ Gate 2 PASSED: Hybrid dynamic/static route allocation verified (6 dynamic routes, 4 static routes)") + # Check that no other unexpected routes exist in table + all_expected = set(EXPECTED_DYNAMIC_ROUTES + EXPECTED_STATIC_ROUTES) + found_routes = set(route_table_map.keys()) + if all_expected != found_routes: + print(f"❌ Gate 2 Violation: Route set mismatch. Missing: {all_expected - found_routes}, Extra: {found_routes - all_expected}") + return False + + print("✅ Gate 2 PASSED: Hybrid dynamic/static route allocation verified (6 dynamic routes, 4 static routes with exact table matching)") return True def test_tsc(): @@ -122,6 +129,13 @@ def test_tsc(): print("✅ Gate 3 PASSED: TypeScript type check clean (exit code 0)") return True +import socket + +def get_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + return s.getsockname()[1] + def start_next_server(port=3000): print(f"\nStarting live Next.js server on port {port}...") server_env = os.environ.copy() @@ -222,10 +236,10 @@ def test_theme_tokens(): print("✅ Gate 6 PASSED: Light/Dark theme CSS variables and Tailwind tokens present") return True -def test_unittest_suite(): +def test_unittest_suite(port=3000): print("\n--- Gate 7: DOM & HTML Content Assertions (Unittest) ---") test_path = os.path.join(REFER_DIR, "tests", "e2e_test_suite.py") - res = run_cmd([sys.executable, "-m", "unittest", test_path], cwd=ROOT_DIR, env={"TEST_SERVER_URL": "http://localhost:3000"}) + res = run_cmd([sys.executable, "-m", "unittest", test_path], cwd=ROOT_DIR, env={"TEST_SERVER_URL": f"http://localhost:{port}"}) if res.returncode != 0: print(f"❌ Unittest suite failed:\n{res.stdout}\n{res.stderr}") return False @@ -439,13 +453,14 @@ def main(): sys.exit(1) # Start live Next.js server once and run Gate 4, Gate 7, Gate 8 + port = get_free_port() server_proc = None try: - server_proc = start_next_server(port=3000) - bodies = fetch_live_routes("http://localhost:3000") + server_proc = start_next_server(port=port) + bodies = fetch_live_routes(f"http://localhost:{port}") success &= test_live_routes(bodies) - success &= test_unittest_suite() + success &= test_unittest_suite(port=port) success &= test_layout_architecture(bodies) finally: if server_proc: