Add system

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+COPY . /app
+
+RUN pip install --no-cache-dir flask flask-cors pandas openpyxl
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

README.md

@@ -1,12 +1,11 @@
 ---
 title: Pledge Tracker
-emoji: 🐨
-colorFrom: blue
-colorTo: yellow
+colorFrom: purple
+colorTo: indigo
 sdk: gradio
-sdk_version: 5.44.1
+sdk_version: 5.34.0
 app_file: app.py
 pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+license: cc-by-nc-4.0
+short_description: Track and fact-check pledges with supporting evidence.
+---

app.py

@@ -0,0 +1,342 @@
+from flask import Flask, jsonify, send_file, request, send_from_directory
+from flask_cors import CORS
+import os, json, uuid, time
+import pandas as pd
+from datetime import datetime, timedelta
+from huggingface_hub import HfApi
+import sys
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from system.pledge_tracking import run_pipeline  
+from huggingface_hub import hf_hub_download
+import spacy
+import traceback
+import threading
+
+nlp = spacy.load("en_core_web_sm")
+
+app = Flask(__name__, static_folder='.')
+CORS(app)
+
+HF_DATASET_REPO = "PledgeTracker/demo_feedback"
+HF_TOKEN = os.environ.get("HF_TOKEN")
+TMP_DIR = "tmp"
+FEEDBACK_DIR = "feedback_logs"
+os.makedirs(TMP_DIR, exist_ok=True)
+os.makedirs(FEEDBACK_DIR, exist_ok=True)
+
+REFERENCE_PLEDGES = []
+
+REFERENCE_PLEDGE_PATH = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",    
+        filename="existing_pledges.txt",            
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]            
+    )
+
+if os.path.exists(REFERENCE_PLEDGE_PATH):
+    with open(REFERENCE_PLEDGE_PATH, "r") as f:
+        REFERENCE_PLEDGES = [line.strip() for line in f if line.strip()]
+else:
+    print(f"Missing reference pledge file: {REFERENCE_PLEDGE_PATH}")
+
+
+def lemmatize(text):
+    doc = nlp(text)
+    return " ".join([token.lemma_ for token in doc if not token.is_punct and not token.is_space])
+
+
+@app.route("/api/similar-pledges", methods=["POST"])
+def similar_pledges():
+    data = request.get_json()
+    claim = data.get("claim", "").strip()
+    if not claim or not REFERENCE_PLEDGES:
+        return jsonify({"suggestions": []})
+
+    all_pledges = [claim] + REFERENCE_PLEDGES
+    lemmatized_pledges = [lemmatize(p) for p in all_pledges]
+
+    vectorizer = TfidfVectorizer().fit_transform(lemmatized_pledges)
+    similarities = cosine_similarity(vectorizer[0:1], vectorizer[1:]).flatten()
+    filtered = [(i, similarities[i]) for i in range(len(similarities)) if similarities[i] > 0.3]
+    top_filtered = sorted(filtered, key=lambda x: x[1], reverse=True)[:5]
+
+    suggestions = [
+        {"text": REFERENCE_PLEDGES[i], "index": int(i)}
+        for i, score in top_filtered
+    ]
+
+    return jsonify({"suggestions": suggestions})
+
+
+def calculate_time_range(option: str, pledge_date: str = None):
+    today = datetime.today()
+    # pledge_date = datetime.strptime(pledge_date, "%Y-%m-%d")
+
+    if isinstance(pledge_date, str):
+        pledge_date = datetime.strptime(pledge_date, "%Y-%m-%d")
+    elif not isinstance(pledge_date, datetime):
+        raise ValueError("pledge_date must be a str or datetime")
+
+    if option == "week":
+        one_week_ago = today - timedelta(days=7)
+        start = max(one_week_ago, pledge_date)
+    elif option == "month":
+        one_month_ago = today - timedelta(days=30)
+        start = max(one_month_ago, pledge_date)
+    elif option == "since_pledge_date":
+        if not pledge_date:
+            raise ValueError("Pledge date is required for 'since_pledge_date' option")
+        start = pledge_date
+    else:
+        raise ValueError("Invalid time range option")
+    print(start, pledge_date)
+    return start.strftime("%Y%m%d"), today.strftime("%Y%m%d")
+
+@app.route("/")
+def serve_html():
+    return send_from_directory('.', 'test.html')
+
+@app.route("/api/status")
+def check_status():
+    user_id = request.args.get("user_id")
+    timestamp = request.args.get("timestamp")
+    log_file_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}_status.log")
+    if not os.path.exists(log_file_path):
+        return jsonify({"status": {}}), 200
+    try:
+        with open(log_file_path, "r") as f:
+            status = json.load(f)
+    except Exception:
+        status = {}
+
+    return jsonify({"status": status})
+
+
+@app.route("/api/run-model", methods=["POST"])
+def run_model():
+    data = request.get_json()
+    claim = data.get("claim", "no input")
+    time_range_option = data.get("time_range", "month")
+    system_start_time = datetime.now()
+
+    suggestion_meta = data.get("suggestion_meta") 
+    pledge_date = data.get("pledge_date", "")
+    pledge_author = data.get("pledge_author", "")
+    timestamp = data.get("timestamp") or time.strftime("%Y-%m-%d_%H-%M-%S")
+    user_id = data.get("user_id") or str(uuid.uuid4())[:8]
+
+    log_file_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}_status.log")
+
+    status_lock = threading.Lock()
+
+    def update_status(step_id, msg):
+        print(f"[STATUS] Step {step_id}: {msg}")
+        with status_lock:
+            if os.path.exists(log_file_path):
+                try:
+                    with open(log_file_path, "r") as f:
+                        current = json.load(f)
+                except Exception:
+                    current = {}
+            else:
+                current = {}
+            current[str(step_id)] = f"{msg}"
+            with open(log_file_path, "w") as f:
+                json.dump(current, f, indent=2)
+
+    try:
+        time_start, time_end = calculate_time_range(time_range_option, pledge_date=pledge_date)
+        print(f"[DEMO] Received claim: {claim}")
+        print(f"[DEMO] Time range: {time_start} ~ {time_end}")
+        print(f"[DEMO] Pledge date range: {pledge_date}")
+
+        # user_id = str(uuid.uuid4())[:8]  
+        # outputs = run_pipeline(claim, pledge_date, pledge_author, time_start, timestamp, user_id)
+
+
+        update_status(0, "📌 Starting the system ...")
+        print(suggestion_meta)
+
+        outputs = run_pipeline(
+                claim, pledge_date, pledge_author, time_start, timestamp, user_id,
+                update_fn=update_status, suggestion_meta=suggestion_meta
+            )
+
+        df = pd.read_excel(outputs["sorted_events"])
+        json_path = os.path.join(TMP_DIR, f"{timestamp}_{user_id}.json")
+        df.to_json(json_path, orient="records", indent=2)
+        
+
+        system_end_time = datetime.now()
+        runtime = system_end_time - system_start_time
+
+        events = df.to_dict(orient="records")
+        log_entry = {
+            "requested_time": timestamp,
+            "user_id": user_id,
+            "pledge": claim,
+            "suggestion_meta": suggestion_meta,
+            "time_start": time_start,
+            "time_end": time_end,
+            "runtime": runtime.total_seconds(),
+            "pledge_author": pledge_author,
+            "pledge_date": pledge_date,
+            "events": events
+        }
+        default_log_path = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+
+        with open(default_log_path, "w") as f:
+            f.write(json.dumps(log_entry, indent=1))
+
+        tsv_path = outputs["augmented_tsv_file"]
+
+        try:
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=default_log_path,
+                path_in_repo=f"logs/feedback_{timestamp}_{user_id}.jsonl",
+                repo_id=HF_DATASET_REPO,
+                repo_type="dataset",
+                token=HF_TOKEN
+            )
+            api.upload_file(
+                path_or_fileobj=tsv_path,
+                path_in_repo=f"logs/augmented_{timestamp}_{user_id}.tsv",
+                repo_id=HF_DATASET_REPO,
+                repo_type="dataset",
+                token=HF_TOKEN
+            )
+            
+
+        except Exception as e:
+            traceback.print_exc()
+            print(f"[Default Feedback Upload Error] {e}")
+
+        return jsonify({
+            "status": "success",
+            "file": f"{timestamp}_{user_id}.json",
+            "user_id": user_id,
+            "timestamp": timestamp
+        })
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify({"status": "error", "detail": str(e)}), 500
+
+@app.route("/api/events")
+def get_events():
+    filename = request.args.get("file")
+    file_path = os.path.join(TMP_DIR, filename)
+
+    if not os.path.exists(file_path):
+        return jsonify({"error": "File not found"}), 404
+
+    with open(file_path, "r") as f:
+        events = json.load(f)
+
+    return jsonify(events)
+
+
+@app.route("/api/feedback", methods=["POST"])
+def receive_feedback():
+    data = request.get_json()
+    pledge = data.get("pledge", "no_pledge_text")
+    feedback_list = data.get("feedback", [])
+    filename = data.get("file")
+    file_path = os.path.join(TMP_DIR, filename)
+
+    timestamp = data.get("timestamp") 
+    user_id = data.get("user_id")
+
+    if not user_id or not timestamp:
+        return jsonify({'status': 'error', 'detail': 'Missing user_id or timestamp'}), 400
+
+    if not os.path.exists(file_path):
+        return jsonify({"error": "Event file not found"}), 400
+
+    with open(file_path, "r") as f:
+        events = json.load(f)
+
+    suggestion_meta = None
+    time_start = None
+    time_end = None
+    try:
+        prev_log_path = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+        with open(prev_log_path, "r") as f:
+            previous_log = json.load(f)
+        suggestion_meta = previous_log.get("suggestion_meta")
+        time_start = previous_log.get("time_start")
+        time_end = previous_log.get("time_end")
+        pledge_author = previous_log.get("pledge_author")
+        pledge_date = previous_log.get("pledge_date")
+        runtime = previous_log.get("runtime")
+    except Exception:
+        pass  
+
+    feedback_dict = {int(item['eventIndex']): item['answer'] for item in feedback_list}
+    for idx, event in enumerate(events):
+        event["user_feedback"] = feedback_dict.get(idx)
+
+    log_entry = {
+        "requested_time": timestamp,
+        "user_id": user_id,
+        "pledge": pledge,
+        "suggestion_meta": suggestion_meta,
+        "time_start": time_start,
+        "time_end": time_end,
+        "runtime": runtime,
+        "pledge_author": pledge_author,
+        "pledge_date": pledge_date,
+        "events": events
+    }
+
+    local_filename = f"{FEEDBACK_DIR}/feedback_{timestamp}_{user_id}.jsonl"
+    with open(local_filename, "w") as f:
+        f.write(json.dumps(log_entry, indent=1))
+
+    try:
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=local_filename,
+            path_in_repo=f"logs/feedback_{timestamp}_{user_id}.jsonl",
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+    except Exception as e:
+        return jsonify({'status': 'partial_success', 'error': str(e)}), 500
+
+    return jsonify({'status': 'success'})
+
+
+@app.route("/download-feedback/<filename>")
+def download_feedback_file(filename):
+    return send_from_directory(FEEDBACK_DIR, filename, as_attachment=True)
+
+@app.route("/feedback-files")
+def list_feedback_files():
+    files = os.listdir(FEEDBACK_DIR)
+    return jsonify(sorted(files))
+
+@app.route("/download")
+def download_excel():
+    file = request.args.get("file")
+    if not file:
+        return "Missing file param", 400
+
+    json_path = os.path.join(TMP_DIR, file)
+    if not os.path.exists(json_path):
+        return "Event file not found", 404
+
+    with open(json_path, "r") as f:
+        data = json.load(f)
+
+    df = pd.DataFrame(data)
+    xlsx_path = os.path.join(TMP_DIR, file.replace(".json", ".xlsx"))
+    df.to_excel(xlsx_path, index=False)
+
+    return send_file(xlsx_path, as_attachment=True)
+
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0", port=7860)

requirements.txt

@@ -0,0 +1,25 @@
+flask
+flask_cors
+pandas
+openpyxl
+huggingface_hub
+PyMuPDF==1.23.25
+huggingface_hub==0.30.2
+lxml==5.3.1
+nltk==3.9.1
+numpy==2.2.6
+openai==1.84.0
+pandas==2.3.0
+rank_bm25==0.2.2
+Requests==2.32.3
+scikit_learn==1.7.0
+sentence_transformers==3.3.1
+spacy==3.8.2
+tiktoken==0.7.0
+torch==2.6.0
+tqdm
+trafilatura==2.0.0
+transformers==4.51.3
+vllm==0.8.4
+accelerate
+

system/augmented_searching.py

@@ -0,0 +1,98 @@
+import json
+import os
+import requests
+import pandas as pd
+from pathlib import Path
+from .date_verifier import is_after_start
+
+def google_search(query, api_key, search_engine_id, start_date, end_date):
+    # print(f"[SYSTEM] Calling Google Search API for: {query}")
+    sort = f"date:r:{start_date}:{end_date}" #20241230:20250130
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        "q": query,
+        "key": api_key,
+        "cx": search_engine_id,
+        "num": 10,
+        "sort": sort,
+        "cr": "countryUK",
+        "gl": "uk"
+    }
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        return response.json().get("items", [])
+    except Exception as e:
+        print(f"[ERROR] Google Search Failed: {e}")
+        return []
+
+def save_tsv(file_name, id_value, string_value, value_list, query):
+
+    data = {
+        'ID': id_value,
+        'String': string_value,
+        'ListValue': value_list,
+        'query': query
+    }
+    df = pd.DataFrame(data)
+    df.to_csv(file_name, sep='\t', index=False, header=False)
+
+def ensure_directory_exists(path):
+    dir_path = Path(path).expanduser().resolve().parent
+    if not str(dir_path).startswith("/home") and not str(dir_path).startswith("/data") and not str(dir_path).startswith("outputs"):
+        raise ValueError(f"[ERROR] Unsafe path: {dir_path}")
+    dir_path.mkdir(parents=True, exist_ok=True)
+
+def run_augmented_searching(qa_file, pipeline_base_dir, suggestion_meta, pledge_author, pledge_date, start_date, end_date):
+    if suggestion_meta==None:
+        qa_lines = open(f"{qa_file}","r").read()
+        qa_lines = json.loads(qa_lines)
+        claim_text = f"{pledge_author}: {qa_lines['claim']} ({pledge_date})"
+        idx=0
+    else:
+        # claim_text = suggestion_meta["text"]
+        idx = suggestion_meta["index"]
+        qa_lines = open(f"{qa_file}","r").readlines()[idx]
+        qa_lines = json.loads(qa_lines)
+        claim_text = f"{qa_lines['claim']}"
+        
+
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    search_engine_id = os.environ.get("GOOGLE_SEARCH_CX")
+    if not api_key or not search_engine_id:
+        raise EnvironmentError("[ERROR] GOOGLE_API_KEY and GOOGLE_SEARCH_CX must be set in environment.")
+
+    # base_dir = pipeline_base_dir
+
+    tsv_file_path = os.path.join(pipeline_base_dir, "augmented_search_results.tsv")
+    ensure_directory_exists(tsv_file_path)
+
+
+    urls = []
+    string_values = []
+    queries = []
+    questions = []
+    questions = [evidence["question"] for evidence in qa_lines["evidence"] if evidence["question"] not in questions]
+    questions = questions[:10]
+
+
+    results = google_search(claim_text, api_key, search_engine_id, start_date, end_date)
+    for result in results:
+        if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"] and is_after_start(result["link"], start_date):
+            string_values.append("claim")
+            urls.append(result["link"])
+            queries.append(f"{pledge_author}: {claim_text}")
+    
+    for question in questions:
+        results = google_search(f"{question}", api_key, search_engine_id, start_date, end_date)
+        for result in results:
+            if result["link"] not in urls and "fullfact.org/government-tracker" not in result["link"] and is_after_start(result["link"], start_date):
+                string_values.append("question")
+                urls.append(result["link"])
+                queries.append(f"{question}")
+
+    urls = list(dict.fromkeys(urls))
+
+    save_tsv(str(tsv_file_path), [0] * len(urls), string_values, urls, queries)
+    print(f"[SYSTEM] Saved {len(urls)} URLs for claim {idx} to {tsv_file_path}")
+    return str(tsv_file_path)

system/baseline/hyde_fc_generation_optimized.py

@@ -0,0 +1,163 @@
+from vllm import LLM, SamplingParams
+import json
+import torch
+import time
+from datetime import datetime, timedelta
+import argparse
+from tqdm import tqdm
+from typing import List, Dict, Any
+import concurrent.futures
+
+class VLLMGenerator:
+    def __init__(self, model_name: str, n: int = 8, max_tokens: int = 512, 
+                 temperature: float = 0.7, top_p: float = 1.0, 
+                 frequency_penalty: float = 0.0, presence_penalty: float = 0.0, 
+                 stop: List[str] = ['\n\n\n'], batch_size: int = 32):
+        self.device_count = torch.cuda.device_count()
+        print(f"Initializing with {self.device_count} GPUs")
+        self.llm = LLM(
+            model=model_name,
+            tensor_parallel_size=self.device_count,
+            max_model_len=4096,
+            gpu_memory_utilization=0.95,
+            enforce_eager=True,
+            trust_remote_code=True,
+            # quantization="bitsandbytes",
+            # dtype="half",
+            # load_format="bitsandbytes",
+            max_num_batched_tokens=4096,
+            max_num_seqs=batch_size
+        )
+        self.sampling_params = SamplingParams(
+            n=n,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            frequency_penalty=frequency_penalty,
+            presence_penalty=presence_penalty,
+            stop=stop,
+            logprobs=1
+        )
+        self.batch_size = batch_size
+        self.tokenizer = self.llm.get_tokenizer()
+        print(f"Initialization complete. Batch size: {batch_size}")
+    
+    def parse_response(self, responses):
+        all_outputs = []
+        for response in responses:
+            to_return = []
+            for output in response.outputs:
+                text = output.text.strip()
+                try:
+                    logprob = sum(logprob_obj.logprob for item in output.logprobs for logprob_obj in item.values())
+                except:
+                    logprob = 0  # Fallback if logprobs aren't available
+                to_return.append((text, logprob))
+            texts = [r[0] for r in sorted(to_return, key=lambda tup: tup[1], reverse=True)]
+            all_outputs.append(texts)
+        return all_outputs
+
+    def prepare_prompt(self, claim: str, model_name: str) -> str:
+        base_prompt = f"Please write a fact-checking article passage to support, refute, indicate not enough evidence, or present conflicting evidence regarding the claim.\nClaim: {claim}"
+        
+        if "OLMo" in model_name:
+            return base_prompt
+        else:
+            messages = [{"role": "user", "content": base_prompt}]
+            return self.tokenizer.apply_chat_template(messages, tokenize=False) + "<|start_header_id|>assistant<|end_header_id|>\n\nPassage: "
+
+    def process_batch(self, batch: List[Dict[str, Any]], model_name: str) -> tuple[List[Dict[str, Any]], float]:
+        start_time = time.time()
+        prompts = [self.prepare_prompt(example["claim"], model_name) for example in batch]
+        
+        try:
+            results = self.llm.generate(prompts, sampling_params=self.sampling_params)
+            outputs = self.parse_response(results)
+            
+            for example, output in zip(batch, outputs):
+                example['hypo_fc_docs'] = output
+            
+            batch_time = time.time() - start_time
+            return batch, batch_time
+        except Exception as e:
+            print(f"Error processing batch: {str(e)}")
+            return batch, time.time() - start_time
+
+# def format_time(seconds: float) -> str:
+#     return str(timedelta(seconds=int(seconds)))
+
+# def estimate_completion_time(start_time: float, processed_examples: int, total_examples: int) -> str:
+#     elapsed_time = time.time() - start_time
+#     examples_per_second = processed_examples / elapsed_time
+#     remaining_examples = total_examples - processed_examples
+#     estimated_remaining_seconds = remaining_examples / examples_per_second
+#     completion_time = datetime.now() + timedelta(seconds=int(estimated_remaining_seconds))
+#     return completion_time.strftime("%Y-%m-%d %H:%M:%S")
+
+def main(args):
+    total_start_time = time.time()
+    print(f"Script started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    
+    # Load data
+    print("Loading data...")
+    with open(args.target_data, 'r', encoding='utf-8') as json_file:
+        examples = json.load(json_file)
+    print(f"Loaded {len(examples)} examples")
+    
+    # Initialize generator
+    print("Initializing generator...")
+    generator = VLLMGenerator(
+        model_name=args.model,
+        batch_size=32
+    )
+    
+    # Process data in batches
+    processed_data = []
+    # batch_times = []
+    batches = [examples[i:i + generator.batch_size] for i in range(0, len(examples), generator.batch_size)]
+    
+    print(f"\nProcessing {len(batches)} batches...")
+    with tqdm(total=len(examples), desc="Processing examples") as pbar:
+        for batch_idx, batch in enumerate(batches, 1):
+            processed_batch, batch_time = generator.process_batch(batch, args.model)
+            processed_data.extend(processed_batch)
+            # batch_times.append(batch_time)
+            
+            # Update progress and timing information
+            # examples_processed = len(processed_data)
+            # avg_batch_time = sum(batch_times) / len(batch_times)
+            # estimated_completion = estimate_completion_time(total_start_time, examples_processed, len(examples))
+            
+            # pbar.set_postfix({
+            #     'Batch': f"{batch_idx}/{len(batches)}",
+            #     'Avg Batch Time': f"{avg_batch_time:.2f}s",
+            #     'ETA': estimated_completion
+            # })
+            # pbar.update(len(batch))
+    
+    # Calculate and display timing statistics
+    # total_time = time.time() - total_start_time
+    # avg_batch_time = sum(batch_times) / len(batch_times)
+    # avg_example_time = total_time / len(examples)
+    
+    # print("\nTiming Statistics:")
+    # print(f"Total Runtime: {format_time(total_time)}")
+    # print(f"Average Batch Time: {avg_batch_time:.2f} seconds")
+    # print(f"Average Time per Example: {avg_example_time:.2f} seconds")
+    # print(f"Throughput: {len(examples)/total_time:.2f} examples/second")
+    
+    # Save results
+    # print("\nSaving results...")
+    with open(args.json_output, "w", encoding="utf-8") as output_json:
+        json.dump(processed_data, output_json, ensure_ascii=False, indent=4)
+    
+    # print(f"Script completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    # print(f"Total runtime: {format_time(total_time)}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-i', '--target_data', default='data_store/averitec/dev.json')
+    parser.add_argument('-o', '--json_output', default='data_store/hyde_fc.json')
+    parser.add_argument('-m', '--model', default="meta-llama/Llama-3.1-8B-Instruct")
+    args = parser.parse_args()
+    main(args)

system/baseline/question_generation_optimized.py

@@ -0,0 +1,244 @@
+import os
+import argparse
+import time
+import json
+import nltk
+from rank_bm25 import BM25Okapi
+import numpy as np
+import torch
+from vllm import LLM, SamplingParams
+from datetime import datetime, timedelta
+from itertools import islice
+
+
+def download_nltk_data(package_name, download_dir='nltk_data'):
+    # Ensure the download directory exists
+    os.makedirs(download_dir, exist_ok=True)
+    
+    # Set NLTK data path
+    nltk.data.path.append(download_dir)
+    
+    try:
+        # Try to find the resource
+        nltk.data.find(f'tokenizers/{package_name}')
+        print(f"Package '{package_name}' is already downloaded")
+    except LookupError:
+        # If resource isn't found, download it
+        print(f"Downloading {package_name}...")
+        nltk.download(package_name, download_dir=download_dir)
+        print(f"Successfully downloaded {package_name}")
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+def claim2prompts(example): 
+    claim = example["claim"]
+    claim_str = "Example [NUMBER]:||Claim: " + claim + "||Evidence: "
+
+    for question in example["questions"]:
+        q_text = question["question"].strip()
+        if len(q_text) == 0:
+            continue
+
+        if not q_text[-1] == "?":
+            q_text += "?"
+
+        answer_strings = []
+
+        for a in question["answers"]:
+            if a["answer_type"] in ["Extractive", "Abstractive"]:
+                answer_strings.append(a["answer"])
+            if a["answer_type"] == "Boolean":
+                answer_strings.append(a["answer"]  + ", because " + a["boolean_explanation"].lower().strip())
+
+        for a_text in answer_strings:
+            if not a_text[-1] in [".", "!", ":", "?"]:
+                a_text += "."
+
+            prompt_lookup_str = a_text
+            this_q_claim_str = claim_str + a_text.strip() + "||Question: " + q_text 
+            yield (prompt_lookup_str, this_q_claim_str.replace("\n", " ").replace("||", "\n")[:1500])
+
+def main(args):
+    # script_start = time.time()
+    # start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    # print(f"Script started at: {start_time}")
+    # print(f"Loading model: {args.model}")
+
+
+    download_nltk_data('punkt')
+    download_nltk_data('punkt_tab')
+
+    # Load and prepare reference corpus
+    # corpus_start = time.time()
+    with open(args.reference_corpus, "r", encoding="utf-8") as json_file:
+        train_examples = json.load(json_file)
+
+    prompt_corpus, tokenized_corpus = [], []
+    for example in train_examples:
+        for lookup_str, prompt in claim2prompts(example):
+            entry = nltk.word_tokenize(lookup_str)
+            tokenized_corpus.append(entry)
+            prompt_corpus.append(prompt)
+
+    prompt_bm25 = BM25Okapi(tokenized_corpus)
+    # print(f"Reference corpus processed in: {format_time(time.time() - corpus_start)}")
+    
+    # Initialize vLLM with optimized settings
+    gpu_count = torch.cuda.device_count()
+    print(f"Using {gpu_count} GPU{'s' if gpu_count > 1 else ''}")
+    
+    # model_start = time.time()
+    llm = LLM(
+        model=args.model,
+        tensor_parallel_size=gpu_count,
+        max_model_len=4096,
+        gpu_memory_utilization=0.95,
+        enforce_eager=True,
+        trust_remote_code=True,
+        # dtype="half",
+    )
+    llm.get_tokenizer().pad_token = "<|end_of_text|>"
+    # print(f"Model loaded in: {format_time(time.time() - model_start)}")
+    
+    sampling_params = SamplingParams(
+        temperature=0.6,
+        top_p=0.9,
+        top_k=1,
+        skip_special_tokens=False,
+        max_tokens=512,
+        stop=['<|end_of_text|>', '</s>', '<|im_end|>', '[INST]', '[/INST]','<|eot_id|>','<|end|>','<|endoftext|>']
+    )
+
+    # processing_start = time.time()
+
+    # Load target data
+    target_examples = []
+    with open(args.top_k_target_knowledge, "r", encoding="utf-8") as json_file:
+        for line in json_file:
+            target_examples.append(json.loads(line))
+
+    if args.end == -1:
+        args.end = len(target_examples)
+    print(f"Processing {args.end} examples")
+
+    # Process in batches
+    with torch.no_grad():
+        with open(args.output_questions, "w", encoding="utf-8") as output_file:
+            for idx in range(0, args.end, args.batch_size):
+                batch_end = min(idx + args.batch_size, args.end)
+                current_batch = target_examples[idx:batch_end]
+                print(f"\nProcessing batch {idx}-{batch_end}...")
+                
+                for example in current_batch:
+                    # batch_start = time.time()
+                    claim = example["claim"]
+                    claim_id = example["claim_id"]
+                    top_k_sentences_urls = example[f"top_{args.top_k}"]
+                    
+                    batch_prompts = []
+                    batch_metadata = []
+                    
+                    # Prepare all prompts for current example
+                    for sentences_urls in top_k_sentences_urls:
+                        prompt_lookup_str = sentences_urls["sentence"]
+                        url = sentences_urls["url"]
+                        
+                        prompt_s = prompt_bm25.get_scores(nltk.word_tokenize(prompt_lookup_str))
+                        prompt_n = 10
+                        prompt_top_n = np.argsort(prompt_s)[::-1][:prompt_n]
+                        prompt_docs = [prompt_corpus[i] for i in prompt_top_n]
+                        
+                        temp_prompt = "\n\n".join(prompt_docs)
+                        for k in range(1, temp_prompt.count("[NUMBER]")+1):
+                            temp_prompt = temp_prompt.replace("[NUMBER]", f"{k}", 1)
+                            
+                        claim_prompt = "Your task is to generate a question based on the given claim and evidence. The question should clarify the relationship between the evidence and the claim\n\n"
+                        evidence = prompt_lookup_str.replace("\n", " ")
+                        full_prompt = claim_prompt + temp_prompt + "\n\nNow, generate a question that links the following claim and evidence:" + f"\n\nClaim: {claim}" + f"\nEvidence: {evidence}"
+                        
+                        if "OLMo" in args.model:
+                            inputs = [full_prompt]
+                        else:
+                            messages = [{"role":"user", "content":full_prompt}]
+                            inputs = llm.get_tokenizer().apply_chat_template(messages, tokenize=False)
+                            inputs += "<|start_header_id|>assistant<|end_header_id|>\n\nQuestion: "
+                            
+                        batch_prompts.append(inputs)
+                        batch_metadata.append((url, prompt_lookup_str))
+                    
+                    # Process batch
+                    outputs = llm.generate(batch_prompts, sampling_params)
+                    
+                    # Process outputs
+                    evidence = []
+                    for output, (url, sent) in zip(outputs, batch_metadata):
+                        question = output.outputs[0].text.strip().split("?")[0].replace("\n", " ") + "?"
+                        evidence.append({
+                            "question": question,
+                            "answer": sent,
+                            "url": url
+                        })
+                    
+                    # Write results
+                    json_data = {
+                        "claim_id": claim_id,
+                        "claim": claim,
+                        "evidence": evidence
+                    }
+                    output_file.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+                    output_file.flush()
+                    
+                    # batch_time = time.time() - batch_start
+                    # print(f"Processed example {claim_id}. Time elapsed: {batch_time:.2f}s")
+
+    # Calculate and display timing information
+    # total_time = time.time() - script_start
+    # processing_time = time.time() - processing_start
+    # end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    # print("\nTiming Summary:")
+    # print(f"Start time: {start_time}")
+    # print(f"End time: {end_time}")
+    # print(f"Total runtime: {format_time(total_time)}")
+    # print(f"Setup time: {format_time(processing_start - script_start)}")
+    # print(f"Processing time: {format_time(processing_time)}")
+    # print(f"Results written to: {args.output_questions}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Use a prompt to generate questions that could be answered by top-k retrieved evidence. Output generated questions.")
+    parser.add_argument("--model", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct")
+    parser.add_argument("--reference_corpus", default="baseline/train.json")
+    parser.add_argument(
+        "-i",
+        "--top_k_target_knowledge",
+        default="data_store/dev_reranking_top_k.json",
+        help="Directory where the sentences for the scraped data is saved.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output_questions",
+        default="data_store/dev_top_k_qa.json",
+        help="Directory where the sentences for the scraped data is saved.",
+    )
+    parser.add_argument(
+        "--top_k",
+        default=10,
+        type=int
+    )
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=4,
+        help="Number of examples to process in each batch"
+    )
+    parser.add_argument(
+        "-e",
+        "--end",
+        type=int,
+        default=-1
+    )
+    
+    args = parser.parse_args()
+    main(args)

system/baseline/reranking_optimized.py

@@ -0,0 +1,230 @@
+import os
+import torch
+import gc
+from transformers import AutoModel, AutoTokenizer
+from sentence_transformers import SentenceTransformer
+import numpy as np
+import json
+import argparse
+import time
+from datetime import datetime, timedelta
+import re
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+def encode_text(model, tokenizer, texts, batch_size=8, max_length=512):
+    """Encode texts to embeddings using AutoModel"""
+    all_embeddings = []
+    
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        
+        # Tokenize
+        encoded_input = tokenizer(
+            batch,
+            padding=True,
+            truncation=True,
+            max_length=max_length,
+            return_tensors='pt'
+        ).to(model.device)
+        
+        # Compute token embeddings
+        with torch.no_grad():
+            with torch.cuda.amp.autocast(dtype=torch.bfloat16):
+                model_output = model(**encoded_input)
+                # Use mean pooling
+                attention_mask = encoded_input['attention_mask']
+                token_embeddings = model_output[0]  # First element contains token embeddings
+                input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+                embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+                all_embeddings.append(embeddings.cpu().numpy())
+        
+        # Clear some memory
+        if i % (batch_size * 4) == 0:
+            torch.cuda.empty_cache()
+            gc.collect()
+    
+    return np.vstack(all_embeddings)
+
+def compute_similarity(emb1, emb2):
+    """Compute cosine similarity between embeddings"""
+    return np.dot(emb1, emb2.T) / (
+        np.linalg.norm(emb1, axis=1).reshape(-1, 1) * 
+        np.linalg.norm(emb2, axis=1).reshape(1, -1)
+    )
+
+def get_detailed_instruct(task_description: str, query: str) -> str:
+    return f'Instruct: {task_description}\nQuery: {query}'
+
+def preprocess_sentences(sentence1, sentence2):
+    vectorizer = TfidfVectorizer().fit_transform([sentence1, sentence2])
+    vectors = vectorizer.toarray()
+    
+    cosine_sim = cosine_similarity(vectors)
+    similarity_score = cosine_sim[0][1]
+    return similarity_score
+
+def remove_trailing_special_chars(text):
+    return re.sub(r'[\W_]+$', '', text)
+
+def remove_special_chars_except_spaces(text):
+    return re.sub(r'[^\w\s]+', '', text)
+
+def select_top_k(claim, results, top_k):
+    '''
+    remove sentence of similarity claim
+    '''
+    dup_check = set()
+    top_k_sentences_urls = []
+    
+    i = 0
+    # print(results)
+    claim = remove_special_chars_except_spaces(claim).lower()
+    while len(top_k_sentences_urls) < top_k and i < len(results):
+        # print(i)
+        sentence = remove_special_chars_except_spaces(results[i]['sentence']).lower()
+        
+        if sentence not in dup_check:
+            if preprocess_sentences(claim, sentence) > 0.97:
+                dup_check.add(sentence)
+                continue
+            
+            if claim in sentence:
+                if len(claim) / len(sentence) > 0.92:
+                    dup_check.add(sentence)
+                    continue 
+            
+            top_k_sentences_urls.append({
+                'sentence': results[i]['sentence'],
+                'url': results[i]['url']}
+            )
+        i += 1
+        
+    return top_k_sentences_urls
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+
+def compute_embeddings_batched(model, texts, batch_size=8):
+    """Compute embeddings in smaller batches to manage memory"""
+    all_embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        with torch.cuda.amp.autocast(dtype=torch.bfloat16):  # Use bfloat16
+            emb = model.encode(batch, batch_size=len(batch), show_progress_bar=False)
+            all_embeddings.append(emb)
+        
+        # Clear some memory
+        if i % (batch_size * 4) == 0:
+            torch.cuda.empty_cache()
+            gc.collect()
+    
+    return np.vstack(all_embeddings)
+
+def main(args):
+
+    
+    device = "cuda" if torch.cuda.is_available() else 'cpu'
+    print(f"Using device: {device}")
+    
+    # Load model and tokenizer
+    model = AutoModel.from_pretrained(
+        "Salesforce/SFR-Embedding-2_R",
+        torch_dtype=torch.bfloat16,
+        low_cpu_mem_usage=True,
+        device_map="auto"
+    )
+    tokenizer = AutoTokenizer.from_pretrained("Salesforce/SFR-Embedding-2_R")
+    
+    # Load target examples
+    target_examples = []
+    with open(args.target_data, "r", encoding="utf-8") as json_file:
+        for i, line in enumerate(json_file):
+            try:
+                example = json.loads(r"{}".format(line))
+                target_examples.append(example)
+            except:
+                print(f"CURRENT LINE broken {i}")
+    
+    if args.end == -1:
+        args.end = len(target_examples)
+    
+    files_to_process = list(range(args.start, args.end))
+    total = len(files_to_process)
+    
+    task = 'Given a web search query, retrieve relevant passages that answer the query'
+    
+    with open(args.json_output, "w", encoding="utf-8") as output_json:
+        done = 0
+        for idx, example in enumerate(target_examples):
+            if idx in files_to_process:
+                print(f"Processing claim {example['claim_id']}... Progress: {done + 1} / {total}")
+                
+                claim = example['claim']
+                query = [get_detailed_instruct(task, claim)] + [
+                    get_detailed_instruct(task, le) 
+                    for le in example['hypo_fc_docs'] 
+                    if len(le.strip()) > 0
+                ]
+                query_length = len(query)
+                sentences = [sent['sentence'] for sent in example[f'top_{5000}']][:args.retrieved_top_k]
+                
+                # st = time.time()
+                try:
+                    # Process query embeddings
+                    query_embeddings = encode_text(model, tokenizer, query, batch_size=4)
+                    avg_emb_q = np.mean(query_embeddings, axis=0)
+                    hyde_vector = avg_emb_q.reshape((1, -1))
+                    
+                    # Process sentence embeddings in smaller chunks
+                    sentence_embeddings = encode_text(
+                        model,
+                        tokenizer,
+                        sentences, 
+                        batch_size=args.batch_size
+                    )
+                    
+                    # Compute similarities in chunks to save memory
+                    chunk_size = 1000
+                    all_scores = []
+                    for i in range(0, len(sentence_embeddings), chunk_size):
+                        chunk = sentence_embeddings[i:i + chunk_size]
+                        chunk_scores = compute_similarity(hyde_vector, chunk)[0]
+                        all_scores.extend(chunk_scores)
+                    
+                    scores = np.array(all_scores)
+                    top_k_idx = np.argsort(scores)[::-1]
+                    results = [example['top_5000'][i] for i in top_k_idx]
+                    top_k_sentences_urls = select_top_k(claim, results, args.top_k)
+                    
+                    # print(f"Top {args.top_k} retrieved. Time elapsed: {time.time() - st:.2f}s")
+                    
+                    json_data = {
+                        "claim_id": example['claim_id'],
+                        "claim": claim,
+                        f"top_{args.top_k}": top_k_sentences_urls
+                    }
+                    output_json.write(json.dumps(json_data, ensure_ascii=False) + "\n")
+                    output_json.flush()
+                    
+                except RuntimeError as e:
+                    print(f"Error processing claim {example['claim_id']}: {e}")
+                    continue
+                
+                done += 1
+    
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target_data", default="data_store/dev_retrieval_top_k.json")
+    parser.add_argument("--retrieved_top_k", type=int, default=5000)
+    parser.add_argument("--top_k", type=int, default=10)
+    parser.add_argument("-o", "--json_output", type=str, default="data_store/dev_reranking_top_k.json")
+    parser.add_argument("--batch_size", type=int, default=32)
+    parser.add_argument("-s", "--start", type=int, default=0)
+    parser.add_argument("-e", "--end", type=int, default=-1)
+    args = parser.parse_args()
+    
+    main(args)

system/baseline/retrieval_optimized.py

@@ -0,0 +1,244 @@
+import argparse
+import json
+import os
+import time
+import numpy as np
+import pandas as pd
+import nltk
+from rank_bm25 import BM25Okapi
+from multiprocessing import Pool, cpu_count, Manager, Lock
+from functools import partial
+import heapq
+from threading import Thread, Event
+import queue
+from datetime import datetime, timedelta
+
+
+def download_nltk_data(package_name, download_dir='nltk_data'):
+    # Ensure the download directory exists
+    os.makedirs(download_dir, exist_ok=True)
+    
+    # Set NLTK data path
+    nltk.data.path.append(download_dir)
+    
+    try:
+        # Try to find the resource
+        nltk.data.find(f'tokenizers/{package_name}')
+        print(f"Package '{package_name}' is already downloaded")
+    except LookupError:
+        # If resource isn't found, download it
+        print(f"Downloading {package_name}...")
+        nltk.download(package_name, download_dir=download_dir)
+        print(f"Successfully downloaded {package_name}")
+
+
+def combine_all_sentences(knowledge_file):
+    sentences, urls = [], []
+    
+    with open(knowledge_file, "r", encoding="utf-8") as json_file:
+        for i, line in enumerate(json_file):
+            data = json.loads(line)
+            sentences.extend(data["url2text"])
+            urls.extend([data["url"] for _ in range(len(data["url2text"]))])
+    return sentences, urls, i + 1
+
+def remove_duplicates(sentences, urls):
+    df = pd.DataFrame({"document_in_sentences":sentences, "sentence_urls":urls})
+    df['sentences'] = df['document_in_sentences'].str.strip().str.lower()
+    df = df.drop_duplicates(subset="sentences").reset_index()
+    return df['document_in_sentences'].tolist(), df['sentence_urls'].tolist()
+                
+def retrieve_top_k_sentences(query, document, urls, top_k):
+    tokenized_docs = [nltk.word_tokenize(doc) for doc in document[:top_k]]
+    bm25 = BM25Okapi(tokenized_docs)
+    
+    scores = bm25.get_scores(nltk.word_tokenize(query))
+    top_k_idx = np.argsort(scores)[::-1][:top_k]
+
+    return [document[i] for i in top_k_idx], [urls[i] for i in top_k_idx]
+
+def process_single_example(idx, example, args, result_queue, counter, lock):
+    try:
+        with lock:
+            current_count = counter.value + 1
+            counter.value = current_count
+            print(f"\nProcessing claim {idx}... Progress: {current_count} / {args.total_examples}")
+        
+        # start_time = time.time()
+        
+        document_in_sentences, sentence_urls, num_urls_this_claim = combine_all_sentences(
+            os.path.join(args.knowledge_store_dir, f"{idx}.jsonl")
+        )
+        
+        print(f"Obtained {len(document_in_sentences)} sentences from {num_urls_this_claim} urls.")
+        
+        document_in_sentences, sentence_urls = remove_duplicates(document_in_sentences, sentence_urls)
+        
+        query = example["claim"] + " " + " ".join(example['hypo_fc_docs'])
+        top_k_sentences, top_k_urls = retrieve_top_k_sentences(
+            query, document_in_sentences, sentence_urls, args.top_k
+        )
+        
+
+        result = {
+            "claim_id": idx,
+            "claim": example["claim"],
+            f"top_{args.top_k}": [
+                {"sentence": sent, "url": url}
+                for sent, url in zip(top_k_sentences, top_k_urls)
+            ],
+            "hypo_fc_docs": example['hypo_fc_docs']
+        }
+        
+        result_queue.put((idx, result))
+        return True
+    except Exception as e:
+        print(f"Error processing example {idx}: {str(e)}")
+        result_queue.put((idx, None))
+        return False
+
+def writer_thread(output_file, result_queue, total_examples, stop_event):
+    next_index = 0
+    pending_results = []
+    
+    with open(output_file, "w", encoding="utf-8") as f:
+        while not (stop_event.is_set() and result_queue.empty()):
+            try:
+                idx, result = result_queue.get(timeout=1)
+                
+                if result is not None:
+                    heapq.heappush(pending_results, (idx, result))
+                
+                while pending_results and pending_results[0][0] == next_index:
+                    _, result_to_write = heapq.heappop(pending_results)
+                    f.write(json.dumps(result_to_write, ensure_ascii=False) + "\n")
+                    f.flush()
+                    next_index += 1
+                    
+            except queue.Empty:
+                continue
+
+# def format_time(seconds):
+#     """Format time duration nicely."""
+#     return str(timedelta(seconds=round(seconds)))
+
+def main(args):
+
+
+
+    download_nltk_data('punkt')
+    download_nltk_data('punkt_tab')
+    
+    with open(args.target_data, "r", encoding="utf-8") as json_file:
+        target_examples = json.load(json_file)
+
+    if args.end == -1:
+        args.end = len(target_examples)
+    
+    print(f"Total examples to process: {args.end - args.start}")
+
+    files_to_process = list(range(args.start, args.end))
+    examples_to_process = [(idx, target_examples[idx]) for idx in files_to_process]
+    
+    num_workers = min(args.workers if args.workers > 0 else cpu_count(), len(files_to_process))
+    print(f"Using {num_workers} workers to process {len(files_to_process)} examples")
+
+    with Manager() as manager:
+        counter = manager.Value('i', 0)
+        lock = manager.Lock()
+        args.total_examples = len(files_to_process)
+        
+        result_queue = manager.Queue()
+        
+        stop_event = Event()
+        writer = Thread(
+            target=writer_thread,
+            args=(args.json_output, result_queue, len(files_to_process), stop_event)
+        )
+        writer.start()
+
+        process_func = partial(
+            process_single_example,
+            args=args,
+            result_queue=result_queue,
+            counter=counter,
+            lock=lock
+        )
+        
+        with Pool(num_workers) as pool:
+            results = pool.starmap(process_func, examples_to_process)
+        
+        stop_event.set()
+        writer.join()
+        
+        # successful = sum(1 for r in results if r)
+        # print(f"\nSuccessfully processed {successful} out of {len(files_to_process)} examples")
+        # print(f"Results written to {args.json_output}")
+        
+        # # Calculate and display timing information
+        # total_time = time.time() - script_start
+        # avg_time = total_time / len(files_to_process)
+        # end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        # print("\nTiming Summary:")
+        # print(f"Start time: {start_time}")
+        # print(f"End time: {end_time}")
+        # print(f"Total runtime: {format_time(total_time)} (HH:MM:SS)")
+        # print(f"Average time per example: {avg_time:.2f} seconds")
+        # if successful > 0:
+        #     print(f"Processing speed: {successful / total_time:.2f} examples per second")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Get top 10000 sentences with BM25 in the knowledge store using parallel processing."
+    )
+    parser.add_argument(
+        "-k",
+        "--knowledge_store_dir",
+        type=str,
+        default="data_store/knowledge_store",
+        help="The path of the knowledge_store_dir containing json files with all the retrieved sentences.",
+    )
+    parser.add_argument(
+        "--target_data",
+        type=str,
+        default="data_store/hyde_fc.json",
+        help="The path of the file that stores the claim.",
+    )
+    parser.add_argument(
+        "-o",
+        "--json_output",
+        type=str,
+        default="data_store/dev_retrieval_top_k.json",
+        help="The output dir for JSON files to save the top 100 sentences for each claim.",
+    )
+    parser.add_argument(
+        "--top_k",
+        default=5000,
+        type=int,
+        help="How many documents should we pick out with BM25.",
+    )
+    parser.add_argument(
+        "-s",
+        "--start",
+        type=int,
+        default=0,
+        help="Starting index of the files to process.",
+    )
+    parser.add_argument(
+        "-e",
+        "--end",
+        type=int,
+        default=-1,
+        help="End index of the files to process.",
+    )
+    parser.add_argument(
+        "-w",
+        "--workers",
+        type=int,
+        default=0,
+        help="Number of worker processes (default: number of CPU cores)",
+    )
+
+    args = parser.parse_args()
+    main(args)

system/date_verifier.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+import re, trafilatura
+from trafilatura.settings import DEFAULT_CONFIG
+
+DEFAULT_CONFIG.MAX_FILE_SIZE = 50000  
+
+_URL_DATE_PATS = [
+    re.compile(r"(?P<y>\d{4})-(?P<m>\d{2})-(?P<d>\d{2})"),  # 2025-07-03
+    re.compile(r"(?P<y>\d{4})/(?P<m>\d{2})/(?P<d>\d{2})"),  # 2025/07/03
+    re.compile(r"(?P<y>\d{4})(?P<m>\d{2})(?P<d>\d{2})"),    # 20250703
+]
+
+def _meta_date(url: str):
+
+    page = trafilatura.fetch_url(url, config=DEFAULT_CONFIG)
+    if not page:
+        return None
+    meta = trafilatura.extract_metadata(page)
+    if not meta or not meta.date:
+        return None
+    try:
+        return datetime.fromisoformat(meta.date)
+    except ValueError:
+
+        try:
+            return datetime.fromisoformat(meta.date.split("T")[0])
+        except Exception:
+            return None
+
+def _regex_date(url: str):
+
+    for pat in _URL_DATE_PATS:
+        m = pat.search(url)
+        if m:
+            try:
+                return datetime(
+                    int(m.group("y")), int(m.group("m")), int(m.group("d"))
+                )
+            except ValueError:
+                pass
+    return None
+
+
+def is_after_start(url: str, start_ymd: str) -> bool:
+    """
+    - start_ymd: 'YYYYMMDD'
+    """
+    t0 = datetime.strptime(start_ymd, "%Y%m%d")
+
+    pub_dt = _meta_date(url)
+
+    if pub_dt is None:
+        pub_dt = _regex_date(url)
+
+    if pub_dt is None:
+        return True
+
+    return pub_dt >= t0

system/ee.py

@@ -0,0 +1,104 @@
+import json
+import os
+import argparse
+from tqdm import tqdm
+import tiktoken
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+
+def gpt_4o(input_text):
+    client=OpenAI(api_key=os.environ.get("OAI"))
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "user", "content": [{"type": "text", "text": input_text}]}
+        ],
+        response_format={"type": "json_object"},
+        temperature=0,
+        max_tokens=4096,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=0
+    )
+    return response.choices[0].message.content
+
+def run_gpt4_event_extraction(data_dir, max_tokens=100000):
+
+    all_info_path = os.path.join(data_dir, "all_info_with_txt.json")
+    output_dir = os.path.join(data_dir, "gpt4_event_extraction")
+    os.makedirs(output_dir, exist_ok=True)
+    icl_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="icl.txt",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+    ICL = open(icl_path, "r").read()
+    all_info = open(all_info_path, "r").readlines()
+
+    enc = tiktoken.encoding_for_model("gpt-4o")
+
+    for i, line in enumerate(all_info):
+        ID = i
+        urls = []
+        results = []
+
+        data = json.loads(line)
+        docs = data["evidence"]
+        claim = data["claim"]
+
+        output_path = os.path.join(output_dir, f"gpt4o_results_{ID}_claim.json")
+        if os.path.exists(output_path):
+            print(f"Already exist: {output_path}")
+
+        else:
+
+            for doc in tqdm(docs):
+                if doc["url"] in urls:
+                    continue
+
+                text = " ".join(doc["text"])
+                input_text = (
+                    f"{ICL}\nNow please only summarize events that are useful for verifying the pledge '{claim}', and their dates in the JSON format.\n\nInput:\n\nTitle: {doc['metadata']['title']}\n"
+                    f"Date: {doc['metadata']['date']}\nArticle: {text}\nPledge: {claim}\n\n"
+                    f"Output:\n"
+                )
+
+                urls.append(doc["url"])
+                text_tokens = enc.encode(input_text)
+                if len(text_tokens) > max_tokens:
+                    input_text = enc.decode(text_tokens[:max_tokens])
+
+                try:
+                    output = gpt_4o(input_text)
+                    # print(f"GPT-4o Response: {output}")
+                    results.append({
+                        "url": doc["url"],
+                        "title": doc["metadata"]["title"],
+                        "date": doc["metadata"]["date"],
+                        "article": text,
+                        "output": json.loads(output)
+                    })
+                except Exception as e:
+                    print(f"Error processing doc: {e}")
+                    continue
+
+            
+            with open(output_path, "w", encoding="utf-8") as f:
+                json.dump(results, f, ensure_ascii=False, indent=4)
+
+    return output_path
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run GPT-4o event extraction")
+    parser.add_argument("--data_dir", type=str, required=True, help="Root data directory")
+    parser.add_argument("--icl_path", type=str, required=True, help="Path to ICL prompt file")
+    parser.add_argument("--max_tokens", type=int, default=100000, help="Maximum token limit for input")
+
+    args = parser.parse_args()
+
+    run_gpt4_event_extraction(
+        base_dir=args.base_dir,
+        icl_path=args.icl_path,
+        max_tokens=args.max_tokens
+    )

system/generate_output.py

@@ -0,0 +1,75 @@
+import json
+import os
+import argparse
+from system.html2lines import html2metadata
+from lxml.etree import tostring
+import lxml.etree
+
+def process_manifesto_data_with_metadata(input_base_dir: str):
+
+    input_file_path = os.path.join(input_base_dir, "hero/manifesto_icl_reranking_top_k_QA.json")
+    output_file_path = os.path.join(input_base_dir, "all_info_with_txt.json")
+
+    url2text_dir = os.path.join(input_base_dir, "augmented_data_store")
+
+    with open(input_file_path, "r", encoding="utf-8") as f:
+        input_file = f.readlines()
+
+    out_file = open(output_file_path, "w", encoding="utf-8")
+
+
+    i = 0
+
+    for id, line in enumerate(input_file):
+        line = json.loads(line)
+        claim = line["claim"]
+        QAs = line["top_50"]
+        new_line = {"claim": claim, "evidence": []}
+
+        json_path = os.path.join(url2text_dir, f"{id}.jsonl")
+        if not os.path.exists(json_path):
+            print(f"Warning: {json_path} not found")
+            continue
+
+        with open(json_path, "r", encoding="utf-8") as f:
+            try:
+                data_store = json.load(f)
+            except json.JSONDecodeError:
+                f.seek(0)
+                data_store = [json.loads(line) for line in f]
+
+        url_txt = {data["url"]: data["url2text"] for data in data_store}
+
+        URLs = []
+        for j, QA in enumerate(QAs):
+            newQA = QA.copy()
+            URL = QA["url"]
+            newQA["text"] = url_txt.get(URL, "")
+
+            if URL not in URLs:
+                try:
+                    meta = html2metadata(URL)
+                    if isinstance(meta, lxml.etree._Element):
+                        meta = tostring(meta, encoding="unicode", pretty_print=True)
+                    meta_save = {
+                        "title": meta["title"],
+                        "date": meta["date"]
+                    }
+                except Exception as e:
+                    print(f"Metadata extraction failed for URL: {URL}, error: {e}")
+                    meta_save = {
+                        "title": "",
+                        "date": ""
+                    }
+
+
+                newQA["metadata"] = meta_save
+                new_line["evidence"].append(newQA)
+
+        out_file.write(json.dumps(new_line) + "\n")
+
+    out_file.close()
+    return output_file_path
+
+
+

system/hero_pipeline.py

@@ -0,0 +1,150 @@
+import os
+from datetime import datetime
+import subprocess
+from huggingface_hub import hf_hub_download
+import json
+
+def run_hero_reranking(pipeline_base_dir, suggestion_meta):
+    base_dir = f"{pipeline_base_dir}"
+    hero_dir = os.path.join(base_dir, "hero")
+    os.makedirs(hero_dir, exist_ok=True)
+
+    if suggestion_meta:
+        hyde_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="manifesto_icl_hyde_fc.json",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+        with open(hyde_path, "r", encoding="utf-8") as f:
+            all_hyde_data = json.load(f)
+
+        idx = suggestion_meta["index"]
+        single_hyde = [all_hyde_data[idx]]
+        save_path = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+        with open(save_path, "w", encoding="utf-8") as f:
+            json.dump(single_hyde, f, indent=2)
+    
+    hyde_output = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+
+    def safe_run(cmd, timeout=600):
+        try:
+            print(f"👉 Running: {' '.join(str(x) for x in cmd)}")
+            subprocess.run(cmd, check=True, timeout=timeout)
+        except subprocess.CalledProcessError as e:
+            print(f"[❌ ERROR] Subprocess failed: {e}")
+            if e.stderr:
+                print("[stderr]:", e.stderr.decode())
+            raise
+        except subprocess.TimeoutExpired:
+            print(f"[❌ TIMEOUT] Command timed out: {' '.join(cmd)}")
+            raise
+
+    # Step 3.2: retrieval
+    print("🔍 Step 3.2: Retrieval from knowledge store ...")
+    knowledge_store_dir = os.path.join(base_dir, "augmented_data_store")
+    retrieval_output = os.path.join(hero_dir, "manifesto_icl_retrieval_top_k_QA.json")
+
+    if not os.path.exists(retrieval_output):
+        safe_run([
+            "python", "system/baseline/retrieval_optimized.py",
+            "--knowledge_store_dir", knowledge_store_dir,
+            "--target_data", hyde_output,
+            "--json_output", retrieval_output,
+        ])
+
+    # Step 3.3: reranking
+    print("🏷️ Step 3.3: Reranking retrieved evidence ...")
+    rerank_output = os.path.join(hero_dir, "manifesto_icl_reranking_top_k_QA.json")
+
+    if not os.path.exists(rerank_output):
+        safe_run([
+            "python", "system/baseline/reranking_optimized.py",
+            "--target_data", retrieval_output,
+            "--json_output", rerank_output,
+            "--top_k", str(50),
+        ])
+
+    return {
+        "hyde": hyde_output,
+        "retrieved": retrieval_output,
+        "reranked": rerank_output,
+    }
+
+
+def run_hero_pipeline(pipeline_base_dir):
+    base_dir = f"{pipeline_base_dir}"
+    hero_dir = os.path.join(base_dir, "hero")
+    os.makedirs(hero_dir, exist_ok=True)
+
+    target_data = os.path.join(base_dir, "claim.json")
+    hyde_output = os.path.join(hero_dir, "manifesto_icl_hyde_fc.json")
+
+    def safe_run(cmd, timeout=600):
+        try:
+            print(f"👉 Running: {' '.join(cmd)}")
+            subprocess.run(cmd, check=True, timeout=timeout)
+        except subprocess.CalledProcessError as e:
+            print(f"[❌ ERROR] Subprocess failed: {e}")
+            if e.stderr:
+                print("[stderr]:", e.stderr.decode())
+            raise
+        except subprocess.TimeoutExpired:
+            print(f"[❌ TIMEOUT] Command timed out: {' '.join(cmd)}")
+            raise
+
+    # Step 3.1: hyde_fc_generation
+    if not os.path.exists(hyde_output):
+        print("🧠 Step 3.1: HyDE ICL generation ...")
+        safe_run([
+            "python", "system/baseline/hyde_fc_generation_optimized.py",
+            "--target_data", target_data,
+            "--json_output", hyde_output
+        ])
+
+    # Step 3.2: retrieval
+    print("🔍 Step 3.2: Retrieval from knowledge store ...")
+    knowledge_store_dir = os.path.join(base_dir, "initial_data_store")
+    retrieval_output = os.path.join(hero_dir, "manifesto_icl_retrieval_top_k.json")
+
+    if not os.path.exists(retrieval_output):
+        safe_run([
+            "python", "system/baseline/retrieval_optimized.py",
+            "--knowledge_store_dir", knowledge_store_dir,
+            "--target_data", hyde_output,
+            "--json_output", retrieval_output
+        ])
+
+    # Step 3.3: reranking
+    print("🏷️ Step 3.3: Reranking retrieved evidence ...")
+    rerank_output = os.path.join(hero_dir, "manifesto_icl_reranking_top_k.json")
+
+    if not os.path.exists(rerank_output):
+        safe_run([
+            "python", "system/baseline/reranking_optimized.py",
+            "--target_data", retrieval_output,
+            "--json_output", rerank_output
+        ])
+
+    # Step 3.4: question generation
+    print("❓ Step 3.4: Generating QA pairs ...")
+    reference_corpus = "system/baseline/train.json"
+    qa_output = os.path.join(hero_dir, "manifesto_icl_top_k_qa.json")
+
+    if not os.path.exists(qa_output):
+        safe_run([
+            "python", "system/baseline/question_generation_optimized.py",
+            "--reference_corpus", reference_corpus,
+            "--top_k_target_knowledge", rerank_output,
+            "--output_questions", qa_output,
+            "--model", "meta-llama/Meta-Llama-3.1-8B-Instruct"
+        ])
+
+    return {
+        "hyde": hyde_output,
+        "retrieved": retrieval_output,
+        "reranked": rerank_output,
+        "qa_pairs": qa_output
+    }
+
+

system/html2lines.py

@@ -0,0 +1,105 @@
+import sys
+from time import sleep
+import trafilatura
+from trafilatura.meta import reset_caches
+from trafilatura.settings import DEFAULT_CONFIG
+import spacy
+from lxml.etree import tostring
+import lxml.etree
+
+
+import spacy
+import subprocess
+
+try:
+    nlp = spacy.load("en_core_web_lg")
+except OSError:
+    print("🔁 Downloading spaCy model 'en_core_web_lg' ...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_lg"], check=True)
+    nlp = spacy.load("en_core_web_lg")
+
+
+DEFAULT_CONFIG.MAX_FILE_SIZE = 50000
+MIN_CHAR = 50
+MAX_CHAR = 5000
+
+
+def get_page(url):
+    page = None
+    for _ in range(3):
+        try:
+            page = trafilatura.fetch_url(url, config=DEFAULT_CONFIG)
+            assert page is not None
+            print("Fetched " + url, file=sys.stderr)
+            break
+        except:
+            sleep(3)
+    return page
+
+
+def url2lines(url):
+    page = get_page(url)
+
+    if page is None:
+        return []
+
+    lines = html2lines(page)
+    return lines
+
+
+def line_correction(lines, max_size=100):
+    out_lines = []
+    for line in lines:
+        if len(line) < MIN_CHAR:
+            continue
+
+        if len(line) > max_size:
+            doc = nlp(
+                line[:MAX_CHAR]
+            )  # We split lines into sentences, but for performance we take only the first 5k characters per line
+            stack = ""
+            for sent in doc.sents:
+                if len(stack) > 0:
+                    stack += " "
+                stack += str(sent).strip()
+                if len(stack) > max_size:
+                    out_lines.append(stack)
+                    stack = ""
+
+            if (
+                len(stack) > MIN_CHAR
+            ):  # Ensure every lines in the out_lines suffice the MIN_CHAR restriction
+                out_lines.append(stack)
+        else:
+            out_lines.append(line)
+
+    return out_lines
+
+
+def html2lines(page):
+    out_lines = []
+
+    if len(page.strip()) == 0 or page is None:
+        return out_lines
+
+    text = trafilatura.extract(page, config=DEFAULT_CONFIG)
+    reset_caches()
+
+    if text is None:
+        return out_lines
+
+    return text.split(
+        "\n"
+    )  # We just spit out the entire page, so need to reformat later.
+
+
+def html2metadata(url):
+    page = get_page(url)
+    metadata = trafilatura.extract_metadata(page)
+    return metadata.as_dict()
+
+if __name__ == "__main__":
+    url = "https://www.bbc.co.uk/news/61407508"
+    metadata = html2metadata(url)
+    text = " ".join(html2lines(page))
+    print(metadata)

system/initial_searching.py

@@ -0,0 +1,100 @@
+import json
+import os
+import time
+import requests
+import pandas as pd
+from datetime import datetime
+from pathlib import Path
+import spacy
+import subprocess
+
+try:
+    nlp = spacy.load("en_core_web_sm")
+except OSError:
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"], check=True)
+    nlp = spacy.load("en_core_web_sm")
+
+def clean_keywords(text):
+    doc = nlp(text)
+    keywords = []
+    for chunk in doc.noun_chunks:
+        words = [token.text for token in chunk if not token.is_stop and token.is_alpha]
+        if words:
+            cleaned_phrase = " ".join(words)
+            if len(cleaned_phrase) > 2:
+                keywords.append(cleaned_phrase)
+    return list(set(keywords))
+
+def google_search(query, api_key, search_engine_id, start_date, end_date):
+    print(f"[SYSTEM] Calling Google Search API for: {query}")
+    sort = f"date:r:{start_date}:{end_date}"
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        "q": query,
+        "key": api_key,
+        "cx": search_engine_id,
+        "num": 10,
+        "sort": sort,
+        "cr": "countryUK",
+        "gl": "uk"
+    }
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        return response.json().get("items", [])
+    except Exception as e:
+        print(f"[ERROR] Google Search Failed: {e}")
+        return []
+
+def save_tsv(file_path, claim_id, claim_text, url_list):
+    df = pd.DataFrame({
+        'ID': [claim_id] * len(url_list),
+        'String': ["claim"] * len(url_list),
+        'ListValue': url_list,
+        'query': [claim_text] * len(url_list)
+    })
+    df.to_csv(file_path, sep='\t', index=False, header=False)
+
+def ensure_directory_exists(path):
+    dir_path = Path(path).expanduser().resolve().parent
+    if not str(dir_path).startswith("/home") and not str(dir_path).startswith("/data") and not str(dir_path).startswith("outputs"):
+        raise ValueError(f"[ERROR] Unsafe path: {dir_path}")
+    dir_path.mkdir(parents=True, exist_ok=True)
+
+def run_initial_searching(claim_text, pipeline_base_dir, start_date, end_date, user_id, claim_id):
+    api_key = os.environ.get("GOOGLE_API_KEY")
+    search_engine_id = os.environ.get("GOOGLE_SEARCH_CX")
+    if not api_key or not search_engine_id:
+        raise EnvironmentError("[ERROR] GOOGLE_API_KEY and GOOGLE_SEARCH_CX must be set in environment.")
+
+    base_dir = pipeline_base_dir
+    manifesto_json_file = os.path.join(base_dir,"claim.json")
+    tsv_file_path = os.path.join(base_dir,"initial_search_results.tsv")
+
+    ensure_directory_exists(tsv_file_path)
+
+    claim_record = {"claim_id": claim_id, "claim": claim_text}
+    # if manifesto_json_file.exists():
+    #     with open(manifesto_json_file, "r") as f:
+    #         records = json.load(f)
+    # else:
+    records = []
+    records.append(claim_record)
+    with open(manifesto_json_file, "w") as f:
+        json.dump(records, f, indent=1)
+
+    urls = []
+    results = google_search(f"{claim_text}", api_key, search_engine_id, start_date, end_date)
+    urls += [r["link"] for r in results if "link" in r]
+    keywords = clean_keywords(claim_text)
+    keyword_text = " ".join(keywords)
+    # for kw in keywords:
+    #     results = google_search(kw, api_key, search_engine_id, start_date, end_date)
+    #     urls += [r["link"] for r in results if "link" in r]
+    results = google_search(keyword_text, api_key, search_engine_id, start_date, end_date)
+    urls += [r["link"] for r in results if "link" in r]
+    urls = list(dict.fromkeys(urls))
+
+    save_tsv(str(tsv_file_path), claim_id, claim_text, urls)
+    print(f"[SYSTEM] Saved {len(urls)} URLs for claim {claim_id} to {tsv_file_path}")
+    return str(tsv_file_path), str(manifesto_json_file)

system/pledge_tracking.py

@@ -0,0 +1,247 @@
+from huggingface_hub import login
+from datetime import datetime
+import os, time
+import pandas as pd
+
+from system.initial_searching import run_initial_searching
+from system.scraper import run_scraper
+from system.hero_pipeline import run_hero_pipeline, run_hero_reranking
+from system.augmented_searching import run_augmented_searching
+from system.generate_output import process_manifesto_data_with_metadata
+from system.ee import run_gpt4_event_extraction
+from system.process_time import extract_and_sort_events
+import spacy
+import subprocess
+from huggingface_hub import hf_hub_download
+import json
+
+try:
+    spacy.load("en_core_web_sm")
+except OSError:
+    print("🔁 Downloading en_core_web_sm model ...")
+    subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
+    nlp = spacy.load("en_core_web_sm")
+
+
+def count_total_events(output_path):
+    with open(output_path, "r", encoding="utf-8") as f:
+        results = json.load(f)
+
+    total_events = 0
+    for result in results:
+        total_events+= len(result["output"]["events"])
+
+    print(f"{total_events} events in total")
+    return total_events
+
+def run_pipeline(claim, pledge_date, pledge_author, start_date, timestamp, user_id, update_fn=None, suggestion_meta=None):
+    pipeline_base_dir = f"outputs/{timestamp}_{user_id}"
+    os.makedirs(pipeline_base_dir, exist_ok=True)
+
+    step_id=1
+
+    # Step 1: Google 搜索
+    if suggestion_meta==None:
+        
+
+        print("🔍 Step 1: Initial searching ...")
+        initial_tsv_file, claim_json_path = run_initial_searching(
+            claim_text=f"{pledge_author} : {claim} ({pledge_date})",
+            # pledge_author=pledge_author,
+            pipeline_base_dir=pipeline_base_dir,
+            start_date=start_date,
+            end_date="",
+            user_id=user_id,
+            claim_id=0,
+        )
+        with open(initial_tsv_file, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f)
+        if update_fn:
+            update_fn(step_id, f"{line_count} URLs are retrieved")
+            step_id+=1
+        
+
+        print("🌐 Step 2: Scraping URLs ...")
+        initial_data_store_dir = os.path.join(pipeline_base_dir, "initial_data_store")
+        os.makedirs(initial_data_store_dir, exist_ok=True)
+        initial_scraped_output_path = os.path.join(initial_data_store_dir, "0.jsonl")
+        run_scraper(initial_tsv_file, initial_scraped_output_path)
+
+        with open(initial_scraped_output_path, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f if json.loads(line)["url2text"] != [])
+        if update_fn: 
+            update_fn(step_id, f"{line_count} URL pages have been successefully scraped")
+            step_id+=1
+
+
+        print("🧠 Step 3: HerO processing ...")
+        hero_output_dir = os.path.join(pipeline_base_dir, "hero")
+        os.makedirs(hero_output_dir, exist_ok=True)
+        run_hero_pipeline(pipeline_base_dir)    
+
+        qa_file_path = os.path.join(hero_output_dir, "manifesto_icl_top_k_qa.json")
+
+        with open(qa_file_path, "r", encoding="utf-8") as f:
+            questions = {line["question"] for line in json.load(f)["evidence"]}
+            questions = list(questions)
+            line_count = len(questions)
+        if update_fn: 
+            update_fn(step_id, f"{line_count} relevant queries are generated, for example:\n"
+                      f"    1. {questions[0]}\n"
+                      f"    2. {questions[1]}\n"
+                      f"    3. {questions[2]}\n"
+                      f"    4. {questions[3]}\n"
+                      f"    5. {questions[4]}")
+            step_id+=1
+
+    else: 
+        claim_json_path = None
+        initial_scraped_output_path = None
+        initial_tsv_file = None
+        hero_output_dir = None
+        qa_file_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="manifesto_with_QA_icl_top_k_qa.json",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+        idx = suggestion_meta["index"]
+        qa_lines = open(f"{qa_file_path}","r").readlines()[idx]
+        questions = {line["question"] for line in json.loads(qa_lines)["evidence"]}
+        questions = list(questions)
+        line_count = len(questions)
+        if update_fn: 
+            update_fn(step_id, f"relevant queries are generated, for example:\n"
+                      f"    1. {questions[0]}\n"
+                      f"    2. {questions[1]}\n"
+                      f"    3. {questions[2]}\n"
+                      f"    4. {questions[3]}\n"
+                      f"    5. {questions[4]}")
+            step_id+=1
+
+    try:
+        augmented_tsv_file = run_augmented_searching(
+            qa_file=qa_file_path,
+            pledge_author=pledge_author,
+            pledge_date=pledge_date,
+            pipeline_base_dir=pipeline_base_dir,
+            start_date=start_date,
+            suggestion_meta=suggestion_meta,
+            end_date="",
+        )
+
+        
+
+        with open(augmented_tsv_file, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f)
+        if update_fn:
+            update_fn(step_id, f"{line_count} URLs are retrieved")
+            step_id+=1
+    except Exception as e:
+        if update_fn:
+            update_fn(step_id, f"❌ run_augmented_searching failed: {e}")
+        raise
+
+    augmented_data_store_dir = os.path.join(pipeline_base_dir, "augmented_data_store")
+    os.makedirs(augmented_data_store_dir, exist_ok=True)
+
+    try: 
+        augmented_scraped_output_path = os.path.join(augmented_data_store_dir, "0.jsonl")
+        run_scraper(augmented_tsv_file, augmented_scraped_output_path)
+
+        with open(augmented_scraped_output_path, "r", encoding="utf-8") as f:
+            line_count = sum(1 for line in f if json.loads(line)["url2text"] != [])
+        if update_fn: 
+            update_fn(step_id, f"{line_count} URL pages have been successefully scraped")
+            step_id+=1
+    except Exception as e:
+        if update_fn:
+            update_fn(step_id, f"❌ run_scraper failed: {e}")
+        raise
+    
+
+    try:
+        run_hero_reranking(pipeline_base_dir, suggestion_meta)
+
+        meta_data_dir = process_manifesto_data_with_metadata(input_base_dir=pipeline_base_dir)
+        all_info_path = os.path.join(pipeline_base_dir, "all_info_with_txt.json")
+        unique_urls = set()
+        with open(all_info_path, "r", encoding="utf-8") as f:
+            for line in f:
+                data = json.loads(line)
+                docs = data.get("evidence", [])
+                for doc in docs:
+                    if "url" in doc:
+                        unique_urls.add(doc["url"])
+        if update_fn: 
+            update_fn(step_id, f"{len(unique_urls)} documents are selected")
+            step_id+=1
+    except Exception as e:
+        if update_fn:
+            update_fn(step_id, f"❌ run_hero_reranking failed: {e}")
+        raise
+    
+    try:
+        extracted_event_path = run_gpt4_event_extraction(data_dir=pipeline_base_dir, max_tokens=100000)
+
+        events_num = count_total_events(extracted_event_path)
+
+        if update_fn: 
+            update_fn(step_id, f"{events_num} events are extracted from those documents.")
+            step_id+=1
+    except Exception as e:
+        if update_fn:
+            update_fn(step_id, f"❌ Event extraction failed: {e}")
+        raise
+
+
+    print("📅 Sorting events temporally ...")
+
+    
+    sorted_events = extract_and_sort_events(
+        data_dir=pipeline_base_dir, 
+        pledge_date=pledge_date, 
+        pledge_author=pledge_author, 
+        claim=claim,
+        suggestion_meta=suggestion_meta
+        )
+
+    df = pd.DataFrame(sorted_events)
+    sorted_event_path = f"{pipeline_base_dir}/sorted_events.xlsx"
+    df.to_excel(sorted_event_path, index=False)
+    print(sorted_event_path)
+
+    if update_fn: 
+        update_fn(step_id, "All done!")
+        step_id += 1
+
+    return {
+        "claim_json": claim_json_path,
+        "initial_scraped_jsonl": initial_scraped_output_path,
+        "initial_tsv_file": initial_tsv_file,
+        "hero_dir": hero_output_dir,
+        "augmented_scraped_jsonl": augmented_scraped_output_path,
+        "augmented_tsv_file": augmented_tsv_file,
+        "meta_data_dir": meta_data_dir,
+        "unsorted_events": extracted_event_path,
+        "sorted_events": sorted_event_path,
+        "step_id": step_id
+    }
+
+
+if __name__ == "__main__":
+    start = time.time()
+
+    if os.environ.get("HF_TOKEN"):
+        login(token=os.environ["HF_TOKEN"])
+    else:
+        print("No Hugging Face token found in environment variable HF_TOKEN.")
+
+    claim = "“We will support families with children by introducing free breakfast clubs in every primary school”"
+    start_date = "20250504"
+    timestamp = "xxxxx"
+    user_id = "xxx"
+
+    outputs = run_pipeline(claim, time_start, timestamp, user_id)
+    print("🎯 Pipeline finished. Outputs:", outputs)
+    print(f"⏱️ Total time: {time.time() - start:.2f} seconds")

system/process_time.py

@@ -0,0 +1,267 @@
+import json
+import datetime
+import re
+import pandas as pd
+import os, argparse
+import random
+import csv
+from openai import OpenAI
+from huggingface_hub import hf_hub_download
+import json
+import os
+
+
+
+def gpt_4o_useful(input):
+    client=OpenAI(api_key=os.environ.get("OAI"))
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": input
+                    }
+                ]
+            }
+        ],
+        response_format={"type": "text"},
+        temperature=0.0000000001,
+        max_tokens=4096,
+        top_p=0,
+        frequency_penalty=0,
+        presence_penalty=0,
+        logprobs=True
+    )
+
+    text = response.choices[0].message.content
+
+    if response.choices[0].logprobs and response.choices[0].logprobs.content:
+        first_token_logprob = response.choices[0].logprobs.content[0]
+        token = first_token_logprob.token
+        logprob = first_token_logprob.logprob
+    else:
+        token = None
+        logprob = None
+
+    return text, token, logprob
+
+
+
+def get_ICL(data, top_k=None):
+
+    ICL =""
+    if top_k == None:
+        data = data
+    else:
+        # print(data)
+        data = data[:top_k]
+    for line in data:
+        # line = json.loads(line)
+        pledge = line["pledge"]
+        event = line["event_description"]
+        time = line["event_date"]
+        input=f"Pledge: {pledge}\nEvent Summary: {event} (Event Date: {time})\nIs this event summary useful to track the fulfilment of this pledge"
+        input = input.strip()
+        output = line["label"].strip()
+        ICL = f"{ICL}Input: {input}\nOutput: {output}\n\n"
+    return ICL
+
+def load_json(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    return data
+
+
+def gpt_eval(test_instance, train_data, instruction, suggestion_meta, ICL_id=None):
+
+    if suggestion_meta:
+        # print(ICL_id)
+        
+        train_data = [line for line in train_data if str(line.get("pledge_id")) == str(ICL_id)]
+
+    else:
+        random.seed(42)
+        random.shuffle(train_data)
+
+    ICL = get_ICL(train_data, top_k=50)
+    # print(ICL)
+    input = f"{instruction}\nBelow are examples:\n\n{ICL}Now, please assign a label for the below instance.\nInput: {test_instance}\nOutput:"
+
+    try:
+        text, tokens, logprobs = gpt_4o_useful(input)
+    except Exception as e:
+        print(e)
+        tokens = None
+        logprobs = None
+
+    return tokens, logprobs
+
+def extract_columns_to_dict(file_path, delimiter='\t'):
+
+    data_dict = {}
+
+    with open(file_path, mode='r', encoding='utf-8') as file:
+        reader = csv.reader(file, delimiter=delimiter)
+        for row in reader:
+            if len(row) >= 4:  
+                key = row[2]  
+                value = row[3]  
+                data_dict[key] = value 
+
+    return data_dict
+
+
+import datetime
+import re
+
+def parse_date(date_str):
+    if not date_str:
+        return None, date_str
+    date_str = date_str.strip()
+
+    # Case 1: YYYY-MM-DD
+    try:
+        return datetime.datetime.strptime(date_str, "%Y-%m-%d"), date_str
+    except ValueError:
+        pass
+
+    # Case 2: Relative date
+    match = re.search(r'(.*) \(relative to (\d{4}-\d{2}-\d{2})\)', date_str)
+    if match:
+        reference = datetime.datetime.strptime(match.group(2), "%Y-%m-%d")
+        relative_term = match.group(1).strip().lower()
+        if relative_term == "last month":
+            target_date = reference - datetime.timedelta(days=30)
+        elif relative_term == "yesterday":
+            target_date = reference - datetime.timedelta(days=1)
+        elif relative_term == "last week":
+            target_date = reference - datetime.timedelta(days=7)
+        elif relative_term == "this week":
+            target_date = reference
+        else:
+            return None, date_str
+        return target_date, date_str  
+
+    # Case 3: YYYY
+    match = re.fullmatch(r'(\d{4})', date_str)
+    if match:
+        year = int(match.group(1))
+        return datetime.datetime(year, 1, 1), date_str
+
+    # Case 4: Month YYYY
+    match = re.fullmatch(r'(\w+) (\d{4})', date_str)
+    if match:
+        try:
+            target_date = datetime.datetime.strptime(date_str, "%B %Y")
+            return target_date, date_str
+        except ValueError:
+            return None, date_str
+
+    # Case 5: YYYY-QX
+    match = re.fullmatch(r'(\d{4})-Q(\d)', date_str)
+    if match:
+        year, quarter = int(match.group(1)), int(match.group(2))
+        month = (quarter - 1) * 3 + 1
+        return datetime.datetime(year, month, 1), date_str
+
+    # Case 6: YYYY Season
+    match = re.fullmatch(r'(\d{4}) (Spring|Summer|Autumn|Fall|Winter)', date_str, re.IGNORECASE)
+    if match:
+        year = int(match.group(1))
+        season_map = {"spring": 3, "summer": 6, "autumn": 9, "fall": 9, "winter": 12}
+        month = season_map[match.group(2).lower()]
+        return datetime.datetime(year, month, 1), date_str
+
+    return None, date_str
+
+
+def extract_and_sort_events(data_dir, pledge_date, pledge_author, claim, suggestion_meta):
+
+    events = []
+
+    # url_path = os.path.join(data_dir, "augmented_search_results.tsv")
+    # url_query_dict = extract_columns_to_dict(file_path=url_path, delimiter='\t')
+    
+    pledge = claim.strip()
+
+    file_path = os.path.join(data_dir, "gpt4_event_extraction", "gpt4o_results_0_claim.json")
+    gpt4_results_json = load_json(file_path)
+
+    # print(gpt4_results_json)
+    train_file_path = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",     
+        filename="train_useful.json",              
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]              
+    )
+
+    with open(train_file_path, "r", encoding="utf-8") as f:
+        train_data = json.load(f)
+        # print(train_data[0])
+
+    
+
+    instruction_path = hf_hub_download(
+                repo_id="PledgeTracker/demo_feedback",   
+                filename="instruction.txt",            
+                repo_type="dataset",                     
+                token=os.environ["HF_TOKEN"]            
+            )
+    
+    instruction = open(instruction_path, "r").read()
+    
+    map_file_path = hf_hub_download(
+        repo_id="PledgeTracker/demo_feedback",     
+        filename="mapping.txt",              
+        repo_type="dataset",                      
+        token=os.environ["HF_TOKEN"]              
+    )
+    mapping_f = open(map_file_path, "r").readlines()
+    mapping = {}
+
+    for map_id, line in enumerate(mapping_f):
+        mapping[map_id] = int(line.strip())
+
+    ICL_id = None
+    if suggestion_meta:
+        try:
+            idx = int(suggestion_meta["index"])
+            ICL_id = mapping.get(idx)
+            print(f"[Suggestion] index: {idx} → pledge_id: {ICL_id}")
+        except Exception as e:
+            print(f"[Mapping error]: {e}")
+
+    for doc in gpt4_results_json:
+        mete_date = doc["date"]
+        for event in doc.get("output", {}).get("events", []):
+            parsed_date, original_date = parse_date(event["date"])
+
+            if parsed_date:
+                parsed_date_str = parsed_date.strftime("%Y-%m-%d")
+                if parsed_date_str != mete_date:
+                    event_date_and_pub_date = f"{parsed_date_str} ({mete_date})"
+                else:
+                    event_date_and_pub_date = parsed_date_str
+
+                test_instance = f"Pledge: {pledge} (Speaker: {pledge_author}; Pledge Date: {pledge_date})\nEvent Summary: {event['event']} (Event Date: {original_date})\nIs this event summary useful to track the fulfilment of this pledge"
+
+                label, score = gpt_eval(test_instance, train_data, instruction, suggestion_meta, ICL_id=ICL_id)
+
+                URL = doc["url"]
+                events.append({
+                    "date": original_date,  
+                    "event date (publication date if different)": event_date_and_pub_date,
+                    "event": event["event"],
+                    "url": URL,
+                    "label": label,
+                    "confident": score,
+                })
+    
+    events.sort(key=lambda x: parse_date(x["date"])[0], reverse=True)
+    return events
+
+
+

system/scraper.py

@@ -0,0 +1,95 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import os
+import csv
+import json
+import fitz
+import time
+import requests
+import pandas as pd
+from time import sleep
+from pathlib import Path
+from system.html2lines import url2lines, line_correction, html2metadata
+
+MAX_RETRIES = 3
+TIMEOUT = 5  # seconds
+
+
+def scrape_text_from_url(url, temp_name):
+    response = None
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = requests.get(url, timeout=TIMEOUT)
+            break
+        except requests.RequestException:
+            if attempt < MAX_RETRIES - 1:
+                sleep(3)
+
+    if response is None or response.status_code == 503:
+        return []
+
+    if url.endswith(".pdf"):
+        pdf_dir = Path("/tmp/pdf_dir")
+        pdf_dir.mkdir(parents=True, exist_ok=True)
+        pdf_path = pdf_dir / f"{temp_name}.pdf"
+        with open(pdf_path, "wb") as f:
+            f.write(response.content)
+
+        extracted_text = ""
+        doc = fitz.open(str(pdf_path))
+        for page in doc:
+            extracted_text += page.get_text() or ""
+
+        return line_correction(extracted_text.split("\n"))
+
+    return line_correction(url2lines(url))
+    
+def process_row(row, claim_id):
+    try:
+        url = row[2]
+        json_data = {
+            "claim_id": claim_id,
+            "type": row[1],
+            "query": row[3],
+            "url": url,
+            "url2text": scrape_text_from_url(url, claim_id),
+            "metadata": {}
+        }
+        meta = html2metadata(url)
+        json_data["metadata"] = {
+            "title": meta.get("title"),
+            "date": meta.get("date")
+        }
+        return json_data
+    except Exception as e:
+        print(f"[WARN] Failed to scrape {row[2]}: {e}")
+        return None
+
+def run_scraper(tsv_file_path: str, output_jsonl_path: str, max_workers: int = 10):
+    claim_id = Path(tsv_file_path).stem
+    output_jsonl_path = Path(output_jsonl_path)
+    output_jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if output_jsonl_path.exists():
+        print(f"[INFO] Skipping processing as output file already exists: {output_jsonl_path}")
+        return str(output_jsonl_path)
+
+    try:
+        df = pd.read_csv(tsv_file_path, sep="\t", header=None)
+        print("[INFO] Data loaded successfully with Pandas.")
+    except Exception as e:
+        raise RuntimeError(f"[ERROR] Failed to load TSV: {e}")
+
+    results = []
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_row, row, claim_id) for _, row in df.iterrows()]
+        for future in as_completed(futures):
+            result = future.result()
+            if result:
+                results.append(result)
+
+    with open(output_jsonl_path, "w", encoding="utf-8") as json_file:
+        for item in results:
+            json_file.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+    print(f"[SYSTEM] Output saved to {output_jsonl_path}")
+    return str(output_jsonl_path)

test.html

@@ -0,0 +1,553 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Pledge Tracker – Demo</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+  </head>
+  <body class="bg-gray-50 text-gray-800">
+    <header class="bg-white shadow py-4 sticky top-0 z-10">
+      <div class="container mx-auto flex items-center justify-between px-4">
+        <div class="flex items-center gap-2">
+          <span class="text-2xl font-bold text-purple-600">🤗</span>
+          <span class="font-semibold text-lg">Pledge Tracking</span>
+        </div>
+        <nav class="hidden md:flex gap-6 font-medium">
+          <a class="hover:text-purple-600" href="#eval-response">Track Your Pledge</a>
+          <a class="hover:text-purple-600" href="#about">About</a>
+        </nav>
+      </div>
+    </header>
+
+    <section class="py-16 bg-gradient-to-r from-purple-50 to-purple-50 text-center">
+      <div class="container mx-auto px-4 max-w-3xl">
+        <h1 class="text-3xl md:text-4xl font-extrabold mb-6">
+          <span style="font-variant: small-caps; font-weight: bold;">PledgeTracker</span>: A System for Monitoring the Fulfilment of Pledges
+        </h1>
+        <div class="text-lg text-gray-600 leading-relaxed space-y-4 text-justify">
+          <p>
+            <span style="font-variant: small-caps;">PledgeTracker</span> is a system to monitor the fulfilment of political pledges. As part of this study, we will collect your inputs to help evaluate and improve the system. We may also collect your feedback if you submit it via the feedback form. No personal information will be collected, and all data will be anonymised and stored securely. By using the system, you agree to participate in this study under these conditions.
+          </p>
+          <p class="text-center">
+            Please contact 
+            <a href="mailto:[email protected]" class="text-purple-600 underline">Andreas Vlachos</a> 
+            and 
+            <a href="mailto:[email protected]" class="text-purple-600 underline">Yulong Chen</a> 
+            if you have any concerns.
+          </p>
+        </div>
+      </div>
+    </section>
+    
+    <section id="eval-response" class="py-12">
+      <div class="container mx-auto px-4 max-w-4xl">
+        <!-- <h2 class="text-2xl font-bold mb-6">Track Manifesto Pledge</h2> -->
+        <label for="claim" class="block text-sm font-medium mb-2">
+          Please enter the pledge:
+        </label>
+        <textarea
+          id="claim"
+          class="w-full border rounded-lg p-3 h-40 focus:outline-none focus:ring-2 focus:ring-purple-500"
+          placeholder="For example: 'We will support families with children by introducing free breakfast clubs in every primary school...'"
+        ></textarea>
+
+        <div id="similar-suggestions" class="mt-3 text-sm text-gray-600 hidden"></div>
+
+        <div class="mt-4">
+          <label for="pledge-date" class="block text-sm font-medium mb-2">
+            When was this pledge made?
+          </label>
+          <div class="grid grid-cols-[1fr_auto] items-center gap-2">
+            <input
+              type="date"
+              id="pledge-date"
+              class="w-full border rounded-lg p-2"
+            />
+            <button
+              onclick="setDefaultDate()"
+              type="button"
+              class="px-2 py-1 text-sm bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+            >
+              Use default: 4th Jul 2024
+            </button>
+          </div>
+          <div id="date-warning" class="text-sm text-red-600 mt-1 hidden">
+            Please select a date or click the button to use the default.
+          </div>
+        </div>
+        
+        <div class="mt-4">
+          <label for="pledge-author" class="block text-sm font-medium mb-2">
+            Who made this pledge?
+          </label>
+          <div class="grid grid-cols-[1fr_auto] items-center gap-2">
+            <input
+              type="text"
+              id="pledge-author"
+              class="w-full border rounded-lg p-2"
+              placeholder="Enter the name of the party or person"
+            />
+            <button
+              onclick="setDefaultAuthor()"
+              type="button"
+              class="px-2 py-1 text-sm bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+            >
+              Use default: Labour
+            </button>
+          </div>
+          <div id="author-warning" class="text-sm text-red-600 mt-1 hidden">
+            Please enter a speaker or click the button to use the default.
+          </div>
+        </div>
+        
+        
+
+        <label for="time-range" class="block text-sm font-medium mt-4 mb-2">
+          Please select a time range:
+        </label>
+        <select id="time-range" class="w-full border rounded-lg p-2">
+          <option value="week">Past one week</option>
+          <option value="month">Past one month</option>
+          <!-- <option value="year">From when the pledge was made</option> -->
+          <option value="since_pledge_date">From when the pledge was made</option>
+        </select>
+
+        <button
+          id="check"
+          class="mt-4 px-6 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500"
+        >
+          Let's track!
+        </button>
+
+        <div id="progress" class="mt-6 hidden border p-4 rounded-lg bg-white shadow">
+          <h3 class="font-semibold mb-2">System Progress</h3>
+          <div id="status" class="text-sm text-gray-800 font-normal leading-relaxed"></div>
+        </div>
+
+
+        <div id="result" class="mt-6 hidden border p-4 rounded-lg bg-white shadow">
+          <h3 class="font-semibold mb-2">Result</h3>
+          <p class="text-gray-700"></p>
+        </div>
+      </div>
+    </section>
+
+    <section id="about" class="py-12">
+      <div class="container mx-auto px-4 max-w-4xl">
+        <h2 class="text-2xl font-bold mb-6">About</h2>
+        <p class="text-gray-700 leading-relaxed">
+          <span style="font-variant: small-caps;">PledgeTracker</span> is a research prototype developed to support the monitoring of political pledge fulfilment. 
+          This demo is developed by researchers at the University of Cambridge, Queen Mary University London, and Full Fact.
+        </p>
+      </div>
+    </section>
+
+
+    
+
+    <script>
+      let suggestedPledge = null; 
+      let currentAbortController = null;
+      const feedbackData = {};
+      let lastUsedFile = null;
+      let lastUserId = null;
+      let lastTimestamp = null;
+      const checkBtn = document.getElementById("check");
+
+      const stepListStandard = {
+        1: "Retrieving evidence related to the pledge",
+        2: "Scraping documents from URLs",
+        3: "Generating more queries based on the retrieved evidence",
+        4: "Searching more articles",
+        5: "Scraping documents from URLs",
+        6: "Finding the most relevant documents",
+        7: "Extracting events from top documents",
+        8: "Sorting events temporally"
+      };
+
+      const stepListSuggestion = {
+        1: "Generating queries to retrieve evidence",
+        2: "Searching more articles",
+        3: "Scraping documents from URLs",
+        4: "Finding the most relevant documents",
+        5: "Extracting events from top documents",
+        6: "Sorting events temporally"
+      };
+
+      let stepList = stepListStandard;
+
+      function renderStatus(statusDict) {
+        let html = "<ul class='list-disc ml-6 space-y-1 text-sm'>";
+        for (let step in stepList) {
+          const raw = statusDict?.[step] || stepList[step];
+          const content = raw.replace(/\n/g, "<br>");
+          const prefix = statusDict?.[step] ? "✅" : "⏳";
+          html += `<li>${prefix} Step ${step}: ${content}</li>`;
+        }
+        html += "</ul>";
+        return html;
+      }
+
+      function setDefaultDate() {
+        const input = document.getElementById("pledge-date");
+        input.value = "2024-07-04";
+        document.getElementById("date-warning").classList.add("hidden");
+      }
+
+      function setDefaultAuthor() {
+        const input = document.getElementById("pledge-author");
+        input.value = "Labour";
+        document.getElementById("author-warning").classList.add("hidden");
+      }
+
+      // function setFeedback(index, answer) {
+      //   feedbackData[index] = answer;
+      //   const message = document.getElementById(`msg-${index}`);
+      //   message.textContent = `✓ Selected: ${answer ? 'Yes' : 'No'}`;
+      //   message.className = answer
+      //     ? "text-sm text-green-600 mt-1"
+      //     : "text-sm text-red-600 mt-1";
+      // }
+      function setFeedback(index, answer) {
+        feedbackData[index] = answer;
+        const message = document.getElementById(`msg-${index}`);
+        
+        let displayText = "";
+        let colorClass = "";
+        
+        switch(answer) {
+          case "not_relevant":
+            displayText = "Not relevant";
+            colorClass = "text-red-300";
+            break;
+          case "relevant_seen":
+            displayText = "Relevant but already seen";
+            colorClass = "text-grey-400";
+            break;
+          case "relevant_updated":
+            displayText = "Relevant and up-to-date";
+            colorClass = "text-blue-400";
+            break;
+        }
+        
+        message.textContent = `✓ Selected: ${displayText}`;
+        message.className = `text-sm ${colorClass} mt-1`;
+      }
+
+      function pollStatus(userId, timestamp, statusElement) {
+        if (window.pollIntervalId) {
+          clearInterval(window.pollIntervalId);
+        }
+
+        window.pollIntervalId = setInterval(async () => {
+          try {
+            const res = await fetch(`/api/status?user_id=${userId}&timestamp=${timestamp}&_=${Date.now()}`);
+            const data = await res.json();
+
+            if (data.status) {
+              statusElement.innerHTML = renderStatus(data.status);
+            }
+
+            const values = Object.values(data.status || {});
+            const finalText = values.join(" ").toLowerCase();
+
+            if (finalText.includes("done") || finalText.includes("finished")) {
+                clearInterval(window.pollIntervalId);
+                window.pollIntervalId = null;
+                statusElement.innerHTML += `<div class="mt-2 text-green-600 font-semibold">✅ All done.</div>`;
+                checkBtn.disabled = false;
+                checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+
+                suggestedPledge = null;
+
+                const waitForFile = setInterval(() => {
+                  if (lastUsedFile) {
+                    clearInterval(waitForFile);
+                    loadEvents(lastUsedFile);
+                  }
+                }, 200);
+              } else if (Object.values(data.status || {}).some(v => v.startsWith("❌"))) {
+              clearInterval(window.pollIntervalId);
+              window.pollIntervalId = null;
+              statusElement.innerHTML += `<div class="mt-2 text-red-600 font-semibold">❌ The process failed.</div>`;
+              checkBtn.disabled = false;
+              checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+            }
+          } catch (err) {
+            clearInterval(window.pollIntervalId);
+            window.pollIntervalId = null;
+            statusElement.innerHTML = `<div class="text-red-600">❌ Failed to check status: ${err.message}</div>`;
+          }
+        }, 2000);
+      }
+
+
+
+      async function submitAllFeedback() {
+        const entries = Object.entries(feedbackData);
+        if (entries.length === 0) {
+          alert("No feedback to submit!");
+          return;
+        }
+        const confirmed = confirm("By submitting feedback, you agree that your feedback may be collected for our analysis. Your data will be anonymised and stored securely. No personal information will be recorded. If you do not wish to take part, please cancel this submission.");
+        if (!confirmed) return;
+
+        const pledgeText = document.getElementById("claim").value.trim();
+
+        const res = await fetch('/api/feedback', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            pledge: pledgeText,
+            file: lastUsedFile,
+            user_id: lastUserId,       
+            timestamp: lastTimestamp,
+            feedback: entries.map(([index, answer]) => ({ 
+              eventIndex: index, 
+              answer: answer 
+            }))
+          })
+        });
+
+        alert(res.ok ? "✅ Feedback submitted successfully!" : "❌ Submission failed.");
+      }
+
+      async function loadEvents(file) {
+        const resultBox = document.getElementById("result");
+        const p = resultBox.querySelector("p");
+        resultBox.classList.remove("hidden");
+
+        try {
+          const fileParam = encodeURIComponent(file);
+          const eventsRes = await fetch(`/api/events?file=${fileParam}`);
+          if (!eventsRes.ok) throw new Error("❌ Event file not found or malformed");
+          const data = await eventsRes.json();
+          if (!Array.isArray(data)) throw new Error("❌ Unexpected data format");
+          
+          if (data.length === 0) {
+            p.innerHTML = `<div class="text-gray-500 italic"> Sorry, we do not find any progress for this pledge.</div>`;
+            return;
+          }
+          // p.innerHTML = `<strong>We have found ${data.length} events for this pledge.</strong><br><br>` +
+          //   data.map((e, index) => `
+          p.innerHTML =
+            data.map((e, index) => `
+              <div class="mb-6 border-b pb-4">
+                🗓️ <b>${e.date}</b>: ${e.event}<br>
+                🔗 <a href="${e.url}" target="_blank" class="text-purple-400 underline">Source</a>
+                
+                <div class="mt-3">
+                  <label class="block text-sm font-medium mb-2">How relevant is this event?</label>
+                  <div class="flex flex-wrap gap-2">
+                    <button onclick="setFeedback(${index}, 'not_relevant')" 
+                            class="px-3 py-1.5 bg-gray-100 hover:bg-gray-200 border border-gray-300 rounded-lg text-gray-700">
+                      Not relevant
+                    </button>
+                    <button onclick="setFeedback(${index}, 'relevant_seen')" 
+                            class="px-3 py-1.5 bg-blue-100 hover:bg-blue-200 border border-blue-300 rounded-lg text-blue-700">
+                      Relevant but seen
+                    </button>
+                    <button onclick="setFeedback(${index}, 'relevant_updated')" 
+                            class="px-3 py-1.5 bg-green-100 hover:bg-green-200 border border-green-300 rounded-lg text-green-700">
+                      Relevant & up-to-date
+                    </button>
+                  </div>
+                  <div id="msg-${index}" class="text-sm mt-1"></div>
+                </div>
+              </div>
+            `).join('') +
+            `<button onclick="submitAllFeedback()" class="mt-6 px-4 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700">
+              📤 Submit All Feedback
+            </button>
+            <button onclick="window.location.href='/download?file=${fileParam}'" class="mt-4 ml-4 px-4 py-2 bg-purple-600 text-white rounded-lg hover:bg-purple-700">
+              📅 Download Excel
+            </button>`;
+        } catch (err) {
+          p.textContent = `❌ Failed to load timeline: ${err.message}`;
+        }
+      }
+
+      let suggestTimer = null;
+        document.getElementById("claim").addEventListener("input", () => {
+          suggestedPledge = null;
+          clearTimeout(suggestTimer);
+          suggestTimer = setTimeout(fetchSuggestions, 300); // 300ms delay to avoid flooding
+        });
+
+        async function fetchSuggestions() {
+          const claimText = document.getElementById("claim").value.trim();
+          const suggestionBox = document.getElementById("similar-suggestions");
+
+          if (!claimText) {
+            suggestionBox.classList.add("hidden");
+            return;
+          }
+
+          const res = await fetch("/api/similar-pledges", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ claim: claimText })
+          });
+          const data = await res.json();
+          const suggestions = data.suggestions || [];
+
+          if (suggestions.length === 0) {
+            suggestionBox.classList.add("hidden");
+          } else {
+            const author = "Labour";
+            const date = "2024-07-04";
+            suggestionBox.innerHTML =
+              "<div class='font-semibold mb-1'>💡 Are you fact-checking this pledge? </div>" +
+              "<ul class='list-disc ml-6 mt-1'>" +
+              suggestions.map(s => `
+                <li class="mb-2">
+                  ${author}: ${s.text} (${date})
+                  <button
+                    onclick="useSuggestedPledge('${s.text.replace(/'/g, "\\'")}', ${s.index})"
+                    class="ml-2 px-2 py-1 text-xs bg-purple-600 text-white rounded hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500">
+                    Fact-check this pledge
+                  </button>
+                </li>
+              `).join("") +
+              "</ul>";
+            suggestionBox.classList.remove("hidden");
+          }
+        }
+
+        
+      checkBtn.addEventListener("click", async () => {
+        const claim = document.getElementById("claim").value.trim();
+        const pledgeDate = document.getElementById("pledge-date").value.trim();
+        const pledgeAuthor = document.getElementById("pledge-author").value.trim();
+        const statusElement = document.getElementById("status");
+        const resultBox = document.getElementById("result");
+        // resultBox.classList.remove("hidden");
+        const p = resultBox.querySelector("p");
+
+
+
+        let valid = true;
+        if (!claim) {
+          alert("Please enter the pledge text.");
+          valid = false;
+        }
+        if (!pledgeDate) {
+          document.getElementById("date-warning").classList.remove("hidden");
+          valid = false;
+        }
+        if (!pledgeAuthor) {
+          document.getElementById("author-warning").classList.remove("hidden");
+          valid = false;
+        }
+
+        if (!valid) return;
+
+        checkBtn.disabled = true;
+        checkBtn.classList.add("opacity-50", "cursor-not-allowed");
+
+        // document.getElementById("status").classList.remove("hidden");
+        // statusElement.innerHTML = renderStatus({}); 
+        // document.getElementById("result").classList.remove("hidden");
+        // document.getElementById("progress").classList.remove("hidden");
+
+        document.getElementById("status").innerHTML = "";
+        document.getElementById("result").classList.add("hidden");
+        document.getElementById("progress").classList.add("hidden");
+        document.getElementById("result").querySelector("p").innerHTML = "";
+        if (window.pollIntervalId) {
+          clearInterval(window.pollIntervalId);
+          window.pollIntervalId = null;
+        }
+        Object.keys(feedbackData).forEach(key => delete feedbackData[key]);
+        lastUsedFile = null;
+        lastUserId = null;
+        lastTimestamp = null;
+
+        // 🔄 可以预先显示提示
+        document.getElementById("result").querySelector("p").textContent = "⏳ Please wait, checking...";
+        document.getElementById("progress").classList.remove("hidden");
+        document.getElementById("result").classList.remove("hidden");
+
+
+      try {
+          const timeRange = document.getElementById("time-range").value;
+          // const pledgeDate = document.getElementById("pledge-date").value;
+          // const pledgeAuthor = document.getElementById("pledge-author").value;
+          if (currentAbortController) currentAbortController.abort();
+          currentAbortController = new AbortController();
+          const signal = currentAbortController.signal;
+          let valid = true;
+
+          stepList = (suggestedPledge !== null) ? stepListSuggestion : stepListStandard;
+
+          if (!pledgeDate) {
+            document.getElementById("date-warning").classList.remove("hidden");
+            valid = false;
+          }
+          if (!pledgeAuthor) {
+            document.getElementById("author-warning").classList.remove("hidden");
+            valid = false;
+          }
+          if (!valid) return; 
+
+          const userId = Math.random().toString(36).substring(2, 10);  
+          const now = new Date();
+          const timestamp = now.toISOString().replace(/[:.]/g, "-").slice(0, 19); 
+          statusElement.textContent = ""; 
+          // pollStatus(userId, timestamp, p); 
+          pollStatus(userId, timestamp, document.getElementById("status"));
+
+
+          const runRes = await fetch("/api/run-model", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              claim,
+              time_range: timeRange,
+              pledge_date: pledgeDate,
+              pledge_author: pledgeAuthor,
+              user_id: userId,
+              timestamp: timestamp,
+              signal: signal,
+              suggestion_meta: suggestedPledge
+            })
+          });
+
+          const runData = await runRes.json();
+
+          lastUsedFile = runData.file;
+          lastUserId = runData.user_id;    
+          lastTimestamp = runData.timestamp;
+        } catch (err) {
+          if (err.name === "AbortError") {
+            console.log("Previous request aborted.");
+            checkBtn.disabled = false;
+            checkBtn.classList.remove("opacity-50", "cursor-not-allowed");
+            return;
+          }
+          p.textContent = `❌ Failed to load timeline: ${err.message}`;
+        }
+
+      });
+
+    
+      async function useSuggestedPledge(text, index) {
+          document.getElementById("claim").value = text;
+          document.getElementById("pledge-author").value = "Labour";
+          document.getElementById("pledge-date").value = "2024-07-04";
+          suggestedPledge = { text, index };
+          alert("✅ This pledge has been filled in. You can now click 'Let's track!'");
+          await fetch("/api/log-similar-selection", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+              selected_text: text,
+              index: index
+            })
+          });
+        }
+
+    </script>
+  </body>
+</html>