structure.js

// This is how a question will be formatted in JSON. These will be in a dictionary 
// formatted as {question ID:question data}.
var sample_question = {
    "question": "What is the best explanation for this law?",
    "answers": ["Option A", "Option B", "Option C", "Option D"],
    "correct_answer_index": 0, 
    "top10_e5": ["Passage 1 from E5", "Passage 2 from E5", "Passage 3..."],
    "generation_e5": "This generation is based on the E5 passages.",
    "top10_colbert": ["Passage 1 from ColBERT", "Passage 2 from Colbert", "..."],
    "generation_colbert": "This generation is based on the ColBERT passages.",
    "top10_contains_gold_passage": false, // in either case
    "gold_passage": "This is the gold passage.",
    "gold_passage_generation": "This generation answers the question with the gold passage."
}

// Each user response will take this form. A list of these objects will be uploaded.
var user_response = {
    "user_id": "[email protected]", // track who the respondent is
    "question_id": "ID ###", // track which question they are responding to
    "user_answer": 0, // A, B, C, or D
    "e5_scores": [[1, 2.5, 1.5, 3.5], [0, 1, 4, 1.5]], // [is a law, relevance, quality, helpfulness]
    "colbert_scores": [[1, 3, 3, 3], [1, 5, 4, 5]], // same format as above
}

// One JSON file for each user -> check if the file exists, download if it does; otherwise, create.
var user_data = {
    "user_id": "[email protected]",
    "order": ["question ID 1", "question ID 2", "...", "question ID 15"], // randomized and different for each user
    "modes": [["e5", "colbert"], ["colbert", "e5"], ["colbert", "e5"], "..."], // randomized and different for each user
    "current": 3, // index in "order" that the user has not yet completed (starts at 0) -> this user would log in at 3
    "responses": [user_response, user_response] // list of user responses in the same order
}
// When the file exists, download it and make a LOCAL copy. Modify the local copy (e.g. increasing order 
// and adding responses), and after each question, rewrite the original JSON object with the new data.
// To avoid losing data in the case of a bad overwrite, each individual user_response should also be added to
// the commit scheduler.