test_exam

#!/usr/bin/env node

require('browser-env')();
global.module = undefined;
window.alert = function() {}

const luxon = require('luxon');
global.DateTime = luxon.DateTime;
const SCORM_API = require(`./api.js`);

const fs = require('fs');
const path = require('path');
const {Console} = require('console');
const stream = require('stream');

var captured_log = '';
const real_console = new Console(process.stdout,process.stderr);
const s = new stream.Writable({write: function(chunk,enc,next) {
    captured_log += chunk;
    next();
}});
const fake_console = new Console(s,process.stderr);
Object.keys(console).forEach(function(n) {
    if(typeof console[n] == 'function' && typeof fake_console[n] == 'function') {
        console[n] = fake_console[n];
    }
});

class PartCreditError extends Error {
}
PartCreditError.prototype.originalMessages = ['headless.not full credit'];

function tick(correct) {
    return correct ? '✔' : '✖';
}

function die(e) {
    process.stdout.write(JSON.stringify({success:false, message: e.message+'\n'+e.stack, originalMessages: e.originalMessages || []}));
    end();
}
function fail(message) {
    process.stdout.write(JSON.stringify({success:false, message: message}));
    end();
}
function succeed(data) {
    const obj = Object.assign({success:true}, data || {});
    process.stdout.write(JSON.stringify(obj));
    end();
}
function end(code) {
    if(captured_log) {
        process.stderr.write(`------------\nConsole log:\n${captured_log}\n----------`);
    }
    process.exit(code);
}

function parse_numbasobject(data) {
    var re_header = /^\/\/ Numbas version: [^\n]*\n/;
    if(!data.match(re_header)) {
        throw(new Error("Not a valid .exam file"));
    }
    data = data.replace(re_header,'');
    return JSON.parse(data);
}

function test_question(q) {
    var promise = q.signals.on('ready').then(function() {
        q.parts.forEach(p=>{
            test_part(p);
        });
    }).catch(e=>{
        if(q.scope) {
            console.log("Question variables:");
            for(let name of Object.keys(q.scope.variables)) {
                const value = Numbas.jme.display.treeToJME({tok:q.scope.getVariable(name)});
                console.log(`  ${name}: ${value}`);
            }
        }
        die(e);
    });
    return promise;
}

function scope_for_part(p) {
    return p.settings.variableReplacementStrategy=='alwaysreplace' ? p.errorCarriedForwardScope() : p.getScope();
}

function answer_for_part(p) {
    if(p.type=='gapfill') {
        return p.gaps.map(answer_for_part);
    }
    var answer = p.getCorrectAnswer(scope_for_part(p));
    return answer;
}

function test_part(p) {
    if(p.type=='information') {
        return;
    }
    if(p.type=='gapfill' && p.gaps.length==0) {
        return;
    }
    var answer = answer_for_part(p);
    p.storeAnswer(answer);
    p.setStudentAnswer();
    const scope = scope_for_part(p);
    p.submit();
    if(Numbas.math.precround(p.credit,5)!=1) {
        const niceName = Numbas.util.capitalise(Numbas.util.nicePartName(p.path));
        real_console.error(`✖ ${niceName}: ${p.type}`);
        real_console.error(`Answer: ${JSON.stringify(answer)}`);
        real_console.error(`Credit: ${p.credit}`);
        if(p.settings.variableReplacementStrategy=='alwaysreplace') {
            real_console.error('Variables:');
            for(let name of Object.keys(scope.variables)) {
                const value = Numbas.jme.display.treeToJME({tok:scope.getVariable(name)});
                real_console.error(`  ${name}: ${value}`);
            }
        }
        p.markingFeedback.filter(m=>m.message).forEach(m=>real_console.error(`* ${m.message}`));
        throw(new PartCreditError(`${niceName} got ${p.credit} credit`));
    }
}

function scorm_api(scorm_cmi) {
    const basic_cmi = {
        'cmi.suspend_data': '',
        'cmi.objectives._count': 0,
        'cmi.interactions._count': 0,
        'cmi.learner_name': '',
        'cmi.learner_id': '',
        'cmi.location': '',
        'cmi.score.raw': 0,
        'cmi.score.scaled': 0,
        'cmi.score.min': 0,
        'cmi.score.max': 0,
        'cmi.total_time': 0,
        'cmi.success_status': '',
        'cmi.completion_status': 'not attempted',
        'cmi.mode': 'normal',
        'cmi.entry': 'ab-initio',
        'numbas.user_role': 'student',
    };
    const cmi = Object.assign({},basic_cmi,scorm_cmi);
    const fixed_cmi = {};
    for(var x in cmi) {
        if(typeof cmi[x]!='object') {
            fixed_cmi[x] = {value: cmi[x], time: 0};
        } else {
            fixed_cmi[x] = cmi[x];
        }
    }
    const api = new SCORM_API({
        standalone: true,
        scorm_cmi: fixed_cmi
    });
    return api;
}

function load_exam(exam_data, scorm_api) {
    window.top.opener = {API_1484_11: scorm_api};
    window.pipwerks.SCORM.API.handle = null;
    window.pipwerks.SCORM.API.isFound = null;
    window.pipwerks.SCORM.connection.isActive = false;
    if(Numbas.schedule.unhalt) {
        Numbas.schedule.unhalt();
    } else {
        Numbas.schedule.halted = false;
        Numbas.schedule.signalboxes.forEach(function(sb) {
            sb.error = null;
        });
    }
    var seed = Math.seedrandom(new Date().getTime());
    var job = Numbas.schedule.add;
    var store = Numbas.store = new Numbas.storage.scorm.SCORMStorage();
    var exam = Numbas.exam = Numbas.createExamFromJSON(exam_data,store,true);
    exam.seed = Numbas.util.hashCode(seed);
    var entry = store.getEntry();
    if(store.getMode() == 'review') {
        entry = 'review';
    }
    exam.entry = entry;

    switch(entry) {
        case 'ab-initio':
            job(exam.init,exam);
            exam.signals.on('ready', function() {
                Numbas.signals.trigger('exam ready');
            })
            break;
        case 'resume':
        case 'review':
            job(exam.load,exam);
            exam.signals.on('ready', function() {
                Numbas.signals.trigger('exam ready');
                job(function() {
                    if(entry == 'review') {
                        job(exam.end,exam,false);
                    }
                });
            });
            break;
    }
    job(function() {
        Numbas.signals.trigger('Numbas initialised');
    });
    const p = new Promise(function(resolve,reject) {
        exam.signals.on('ready',function() {
            resolve(exam);
        }).catch(reject);
    });
    return p;
}

function test_ab_initio(exam_data) {
    const api = scorm_api({});
    load_exam(exam_data, api).then(exam => {
        var promises = [];
        exam.questionList.forEach(q => {
            var promise = test_question(q);
            promises.push(promise);
        });
        Promise.all(promises).then(function() {
            test_review(exam_data,api.data);
        }).catch(die);
    }).catch(die);
}

function test_review(exam_data,scorm_data) {
    const data = Object.assign({},scorm_data,{'cmi.mode': 'review', 'cmi.entry': 'review'});
    const api = scorm_api(data);
    load_exam(exam_data,api).then(exam => {
        const saved_score = scorm_data['cmi.score.raw'];
        if(exam.score != saved_score) {
            fail(`The resumed exam's score was ${exam.score} instead of ${saved_score}`);
        }
        succeed();
    });
}

function run(exam_dir,fn,options) {
    require(exam_dir+'/scripts.js');

    try {
        Numbas.queueScript('headless-test',['start-exam'],function() {
            try {
                for(var x in Numbas.extensions) {
                    Numbas.activateExtension(x);
                }
                Numbas.display = null;
                fs.readFile(exam_dir+'/source.exam','utf8',(err,data) => {
                    const parsed_data = parse_numbasobject(data);
                    try {
                        fn(parsed_data,options);
                    } catch(e) {
                        die(e);
                    }
                });
            } catch(e) {
                die(e);
            }
        });
    } catch(e) {
        die(e);
    }
}

function remark_sessions(exam_data,sessions,options) {
    const results = [];

    const reevaluate_variables = {};
    if(options.reevaluate_variables) {
        options.reevaluate_variables.forEach(d=>{
            const [n,name] = d.split(':');
            reevaluate_variables[n] = reevaluate_variables[n] || [];
            reevaluate_variables[n].push(name);
        });
    }

    function remark_session(resolve_all,i) {
        const session_data = sessions[i];
        const promise = new Promise((resolve,reject) => {
            const api = scorm_api(session_data.cmi);
            api.allow_set = true;
            load_exam(exam_data, api).then(exam => {
                exam.questionList.forEach(function(q) {
                    let reevaluated = false;
                    if(reevaluate_variables[q.number]) {
                        reevaluated = true;
                        const todo = {};
                        reevaluate_variables[q.number].forEach(name => {
                            q.scope.deleteVariable(name);
                            todo[name] = q.variablesTodo[name];
                        });
                        Numbas.jme.variables.makeVariables(todo,q.scope);
                    }
                    q.allParts().forEach(function(p) {
                        p.store.partAnswered(p);
                        p.revealed = false;
                        if(options.unsubmitted) {
                            p.stagedAnswer = p.resume_stagedAnswer || p.stagedAnswer;
                        }
                    });
                    if(options.unsubmitted || reevaluated) {
                        q.parts.forEach(function(p) {
                            p.submit();
                        });
                    }
                    q.store.saveQuestion(q);
                });
                exam.store.saveExam(exam);

                const changed_keys = {};
                for(var x in api.data) {
                    if(!(x.match(/\._count$/)) && (session_data.cmi[x]===undefined || api.data[x]!=session_data.cmi[x].value)) {
                        changed_keys[x] = api.data[x];
                    }
                }
                resolve({success: true, changed_keys: changed_keys});
            }).catch(err=>{
                resolve({success: false, error: err});
            })
        });
        promise.then(result => {
            result.attempt_pk = session_data.attempt_pk;
            results.push(result);
            if(i+1<sessions.length) {
                remark_session(resolve_all,i+1);
            } else {
                resolve_all(results);
            }
        }).catch(die);
    }
    
    return new Promise((resolve,reject) => {
        if(sessions.length){ 
            remark_session(resolve,0);
        }
    });
}

const commands = {
    test: function(data) {
        test_ab_initio(data);
    },

    remark: function(data,options) {
        let stdin = '';
        process.stdin.setEncoding('utf-8');
        process.stdin.on('readable', function() {
            var chunk;
            while (chunk = process.stdin.read()) {
                stdin += chunk;
            }
        });
        process.stdin.on('end', function() {
            try {
                const sessions = JSON.parse(stdin);
                if(!sessions.length) {
                    succeed({results: []});
                }
                remark_sessions(data,sessions,options).then(results => {
                    succeed({results: results});
                });
            } catch(e) {
                die(e);
            }
        });
    }
}

const exam_dir = process.argv[2];
let command;
const switches = {
    'unsubmitted': false
}
const multi = {
    'reevaluate_variables': true
}
const options = {};
for(let i=3;i<process.argv.length;i++) {
    const arg = process.argv[i];
    if(arg.match(/^--.*/)) {
        const option = arg.slice(2);
        if(option in switches) {
            options[option] = true;
        } else if(option in multi) {
            let j = i+1;
            while(j<process.argv.length && !process.argv[j].match(/^--/)) {
                j += 1;
            }
            options[option] = process.argv.slice(i+1,j);
            i = j-1;
        } else {
            options[option] = process.argv[i+1];
        }
        i += 1;
    } else if(!command) {
        command = arg;
    } else {
        throw(new Error(`Unexpected positional argument ${arg}`));
    }
}

if(!commands[command]) {
    const command_names = Object.keys(commands).join(', ');
    fail(`The command ${command} is not valid. Valid commands are: ${command_names}`);
}

run(exam_dir, commands[command], options);