Skip to content
Snippets Groups Projects
Commit 18cbd68c authored by Iain Bryson's avatar Iain Bryson
Browse files

Load config on every request.

Fix race condition error in archive finalization.
Make logs more consistent.
parent 9aa00cb6
No related branches found
No related tags found
No related merge requests found
#
# Logs
```
docker exec -it get-pdf cat /root/getpdf_server_out.log
```
```
docker exec -it get-pdf cat /root/getpdf_server_err.log
```
# Based on ...
......
......@@ -3,18 +3,28 @@ import logger from './logger';
import BuildPDFJob from './jobs/build_pdf_job';
import ZipJob from './jobs/zip_job';
import async from "async";
import config from "./config.js";
import getConfig from "./config.js";
export var agenda;
export var tasks = {getpdf: {}, zip:{}};
let tasks = { getpdf: { }, zip: { } };
export function initAgenda(dbs) {
const a = new Agenda({mongo: dbs.getpdf_agenda_db});
let sharelatex_db;
const sldb = dbs.sharelatex_db;
export function getTasks(task, course_id) {
if (tasks[task][course_id]) {
return tasks[task][course_id];
} else {
const config = getConfig();
if (Object.keys(config).includes(course_id)) {
makeJobForCourse(course_id, config, sharelatex_db, agenda);
return tasks[task][course_id];
}
}
return undefined;
}
for (let course of Object.keys(config.sharelatex_config)) {
function makeJobForCourse(course, config, sldb, a) {
logger.debug(`Creating job definition for getpdf for ${course}`);
const opts = {priority: 'high', concurrency: 1};
......@@ -23,10 +33,13 @@ export function initAgenda(dbs) {
const getpdf_task_name = `GetPDF for ${course}`;
tasks.getpdf[course] = {name: getpdf_task_name};
const after_zip = (zip_details, project, job, done) => {
logger.info("Zip completed; About to start conversion");
const buildPDFJob = new BuildPDFJob(course, project, job);
buildPDFJob.jobFunc(job, done);
}
const zipAndGetPDFJob = new ZipJob(sldb, project_id, course, (zip_details, project, job, done) => { setTimeout(()=>after_zip(zip_details, project, job, done), 20000) });
};
const zipAndGetPDFJob = new ZipJob(sldb, project_id, course, (zip_details, project, job, done) => {
setTimeout(() => after_zip(zip_details, project, job, done), 500)
});
a.define(getpdf_task_name, opts, zipAndGetPDFJob.jobFunc.bind(zipAndGetPDFJob));
const zip_task_name = `Zip for ${course}`;
......@@ -35,6 +48,17 @@ export function initAgenda(dbs) {
a.define(zip_task_name, opts, zipOnlyJob.jobFunc.bind(zipOnlyJob));
}
export function initAgenda(dbs) {
const a = new Agenda({mongo: dbs.getpdf_agenda_db});
const sldb = dbs.sharelatex_db;
sharelatex_db = sldb;
const config = getConfig();
for (let course of Object.keys(config.sharelatex_config)) {
makeJobForCourse(course, config, sldb, a);
}
function graceful() {
a.stop(function () {
process.exit(0);
......@@ -47,15 +71,23 @@ export function initAgenda(dbs) {
a.on('start', function (job) {
job.attrs.status = 'running';
job.attrs.data.status = 'running';
job.save()
job.save();
logger.info('Job %s starting', job.attrs.name);
});
a.on('complete', function (job) {
if (job.attrs.status !== 'failed') {
if (job.attrs.failReason) {
job.attrs.data.logs.push(job.attrs.failReason);
job.attrs.status = 'failed';
job.attrs.data.status = 'failed';
} else {
job.attrs.status = 'completed';
job.attrs.data.status = 'completed';
job.save()
logger.info('Job %s finished', job.attrs.name);
}
job.save();
}
logger.info('Job %s finished: %s', job.attrs.name, job.attrs.status);
});
a.on('fail', function (job) {
......
import {Router} from 'express';
import {ObjectId} from 'mongojs';
import logger from '../logger';
import { agenda, tasks, purgeOldJobs } from '../agenda';
import { agenda, getTasks, purgeOldJobs } from '../agenda';
class GetPDFRouter {
constructor({config, dbs, router}) {
......@@ -11,7 +11,14 @@ class GetPDFRouter {
}
listJobs(req, res) {
const job_name = tasks.getpdf[req.params.course_id].name;
const course_id = req.params.course_id;
const job_def = getTasks('getpdf', course_id);
if (!job_def) {
res.status(404)
.json({});
}
const job_name = job_def.name;
agenda.jobs({ name: job_name }, function(err, jobs) {
logger.info(`Got ${jobs.length} jobs for job ${job_name}`);
......@@ -21,15 +28,25 @@ class GetPDFRouter {
}
newJob(req, res) {
console.dir(req.params);
var job = agenda.create(tasks.getpdf[req.params.course_id].name, {'status':'running'});
const course_id = req.params.course_id;
const job_def = getTasks('getpdf', course_id);
if (!job_def) {
res.status(404)
.json({});
return;
}
let job = agenda.create(job_def.name, {'status':'running'});
job.unique({'data.status':'running'});
job.save((err, j) => {
if (err) {
logger.error(err);
logger.error(`Failed to create job ${err}`);
res.status(500).json({err});
return;
}
console.dir(j.attrs);
console.dir(job.attrs);
logger.info(job.attrs);
res.status(201)
.json(job);
});
......@@ -51,7 +68,13 @@ function getPDFRouter({config, dbs}) {
router.route('/summary')
.get(function (req, res) {
const job_name = tasks.getpdf[req.params.course_id].name;
const course_id = req.params.course_id;
const job_def = getTasks('getpdf', course_id);
if (!job_def) {
res.status(404)
.json({});
}
const job_name = job_def.name;
agenda.jobs({ name: job_name }, function(err, jobs) {
logger.info(`Got ${jobs.length} jobs for job ${job_name}`);
......@@ -63,7 +86,13 @@ function getPDFRouter({config, dbs}) {
router.route('/purge')
.get(function (req, res) {
const job_name = tasks.getpdf[req.params.course_id].name;
const course_id = req.params.course_id;
const job_def = getTasks('getpdf', course_id);
if (!job_def) {
res.status(404)
.json({});
}
const job_name = job_def.name;
purgeOldJobs(job_name, () => {
res.status(200)
......
......@@ -24,15 +24,16 @@ const env = process.env.NODE_ENV || 'development';
const vol_root = process.env.DTU_ENOTE_VOL || "/data";
const config_root = path.join(vol_root, "config");
const sharelatex_config_file = fs.readFileSync(path.join(config_root, "sharelatex.yaml"), {encoding: 'utf8'});
console.log(sharelatex_config_file)
const sharelatex_config = yaml.safeLoad(sharelatex_config_file);
console.dir(sharelatex_config);
console.log(`Configuring environment for ${process.env.NODE_ENV} ${env}`);
export default {
export default () => {
const sharelatex_config_file = fs.readFileSync(path.join(config_root, "sharelatex.yaml"), {encoding: 'utf8'});
//console.log(sharelatex_config_file);
const sharelatex_config = yaml.safeLoad(sharelatex_config_file);
//console.dir(sharelatex_config);
return {
env: env,
db_urls: urls_by_env[env],
sharelatex_data_path: sharelatex_data_path_by_env[env],
......@@ -40,3 +41,4 @@ export default {
config_root: config_root,
sharelatex_config: sharelatex_config
}
}
import mongojs from "mongojs";
import { MongoClient } from 'mongodb'; // Agenda needs MongoDB-Native, not mongojs ¯\_(ツ)_/¯
import config from "./config.js";
import getConfig from "./config.js";
import async from 'async';
import { URL } from 'url';
import logger from "./logger";
export var dbs = new Map(Object.keys(config.db_urls).map((db) => [db, null]))
export var dbs = new Map(Object.keys(getConfig().db_urls).map((db) => [db, null]))
function connectMongoClient(url, setDB) {
......@@ -61,7 +61,9 @@ function connectMongoJS(url, collections, setDB) {
export function initializeDBs(callback) {
var jobs = [];
let jobs = [];
const config = getConfig();
if (!dbs.getpdf_agenda_db) {
jobs.push(connectMongoClient(config.db_urls.getpdf_agenda_db, (db) => { dbs.getpdf_agenda_db = db } ));
......@@ -75,7 +77,7 @@ export function initializeDBs(callback) {
jobs.push(connectMongoJS(config.db_urls.getpdf_db, ["projects", "docs", "docOps"], (db) => { dbs.getpdf_db = db } ));
}
if (jobs.length == 0) {
if (jobs.length === 0) {
callback(dbs);
return;
}
......@@ -83,6 +85,5 @@ export function initializeDBs(callback) {
async.parallel(jobs, () => {
logger.info(`Finished attaching to all databases`);
callback(dbs);
return;
})
}
import { zipFilePath } from "../lib/zip_project";
const { spawn } = require('child_process');
import logger from '../logger';
class BuildPDFJob {
constructor(course_id, project) {
constructor(course_id, project, job) {
this.course_id = course_id;
this.project = project;
this.job = job;
}
jobFunc(job, done) {
logger.info("Starting BuildPDF Job");
const zipPath = zipFilePath(this.project._id, this.project.name);
const child = spawn(`./build-pdf.sh ${this.course_id} ${zipPath} ${job.attrs._id.toString()} `, {
const cmd = `./build-pdf.sh ${this.course_id} ${zipPath} ${job.attrs._id.toString()} `;
logger.info(cmd);
const child = spawn(cmd, {
shell: true,
cwd: '.'
});
child.stdout.on('data', (data) => {
console.dir(data.toString());
logger.info(data.toString());
job.attrs.data.logs.push(data.toString());
job.save();
});
child.stderr.on('data', (data) => {
console.dir(data.toString());
logger.info(data.toString());
job.attrs.data.logs.push(data.toString());
job.save();
});
......@@ -30,7 +38,7 @@ class BuildPDFJob {
child.on('exit', function (code, signal) {
const msg = 'child process exited with ' +
`code ${code} and signal ${signal}`;
console.log(msg);
logger.info(msg);
if (code !== 0) {
job.fail(msg);
}
......
import { zipProject } from "../lib/zip_project";
import { ObjectId } from "mongojs";
import logger from '../logger';
function logToLoggerAndJob(msg, job) {
logger.info(msg);
job.attrs.data.logs.push(msg);
job.save();
}
export default class ZipJob {
constructor(sldb, project_id, course, after_zip) {
this.sldb = sldb;
......@@ -11,29 +18,39 @@ export default class ZipJob {
}
jobFunc(job, done) {
const course = this.course;
const project_id = this.project_id;
logger.info(`Starting zip job for course ${course} ${project_id}`);
job.attrs.data.logs = ["Loading Project"];
job.save();
this.sldb.projects.findOne({_id: ObjectId(this.project_id)}, (err, project) => {
this.sldb.projects.findOne({_id: ObjectId(project_id)}, (err, project) => {
if (err || !project) {
job.fail(`Cannot find sharelatex project id ${this.project_id} for course ${this.course}`);
const msg = `Cannot find sharelatex project id ${project_id} for course ${course} ${err}`;
logger.error(msg);
job.fail(msg);
done();
return;
}
job.attrs.data.logs.push("Loaded Project");
job.save();
logToLoggerAndJob("Loaded Project", job);
zipProject(this.sldb, project, (err, details) => {
if (err) {
job.fail(`Zip Failed for course ${this.course}`);
job.fail(`Zip failed for course ${course}`);
return;
}
logToLoggerAndJob(`Completed zip job for course ${course} ${project_id}`, job);
if (this.after_zip) {
logger.info(`Running after zip hook for course ${course} ${project_id}`);
this.after_zip(details, project, job, done);
} else {
done();
}
}, (progress_msg) => {
job.attrs.data.logs.push(progress_msg);
job.save();
logToLoggerAndJob(progress_msg, job);
})
});
}}
......@@ -6,7 +6,9 @@ import path from 'path';
import fs from 'fs';
import config from '../config.js';
const SL_DATA_PATH = config.sharelatex_data_path;
function sharelatexDataPath() {
return config().sharelatex_data_path;
}
function _getAllFoldersFromProject(project, callback) {
var folders = {}
......@@ -18,20 +20,73 @@ function _getAllFoldersFromProject(project, callback) {
}
}
}
processFolder("/", project.rootFolder[0])
processFolder("/", project.rootFolder[0]);
callback(null, folders)
}
export function zipFilePath(project_id, project_name) {
return path.join(SL_DATA_PATH, "backups", `${project_id}_${project_name}.zip`);
return path.join(sharelatexDataPath(), "backups", `${project_id}_${project_name}.zip`);
}
export function zipProject(db, project, zipDone, progress = () => {} ) {
const archive = archiver("zip");
const zipPath = zipFilePath(project._id, project.name);
archive.on("error", (err) => {
console.log("Can't create zip");
console.dir(err);
logger.error("Can't create zip");
logger.info(err);
zipDone(err, null);
});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on('warning', function(err) {
if (err.code === 'ENOENT') {
logger.warn(`archiver warning ${err}`);
} else {
zipDone(err, null);
}
});
progress(`ZIP: unlinking existing file ${zipPath}`);
// First, unlink the existing archive.
try {
if (fs.statSync(zipPath).isFile()) {
try {
fs.unlinkSync(zipPath);
progress(`ZIP: unlinked existing file ${zipPath}`);
} catch (ue) {
logger.error(`failed to unlink ${zipPath}: ${ue}`);
zipDone(ue, null);
return;
}
}
} catch (e) {
if (e.code !== 'ENOENT') {
logger.error(`error stat'ing ${zipPath}: ${e}`);
zipDone(e, null);
return;
}
}
const output = fs.createWriteStream(zipPath);
output.on("close", () => {
progress(`ZIP: Archive has been closed: ${zipPath}`);
progress("ZIP: " + archive.pointer() + ' total bytes');
zipDone(null, {
project_id: project._id.toString(),
path: zipPath
})
});
output.on("end", () => {
progress(`ZIP: Archive has been drained: ${zipPath}`);
});
archive.pipe(output);
const query = {project_id: ObjectId(project._id)};
progress(`ZIP: Getting project tree for ${project.name}`);
db.docs.find(query, (err, docs) => {
......@@ -65,13 +120,13 @@ export function zipProject(db, project, zipDone, progress = () => {} ) {
rev: content.rev
});
} else {
console.log("STRANGE! no content for ")
console.log(`${docPath}: ${content}`);
console.dir(doc._id.toString());
logger.info("STRANGE! no content for ");
logger.info(`${docPath}: ${content}`);
logger.info(doc._id.toString());
}
}
const filesRootPath = path.join(SL_DATA_PATH, "data", "user_files");
const filesRootPath = path.join(sharelatexDataPath(), "data", "user_files");
for (let fileRef of (folder.fileRefs || [])) {
const filePath = path.join(folderPath, fileRef.name);
......@@ -83,14 +138,14 @@ export function zipProject(db, project, zipDone, progress = () => {} ) {
content: content,
});
} else {
console.log("STRANGE! can't find file for fileRef")
console.log(`${filePath}: ${content}`);
console.dir(fileRef._id.toString());
logger.debug("STRANGE! can't find file for fileRef");
logger.debug(`${filePath}: ${content}`);
logger.info(fileRef._id.toString());
}
}
}
var jobs = [];
let jobs = [];
for (let [path, doc] of docArchiveInfo) {
const relPath = path[0] === "/" ? path.slice(1) : path;
jobs.push( (callback) => {
......@@ -113,21 +168,12 @@ export function zipProject(db, project, zipDone, progress = () => {} ) {
})
}
const zipPath = zipFilePath(project._id, project.name);
var output = fs.createWriteStream(zipPath);
archive.pipe(output);
const msg = `ZIP: Archiving ${jobs.length} docs to archive`;
logger.info(msg);
progress(msg);
async.series(jobs, () => {
const msg = `ZIP: Finalizing to archive`;
logger.debug(msg);
progress(msg);
archive.finalize();
zipDone(null, {
project_id: project._id.toString(),
path: zipPath
})
})
});
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment