Add job framework (‘agenda’)

Add runjs script to clean db.
Add getpdf jobs route.

getpdf-server/gruntfile.js

+module.exports = function(grunt) {
+
+    grunt.initConfig({
+        pkg: grunt.file.readJSON('package.json'),
+        jshint: {
+            files: ['Gruntfile.js', 'src/**/*.js', 'test/**/*.js'],
+            options: {
+                // options here to override JSHint defaults
+                globals: {
+                    jQuery: true,
+                    console: true,
+                    module: true,
+                    document: true
+                }
+            }
+        },
+        watch: {
+            files: ['<%= jshint.files %>'],
+            tasks: ['jshint']
+        }
+    });
+
+    grunt.loadNpmTasks('grunt-contrib-jshint');
+    grunt.loadNpmTasks('grunt-contrib-watch');
+
+    grunt.registerTask('test', ['jshint']);
+
+    grunt.registerTask('default', ['jshint']);
+};
\ No newline at end of file

getpdf-server/package.json

@@ -24,16 +24,28 @@
       "no-unused-vars": 1
     }
   },
+  "babel": {
+    "presets": [
+      "es2015"
+    ]
+  },
+  "runjs": {
+    "requires": [
+      "./node_modules/babel-register"
+    ]
+  },
   "repository": "developit/express-es6-rest-api",
   "author": "Iain Bryson <bryson.iain@gmail.com>",
   "license": "MIT",
   "dependencies": {
+    "agenda": "^1.0.3",
     "archiver": "^2.1.1",
     "async": "^2.6.0",
     "body-parser": "^1.13.3",
     "compression": "^1.5.2",
     "cors": "^2.7.1",
     "express": "^4.13.3",
+    "js-yaml": "^3.11.0",
     "mongojs": "^2.5.0",
     "morgan": "^1.8.0",
     "npm": "^5.7.1",
@@ -45,8 +57,12 @@
     "babel-core": "^6.9.0",
     "babel-preset-es2015": "^6.9.0",
     "babel-preset-stage-0": "^6.5.0",
+    "babelify": "^8.0.0",
     "eslint": "^3.1.1",
-    "nodemon": "^1.9.2"
+    "grunt": "^1.0.2",
+    "grunt-browserify": "^5.2.0",
+    "nodemon": "^1.9.2",
+    "runjs": "^4.3.0"
   },
   "engines": {
     "node": ">= 9.0"

getpdf-server/runfile.js

+import { run } from 'runjs';
+import * as jobs from './tasks/jobs';
+
+export default {
+    jobs
+}
\ No newline at end of file

getpdf-server/src/agenda.js

+import Agenda from "agenda";
+import yaml from 'js-yaml';
+import path from 'path';
+import fs from 'fs';
+import logger from './logger';
+
+const vol_root = process.env.DTU_ENOTE_VOL || "/data";
+const config_root = path.join(vol_root, "config");
+
+const mongoConnectionString = "sharelatex-db.docker.local:27018/getpdf_agenda";
+
+export const a = new Agenda({db: {address: mongoConnectionString}});
+
+const sharelatex_config_file = fs.readFileSync(path.join(config_root, "sharelatex.yaml"), {encoding:'utf8'});
+console.log(sharelatex_config_file)
+const sharelatex_config = yaml.safeLoad(sharelatex_config_file);
+
+console.dir(sharelatex_config);
+
+for (let course of Object.keys(sharelatex_config)) {
+    logger.debug(`Creating job definition for getpdf for ${course}`);
+
+    const opts =  {priority: 'high', concurrency: 10};
+    continue;
+    a.define(`GetPDF for ${course}`, opts, function(job, done) {
+//        User.remove({lastLogIn: { $lt: twoDaysAgo }}, done);
+        done();
+    });
+}

getpdf-server/src/api/getpdf.js

+import {Router} from 'express';
+import resource from 'resource-router-middleware';
+import {ObjectId} from 'mongojs';
+import path from 'path';
+import async from 'async';
+import fs from 'fs';
+import logger from '../logger';
+
+class GetPDFRouter {
+    constructor({config, dbs, router}) {
+        this.dbs = dbs;
+        this.config = config;
+        this.router = router;
+    }
+
+    listJobs(req, res) {
+        this.dbs.getpdf_db.jobs.find({active: {$eq: true}}, (err, existing) => {
+            if (err) {
+                throw err;
+            }
+            console.dir(existing);
+            if (existing.length > 0) {
+                res.status(403)
+                    .json({error: "Cannot create a new job; current job exists"});
+
+            } else {
+                const insert = this.dbs.getpdf_db.jobs.insert({active: true}, (err, result) => {
+                    if (err) {
+                        throw err;
+                    }
+                    console.dir(result);
+                    res.status(200)
+                        .json(result);
+                });
+            }
+
+        });
+    }
+
+    newJob(req, res) {
+        console.dir(req.params)
+        res.status(200)
+            .send(`${req.params.course_id}`);
+    }
+}
+
+
+function getPDFRouter({config, dbs}) {
+    const router = Router({mergeParams: true});
+    const getpdfRouter = new GetPDFRouter({config, dbs, router});
+
+    router.route('/')
+        .get(function (req, res) {
+            getpdfRouter.listJobs(req, res);
+        })
+        .post(function (req, res) {
+            getpdfRouter.newJob(req, res);
+        });
+
+    router.route('/:job_id')
+        .get(function (req, res) {
+            res.status(200)
+                .send(`${req.params.course_id} ${req.params.job_id}`);
+        });
+
+    return router;
+}
+
+export default getPDFRouter;

getpdf-server/src/api/index.js

 import { version } from '../../package.json';
 import { Router } from 'express';
 import zip from './zip';
+import getpdf from './getpdf';
 
-export default ({ config, db }) => {
+export default ({ config, dbs }) => {
 	let api = Router();
 
     // mount the zip resource
-    api.use('/zip', zip({ config, db }));
+    api.use('/zip', zip({ config, db: dbs.sharelatex_db }));
+
+    api.use('/getpdf/:course_id', getpdf({ config, dbs }));
 
     // perhaps expose some API metadata at the root
 	api.get('/', (req, res) => {

getpdf-server/src/db.js

@@ -2,7 +2,9 @@ import mongojs from "mongojs";
 
 export default callback => {
 	// connect to a database if needed, then pass it to `callback`:
-//    const db = mongojs("sharelatex-db.docker.local:27018/sharelatex", ["projects", "docs", "docOps"]);
-    const db = mongojs("sharelatex-db/sharelatex", ["projects", "docs", "docOps"]);
-	callback(db);
+    const sharelatex_db = mongojs("sharelatex-db.docker.local:27018/sharelatex", ["projects", "docs", "docOps"]);
+//    const db = mongojs("sharelatex-db/sharelatex", ["projects", "docs", "docOps"]);
+    const getpdf_db = mongojs("sharelatex-db.docker.local:27018/getpdf", ["jobs"]);
+//    const db = mongojs("sharelatex-db/getpdf", ["jobs"]);
+	callback({sharelatex_db, getpdf_db});
 }

getpdf-server/src/index.js

@@ -8,6 +8,7 @@ import middleware from './middleware';
 import api from './api';
 import config from './config.json';
 import logger from './logger';
+import agenda from './agenda';
 
 let app = express();
 app.server = http.createServer(app);
@@ -25,13 +26,13 @@ app.use(bodyParser.json({
 }));
 
 // connect to db
-initializeDb( db => {
+initializeDb( dbs => {
 
 	// internal middleware
-	app.use(middleware({ config, db }));
+	app.use(middleware({ config, dbs }));
 
 	// api router
-	app.use('/api', api({ config, db }));
+	app.use('/api', api({ config, dbs }));
 
 	app.server.listen(process.env.PORT || config.port, () => {
 		logger.info(`Started on port ${app.server.address().port}`);

getpdf-server/tasks/jobs.js

+import db from '../src/db.js';
+
+export async function clean() {
+    await new Promise(resolve => {
+        db((dbs) => {
+            dbs.getpdf_db.jobs.remove({}, (err, result) => {
+                if (err) {
+                    console.log(`Error ${err}`);
+                    console.dir(err);
+                } else {
+                    console.log(`Jobs deleted: ${result.deletedCount}`);
+                    console.dir(result);
+                }
+                dbs.getpdf_db.close();
+                resolve();
+            })
+        })
+    });
+    console.log("DONE");
+}
\ No newline at end of file