diff options
Diffstat (limited to 'modules')
| -rw-r--r-- | modules/$common.js | 17 | ||||
| -rw-r--r-- | modules/blog/blog.js | 2 | ||||
| -rw-r--r-- | modules/changes/changes.js | 2 | ||||
| -rw-r--r-- | modules/proxy/proxy.js | 2 | ||||
| -rw-r--r-- | modules/save/save.js | 2 | ||||
| -rw-r--r-- | modules/statusbot/statusbot.js | 50 | ||||
| -rw-r--r-- | modules/timetrack/timetrack.js | 2 | ||||
| -rw-r--r-- | modules/timetrack2/timetrack2.js | 2 | ||||
| -rw-r--r-- | modules/timetrack3/timetrack3.js | 2 | ||||
| -rw-r--r-- | modules/todo/todo.js | 2 | ||||
| -rw-r--r-- | modules/unicode/unicode.js | 6 | ||||
| -rw-r--r-- | modules/up_log/index.html | 37 | ||||
| -rw-r--r-- | modules/up_log/up_log.js | 27 | ||||
| -rwxr-xr-x | modules/vacancies/bak/chalmers.sh | 30 | ||||
| -rwxr-xr-x | modules/vacancies/getters/aarhus.sh | 8 | ||||
| -rwxr-xr-x | modules/vacancies/getters/cambridge.sh | 9 | ||||
| -rwxr-xr-x | modules/vacancies/getters/glasgow.sh | 8 | ||||
| -rwxr-xr-x | modules/vacancies/getters/mpg.sh | 8 | ||||
| -rw-r--r-- | modules/vacancies/vacancies.js | 84 | ||||
| -rw-r--r-- | modules/zelfoverhoor/zelfoverhoor.js | 4 |
20 files changed, 271 insertions, 33 deletions
diff --git a/modules/$common.js b/modules/$common.js index 59b9a66..f7ec42a 100644 --- a/modules/$common.js +++ b/modules/$common.js @@ -4,13 +4,28 @@ var path=require("path"), var cwd=process.cwd(); var globalAccounts=require(cwd+"/globalAccounts.json"); +var statusbotObject={ + // calls cb with http status code + "send":function statusbotSend(sender,text,cb){ + if(statusbotObject._handler==null){ + console.log("[common.statusbot] No statusbot yet"); + cb(500); + return; + } + statusbotObject._handler(sender,text,cb); + }, + "_handler":null, // to be set from the statusbot module +}; + module.exports={ "serverdir":cwd, "webfilesdir":cwd+"/web_files", "persistdir":cwd+"/persist", + "simpleHTMLescape":function simpleHTMLescape(str){ return str.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>'); }, + "authgen":function authgen(accounts){ //omit `accounts` to use the globalAccounts list if(!accounts)accounts=globalAccounts; return function (req,res,next){ @@ -32,4 +47,6 @@ module.exports={ return unauth(res); }; }, + + "statusbot":statusbotObject, }; diff --git a/modules/blog/blog.js b/modules/blog/blog.js index 3985fdb..4843d4b 100644 --- a/modules/blog/blog.js +++ b/modules/blog/blog.js @@ -112,7 +112,7 @@ module.exports = (app, io, _moddir) => { next(); }); - app.get("/blog/*", (req, res) => { + app.get("/blog/*rest", (req, res) => { if (req.path.indexOf("/.") != -1) { res.sendStatus(404); return; diff --git a/modules/changes/changes.js b/modules/changes/changes.js index 0d8fa24..9fa1e44 100644 --- a/modules/changes/changes.js +++ b/modules/changes/changes.js @@ -320,7 +320,7 @@ function performCleanup(){ module.exports=function(app,io,_moddir){ moddir=_moddir; - app.all(["/changes","/changes/*"],cmn.authgen()); + app.all(["/changes","/changes/*rest"],cmn.authgen()); app.get("/changes",function(req,res){ res.sendFile(moddir+"/changes.html"); }); diff --git a/modules/proxy/proxy.js b/modules/proxy/proxy.js index ccec53a..3ebe236 100644 --- a/modules/proxy/proxy.js +++ b/modules/proxy/proxy.js @@ -39,7 +39,7 @@ module.exports=function(app,io,moddir){ return false; } - app.all("/proxy/:id/*",function(req,res){ + app.all("/proxy/:id/*rest",function(req,res){ var id=req.params.id; var path="/"+req.path.split("/").slice(3).join("/"); if(iddict[id]){ diff --git a/modules/save/save.js b/modules/save/save.js index 9873a52..ae1cfaf 100644 --- a/modules/save/save.js +++ b/modules/save/save.js @@ -46,7 +46,7 @@ module.exports=function(app,io,_moddir){ }); }); - app.all(["/save/read","/save/read/*"],cmn.authgen()); + app.all(["/save/read","/save/read/*rest"],cmn.authgen()); app.get("/save/read",function(req,res){ res.sendFile(moddir+"/read.html"); diff --git a/modules/statusbot/statusbot.js b/modules/statusbot/statusbot.js index 94894a9..38d115b 100644 --- a/modules/statusbot/statusbot.js +++ b/modules/statusbot/statusbot.js @@ -279,6 +279,35 @@ function matrixSendMsgLogin(text, cb) { }); } +// controller for message sending +const ratelimit = new PRatelimit(1000); + +// Calls cb with http status code +function handleMessage(sender, text, cb) { + if (typeof sender != "string" || typeof text != "string" || sender.indexOf("\n") != -1) { + cb(400); + return; + } + + ratelimit.submit(rlcb => { + ptimeout(5000, + cb2 => matrixSendMsgLogin(`[${sender}] ${text}`, cb2), + (finished, success) => { + if (!finished) { + logFailure(`Timed out on message: [${sender}] ${text}`, () => {}); + cb(504); // gateway timeout + } else if (!success) { + logFailure(`Unsuccessful for message: [${sender}] ${text}`, () => {}); + cb(503); // service unavailable + } else cb(200); + rlcb(); + } + ); + }); +} + +cmn.statusbot._handler = handleMessage; + module.exports = function(app, io, _moddir) { moddir = _moddir; @@ -291,30 +320,11 @@ module.exports = function(app, io, _moddir) { return false; } - const ratelimit = new PRatelimit(1000); - augmentConfig(config, state => { gstate = state; app.post("/statusbot", bodyParser.json(), cmn.authgen(accounts), (req, res) => { - if (typeof req.body.sender != "string" || typeof req.body.text != "string") { - return res.sendStatus(400); - } - ratelimit.submit(rlcb => { - ptimeout(5000, - cb => matrixSendMsgLogin(`[${req.body.sender}] ${req.body.text}`, cb), - (finished, success) => { - if (!finished) { - res.sendStatus(504); // gateway timeout - logFailure(`Timed out on message: [${req.body.sender}] ${req.body.text}`, () => {}); - } else if (!success) { - res.sendStatus(503); // service unavailable - logFailure(`Unsuccessful for message: [${req.body.sender}] ${req.body.text}`, () => {}); - } else res.sendStatus(200); - rlcb(); - } - ); - }); + handleMessage(req.body.sender, req.body.text, status => res.sendStatus(status)); }); }); }; diff --git a/modules/timetrack/timetrack.js b/modules/timetrack/timetrack.js index 886ea1e..b654ef1 100644 --- a/modules/timetrack/timetrack.js +++ b/modules/timetrack/timetrack.js @@ -159,7 +159,7 @@ module.exports=function(app,io,_moddir){ }); }); - app.all(["/timetrack","/timetrack/*"],authMiddleware); //for all the other endpoints + app.all(["/timetrack","/timetrack/*rest"],authMiddleware); //for all the other endpoints app.get("/timetrack",function(req,res){ res.sendFile(moddir+"/timetrack.html"); diff --git a/modules/timetrack2/timetrack2.js b/modules/timetrack2/timetrack2.js index cb22132..f3d35ae 100644 --- a/modules/timetrack2/timetrack2.js +++ b/modules/timetrack2/timetrack2.js @@ -163,7 +163,7 @@ module.exports = function(app, io, _moddir){ }); }); - app.all([ROOT_ENDPOINT, ROOT_ENDPOINT+"/*"], authMiddleware); //for all the other endpoints + app.all([ROOT_ENDPOINT, ROOT_ENDPOINT+"/*rest"], authMiddleware); //for all the other endpoints app.get(ROOT_ENDPOINT, function(req, res){ res.sendFile(moddir + "/timetrack.html"); diff --git a/modules/timetrack3/timetrack3.js b/modules/timetrack3/timetrack3.js index dbb0e5b..ba1d9c0 100644 --- a/modules/timetrack3/timetrack3.js +++ b/modules/timetrack3/timetrack3.js @@ -202,7 +202,7 @@ module.exports = function(app, io, _moddir){ }); // for all the other endpoints, authorisation is needed - app.all([ROOT_ENDPOINT, ROOT_ENDPOINT + "/*"], authMiddleware); + app.all([ROOT_ENDPOINT, ROOT_ENDPOINT + "/*rest"], authMiddleware); // - -> html app.get(ROOT_ENDPOINT, (req, res) => { diff --git a/modules/todo/todo.js b/modules/todo/todo.js index b5d3417..239dc19 100644 --- a/modules/todo/todo.js +++ b/modules/todo/todo.js @@ -170,7 +170,7 @@ module.exports=function(app,io,_moddir){ }); }); - app.all(["/todo","/todo/*"],authMiddleware); //for all the other endpoints + app.all(["/todo","/todo/*rest"],authMiddleware); //for all the other endpoints app.get("/todo",function(req,res){ var contents=fs.readFileSync(moddir+"/todo.html","utf8"); diff --git a/modules/unicode/unicode.js b/modules/unicode/unicode.js index 4b313fc..84191fa 100644 --- a/modules/unicode/unicode.js +++ b/modules/unicode/unicode.js @@ -1,11 +1,15 @@ "use strict"; +// TODO +// - https://www.unicode.org/Public/UCD/latest/ucd/NameAliases.txt (https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-4/#G2082) + const cmn = require("../$common.js"); const fs = require("fs"); const path = require("path"); const https = require("https"); +// https://www.unicode.org/reports/tr44/#UnicodeData.txt const fCODE = 0; const fNAME = 1; const fCATEGORY = 2; @@ -15,7 +19,7 @@ const fDIGIT = 7; const fNUMERIC = 8; const fMIRRORED = 9; const fOLDNAME = 10; -const fCOMMENT = 11; +// comment (always null by spec, deprecated) const fUPPERCASE = 12; const fLOWERCASE = 13; const fTITLECASE = 14; diff --git a/modules/up_log/index.html b/modules/up_log/index.html new file mode 100644 index 0000000..d8371d9 --- /dev/null +++ b/modules/up_log/index.html @@ -0,0 +1,37 @@ +<!doctype html> +<html> +<head> +<meta charset="utf-8"> +<title>Uplog</title> +<style> +#log { + border: 1px gray solid; +} +</style> +<script> +function get() { + var xhr = new XMLHttpRequest(); + xhr.open("GET", location.href + "/log/" + document.getElementById("count").value); + xhr.responseType = "text"; + xhr.onreadystatechange = function() { + if (xhr.readyState == 4) { + if (xhr.status == 200) { + document.getElementById("log").innerHTML = ""; + document.getElementById("log").appendChild(document.createTextNode(xhr.responseText)); + } else { + alert(xhr.responseText); + } + } + }; + xhr.send(); +} +</script> +</head> +<body> +Count: <input type="number" id="count" value="100" min="1" onchange="get()"> +<input type="button" onclick="get()" value="Get"> +<br> +<pre id="log"></pre> +<script>get();</script> +</body> +</html> diff --git a/modules/up_log/up_log.js b/modules/up_log/up_log.js index 609d705..61d2af0 100644 --- a/modules/up_log/up_log.js +++ b/modules/up_log/up_log.js @@ -4,7 +4,7 @@ const bodyParser = require("body-parser"); let moddir = null; -module.exports=function(app,io,_moddir){ +module.exports = function(app, io, _moddir) { moddir = _moddir; let config, accounts; @@ -15,7 +15,7 @@ module.exports=function(app,io,_moddir){ return false; } - app.post("/up_log/log", bodyParser.json(), cmn.authgen(accounts), (req,res) => { + app.post("/up_log/log", bodyParser.json(), cmn.authgen(accounts), (req, res) => { if (typeof req.body.machine != "string") { return res.sendStatus(400); } @@ -26,4 +26,27 @@ module.exports=function(app,io,_moddir){ res.status(200).end(); }); + + app.get("/up_log/log/:count", cmn.authgen(), (req, res) => { + const reqln = +req.params.count; + if (reqln < 0 || reqln % 1 != 0 || isNaN(reqln)) { res.sendStatus(400); return; } + + fs.readFile(moddir + "/log.txt", (err, data) => { + if (err) { res.sendStatus(500); return; } + + let i = data.length - 1 - (data[data.length-1] == 10); + let nln = 0; + while (true) { + if (data[i] == 10) { nln++; if (nln >= reqln) { i++; break; } } + i--; + if (i <= 0) { nln++; break; } + } + + res.send(data.slice(i)); + }); + }); + + app.get("/up_log", cmn.authgen(), (req, res) => { + res.sendFile(moddir + "/index.html"); + }); }; diff --git a/modules/vacancies/bak/chalmers.sh b/modules/vacancies/bak/chalmers.sh new file mode 100755 index 0000000..5583223 --- /dev/null +++ b/modules/vacancies/bak/chalmers.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +url='https://web103.reachmee.com/ext/I003/304/main?site=5&validator=a72aeedd63ec10de71e46f8d91d0d57c&lang=UK' + +script=' +0,/<div id="mainjoblist">/d +/<tbody>/,/<\/tbody>/!d + +/<\/tr>/ { + s/.*// + x + s/\n//g + s/^ | // + /Technical and Administrative staff/d + /PhD Student Positions/d + p + d +} + +/<td>/!d +/^\s*<\/td>\s*$/d +s|\s*<td>\(.*\)</td>\s*|\1| +s|.*Application deadline:.*display:\s*none">\([^<]*\)</span>.*|\1| +s|.*a href=.*reachmee.*/job.*job_id[^>]*>\([^<]*\)</a>.*|\1| +s/^/ | / +H +' + +curl -s "$url" | sed -n "$script" diff --git a/modules/vacancies/getters/aarhus.sh b/modules/vacancies/getters/aarhus.sh new file mode 100755 index 0000000..8acbe6f --- /dev/null +++ b/modules/vacancies/getters/aarhus.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +curl -sL 'https://cs.au.dk/about-us/vacancies/' \ + | grep 'DYCON\.Em[^ ]*vacancies' \ + | head -1 \ + | sed 's/^[^[]*//; s/; *$//' \ + | jq -r '.[] | (.title + " [https://cs.au.dk" + .link + "]")' diff --git a/modules/vacancies/getters/cambridge.sh b/modules/vacancies/getters/cambridge.sh new file mode 100755 index 0000000..f5f9fe1 --- /dev/null +++ b/modules/vacancies/getters/cambridge.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +curl -sL 'https://www.cam.ac.uk/jobs/term/Department-of-Computer-Science-and-Technology' | \ + sed ' +/<tbody>/,/<\/tbody>/!d +s|<a href="/jobs/term/[^"]*">[^<]*</a>||g +/<a href="\/jobs\//!d +s|^ *<a href="\([^"]*\)">\(.*\)</a>.*|\2 (https://www.cam.ac.uk\1)|' diff --git a/modules/vacancies/getters/glasgow.sh b/modules/vacancies/getters/glasgow.sh new file mode 100755 index 0000000..c6beef5 --- /dev/null +++ b/modules/vacancies/getters/glasgow.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +curl -sL 'https://www.jobs.gla.ac.uk/jobs/college-of-science-and-engineering/school-of-computing-science-1' \ + | sed ' +/a href="\/job\// !d +/>Read More</ d +s|^ *<a href="\([^"]*\)">\(.*\)</a>.*|\2 (https://www.jobs.gla.ac.uk\1)|' diff --git a/modules/vacancies/getters/mpg.sh b/modules/vacancies/getters/mpg.sh new file mode 100755 index 0000000..bf7f222 --- /dev/null +++ b/modules/vacancies/getters/mpg.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +# <td headers="view-ocm-pp-project-title-table-column" class="views-field views-field-ocm-pp-project-title"><a href="/node/21090">Identifying novel ways of inducing effective angiogenesis and the development of arteries</a> </td> + + +curl -s 'https://postdocprogram.mpg.de/all-postdoc-positions' \ + | sed -n '/td.*project-title/ s/^ *<td[^>]*> *<a href="\([^"]*\)"[^>]*>\([^<]*\).*/\2 (https:\/\/postdocprogram.mpg.de\1)/p' diff --git a/modules/vacancies/vacancies.js b/modules/vacancies/vacancies.js new file mode 100644 index 0000000..be3c4b4 --- /dev/null +++ b/modules/vacancies/vacancies.js @@ -0,0 +1,84 @@ +const cmn = require("../$common.js"); +const fs = require("fs"); +const child_process = require("child_process"); +const persist = require("node-persist"); +const mkdirp = require("mkdirp"); + +const FIRST_FETCH_DELAY = 10 * 60 * 1000; // 10 seconds +const INTERVAL = 3 * 24 * 3600 * 1000; // 3 days +// const FIRST_FETCH_DELAY = 2 * 1000; +// const INTERVAL = 20 * 1000; + +let moddir = null; + +mkdirp.sync(cmn.persistdir + "/vacancies"); +const DB = persist.create({ + dir: cmn.persistdir + "/vacancies", + continuous: false, + interval: false, +}); +DB.initSync(); + +function sendNotification(text, cb) { + cmn.statusbot.send("vacancies", text, cb); + // console.log("--> " + text); + // cb(200); +} + +function refreshSite(name) { + child_process.execFile(moddir + "/getters/" + name + ".sh", (err, stdout, _stderr) => { + if (err != null) { + sendNotification("Error getting <" + name + ">: " + err, () => {}); + return; + } + + const lines = stdout.split("\n").filter(s => s.length > 0); + let news = []; + + let prev = DB.getItemSync(name); + if (prev != null) { + prev = new Set(prev); + for (let line of lines) { + if (!prev.has(line)) news.push(line); + } + } else { + news = lines; + } + + if (news.length == 0) { + console.log("[vacancies] No news for <" + name + ">"); + return; + } + console.log("[vacancies] " + news.length + " news for <" + name + ">"); + + const message = news.map(s => "<" + name + "> " + s).join("\n"); + sendNotification(message, status => { + if (status == 200) { + DB.setItemSync(name, lines); + } else { + console.log("[vacancies] Failed sending: [[[" + message + "]]]"); + } + }); + }); +} + +function refreshAll() { + console.log("[vacancies] Refreshing"); + let i = 0; + for (let name of fs.readdirSync(moddir + "/getters")) { + if (!name.endsWith(".sh")) continue; + setTimeout(() => refreshSite(name.slice(0, -3)), 2000 * i); + i++; + } +} + +module.exports = (app, io, _moddir) => { + moddir = _moddir; + + setTimeout(() => { + refreshAll(); + setInterval(() => { + refreshAll(); + }, INTERVAL); + }, FIRST_FETCH_DELAY); // wait a while before the first fetch +}; diff --git a/modules/zelfoverhoor/zelfoverhoor.js b/modules/zelfoverhoor/zelfoverhoor.js index 0acb322..9474378 100644 --- a/modules/zelfoverhoor/zelfoverhoor.js +++ b/modules/zelfoverhoor/zelfoverhoor.js @@ -117,7 +117,7 @@ module.exports=function(app,io,_moddir){ persistDB(); } - app.all("/zelfoverhoor*",function(req,res,next){ + app.all("/zelfoverhoor*rest",function(req,res,next){ res.header("Cache-Control","private, no-cache, no-store, must-revalidate"); res.header("Expires","-1"); res.header("Pragma","no-cache"); @@ -161,7 +161,7 @@ module.exports=function(app,io,_moddir){ res.send(JSON.stringify(resset)); }); - app.use(["/zelfoverhoor/docent","/zelfoverhoor/docent/*"],cmn.authgen(accounts)); + app.use(["/zelfoverhoor/docent","/zelfoverhoor/docent/*rest"],cmn.authgen(accounts)); app.get("/zelfoverhoor/docent",function(req,res){ fs.readFile(moddir+"/docent.html",function(err,data){ if(err)throw err; |