buien: Initial version using external buienradar-store

author: Tom Smeding <tom@tomsmeding.com> 2021-08-08 11:04:28 +0200
committer: Tom Smeding <tom@tomsmeding.com> 2021-08-08 11:04:38 +0200
commit: aadd3715f9ced40b0bf25729a9968d9c42a19cb2 (patch)
tree: c059e395c56e31ecf3b82a09308ac5d5a219c909
parent: b1955e8124e146fae1940226634bb09dd80d0935 (diff)
3 files changed, 413 insertions, 0 deletions
diff --git a/modules/subd-buien/.gitignore b/modules/subd-buien/.gitignore
new file mode 100644
index 0000000..6198c27
--- /dev/null
+++ b/modules/subd-buien/.gitignore
@@ -0,0 +1 @@
+buienradar-store
diff --git a/modules/subd-buien/index.html b/modules/subd-buien/index.html
new file mode 100644
index 0000000..03c56e8
--- /dev/null
+++ b/modules/subd-buien/index.html
@@ -0,0 +1,279 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Buien</title>
+<script>
+var stamps = null;
+var currentIndex = -1;
+var imageMap = new Map();  // stamp => (Image | false); false means 'still downloading'
+var pendingImage = null;  // null | [idx, stamp]
+
+var dbgpageload = new Date();
+
+function dbgnow() {
+	return new Date() - dbgpageload;
+}
+
+function renderStamp(s) {
+	function zeropad2(num) {
+		if (num < 10) return "0" + num;
+		else return num.toString();
+	}
+
+	var date = new Date(Date.UTC(
+		parseInt(s.slice(0, 4), 10),
+		parseInt(s.slice(5, 7), 10) - 1,
+		parseInt(s.slice(8, 10), 10),
+		parseInt(s.slice(11, 13), 10),
+		parseInt(s.slice(14, 16), 10),
+		parseInt(s.slice(17, 19), 10)
+	));
+	return date.toLocaleString("nl-NL", {timeZone: "Europe/Amsterdam"});
+	return date.getFullYear() + "-" +
+		zeropad2(date.getMonth() + 1) + "-" +
+		zeropad2(date.getDate()) + " " +
+		zeropad2(date.getHours()) + ":" +
+		zeropad2(date.getMinutes()) + ":" +
+		zeropad2(date.getSeconds());
+}
+
+function getStamps() {
+	var xhr = new XMLHttpRequest();
+	xhr.onreadystatechange = function () {
+		if (xhr.readyState == 4) {
+			if (xhr.status != 200){ 
+				alert("Error getting list of radar frames from server!");
+			} else {
+				stamps = xhr.response;
+				if (currentIndex == -1) {
+					doGotoLast();
+				}
+			}
+		}
+	};
+	xhr.open("GET", "/api/list/json");
+	xhr.responseType = "json";
+	xhr.send();
+}
+
+// direction: -1 to download earlier frames first; 1 to download later frames first.
+// Returns a list of indices in stamps.
+function determineImageSetAround(idx, direction) {
+	if (stamps.length == 0) return [];
+	if (idx < 0) idx = 0;
+	if (idx >= stamps.length) idx = stamps.length - 1;
+
+	var res = [];
+
+	var forwardPeek = 20;
+	var backwardPeek = 10;
+	var sizeLimit = 10;
+
+	function loopcheck(start, end, incr) {
+		for (var i = start; i != end; i += incr) {
+			if (res.length >= sizeLimit) return;
+			if (!imageMap.has(stamps[i])) res.push(i);
+		}
+	}
+
+	switch (direction) {
+		case -1:
+			loopcheck(idx, Math.max(-1, idx - forwardPeek - 1), -1);
+			loopcheck(Math.min(stamps.length, idx + 1), Math.min(stamps.length, idx + backwardPeek + 1), 1);
+			break;
+
+		case 1:
+		default:
+			loopcheck(idx, Math.min(stamps.length, idx + forwardPeek + 1), 1);
+			loopcheck(Math.max(-1, idx - 1), Math.max(-1, idx - backwardPeek - 1), -1);
+			break;
+	}
+
+	return res;
+}
+
+function readUint64LE(buffer, offset) {
+	var arr = new DataView(buffer, offset, 8);
+	var hi = arr.getUint32(4, true);
+	if (hi >= (1 << (53 - 32))) {
+		throw new Error("readUint64LE: Doesn't fit in Number");
+	}
+	return arr.getUint32(0, true) + (hi << 32);
+}
+
+// cb : (err) -> ()
+function downloadImages(wantedindices, cb) {
+	var indices = [];
+	for (var i = 0; i < wantedindices.length; i++) {
+		if (!imageMap.has(stamps[wantedindices[i]])) {
+			indices.push(wantedindices[i]);
+			imageMap.set(stamps[wantedindices[i]], false);
+		}
+	}
+
+	// console.log(dbgnow(), "Downloading:", indices);
+
+	var xhr = new XMLHttpRequest();
+	xhr.onreadystatechange = function(){
+		if (xhr.readyState == 4) {
+			if (xhr.status != 200){ 
+				cb("Server error while downloading images");
+			} else {
+				var buffer = xhr.response;
+
+				var numLoaded = 0, numTotal = 0;
+
+				var cursor = 0;
+				for (var i = 0; cursor < buffer.byteLength; i++) {
+					var pnglen = readUint64LE(buffer, cursor);
+					if (!(imageMap.get(stamps[indices[i]]) instanceof Image)) {
+						var arr = new Uint8Array(buffer, cursor + 8, pnglen);
+						var blob = new Blob([arr], { type: "image/png" })
+						var url = URL.createObjectURL(blob);
+						var image = new Image();
+						numTotal++;
+						image.onload = function () {
+							URL.revokeObjectURL(url);
+							numLoaded++;
+							if (numLoaded >= numTotal) {
+								// console.log(dbgnow(), "Done:", indices);
+								cb(null);
+							}
+						};
+						image.src = url;
+						imageMap.set(stamps[indices[i]], image);
+					}
+					cursor += 8 + pnglen;
+				}
+			}
+		}
+	};
+	xhr.open("GET", "/api/images?s=" + indices.map(i => stamps[i]).join(","));
+	xhr.responseType = "arraybuffer";
+	xhr.send();
+}
+
+function showPendingImage() {
+	if (pendingImage != null) {
+		var idx = pendingImage[0], stamp = pendingImage[1];
+		pendingImage = null;
+		showImage(idx, stamp);
+	}
+}
+
+function showImage(idx, stamp) {
+	// console.log("showImage(" + idx + ", '" + stamp + "') -> " + imageMap.has(stamp));
+	if (imageMap.has(stamp)) {
+		var image = imageMap.get(stamp);
+		if (image === false) {
+			pendingImage = [idx, stamp];
+			return;
+		}
+
+		var cvs = document.getElementById("imgcvs");
+		var ctx = cvs.getContext("2d");
+		ctx.drawImage(image, 0, 0);
+
+		var dateelem = document.getElementById("imgdate");
+		dateelem.innerHTML = "";
+		dateelem.appendChild(document.createTextNode(renderStamp(stamp)));
+
+		pendingImage = null;
+	} else {
+		pendingImage = [idx, stamp];
+
+		downloadImages(determineImageSetAround(idx), function (err) {
+			if (err) {
+				alert("Error getting images from server:\n" + err);
+				pendingImage = null;
+			} else {
+				showPendingImage();
+			}
+		});
+	}
+}
+
+function gotoIndex(idx) {
+	if (stamps == null || stamps.length == 0) { currentIndex = -1; return; }
+	if (idx < 0) idx = 0;
+	if (idx >= stamps.length) idx = stamps.length - 1;
+
+	var dir = idx == currentIndex - 1 ? -1
+				: idx == currentIndex + 1 ? 1
+				: 0;
+
+	currentIndex = idx;
+
+	// console.log(dbgnow(), "Goto " + currentIndex);
+
+	if (dir != 0) {
+		var missing = false;
+		for (var i = 1; i <= 8; i++) {
+			if (currentIndex + i * dir < 0 || currentIndex + i * dir >= stamps.length) break;
+			if (!imageMap.has(stamps[currentIndex + i * dir])) { missing = true; break; }
+		}
+
+		if (missing) {
+			downloadImages(determineImageSetAround(currentIndex, dir), function () {
+				showPendingImage();
+			});
+		}
+	}
+
+	showImage(currentIndex, stamps[currentIndex]);
+}
+
+function doGotoFirst() {
+	if (stamps == null) return;
+	gotoIndex(0);
+}
+
+function doGotoOffset(off) {
+	if (stamps == null) return;
+	var newIndex = Math.min(stamps.length - 1, Math.max(0, currentIndex + off));
+	gotoIndex(newIndex);
+}
+
+function doGotoLast() {
+	if (stamps == null) return;
+	gotoIndex(stamps.length > 0 ? stamps.length - 1 : 0);
+}
+
+window.addEventListener("load", function () {
+	getStamps();
+});
+
+window.addEventListener("keydown", function (ev) {
+	if (ev.key == "ArrowLeft") { doGotoOffset(-1); ev.preventDefault(); }
+	else if (ev.key == "ArrowRight") { doGotoOffset(1); ev.preventDefault(); }
+});
+</script>
+<style>
+button.wide {
+	width: 40px;
+	margin-right: 20px;
+}
+</style>
+</head>
+<body>
+<noscript><b>This page requires JavaScript to do anything.</b></noscript>
+
+<h1>Buien</h1>
+
+<p>Warning: This page can load a lot of images; one frame is between 30KB and 70KB.
+One day (288 frames) is about 14MB.</p>
+
+<canvas id="imgcvs" width="550" height="512">Your browser does not support the <code>&lt;canvas&gt;</code> element. Get a newer/different browser.</canvas>
+<br>
+<span id="imgdate"></span><br>
+<button class="wide" onclick="doGotoFirst()">&lt;&lt;</button>
+<button class="wide" onclick="doGotoOffset(-288)">-1d</button>
+<button class="wide" onclick="doGotoOffset(-12)">-1h</button>
+<button class="wide" onclick="doGotoOffset(-1)">&lt;</button>
+<button class="wide" onclick="doGotoOffset(1)">&gt;</button>
+<button class="wide" onclick="doGotoOffset(12)">+1h</button>
+<button class="wide" onclick="doGotoOffset(288)">+1d</button>
+<button class="wide" onclick="doGotoLast()">&gt;&gt;</button>
+</body>
+</html>
diff --git a/modules/subd-buien/subd-buien.js b/modules/subd-buien/subd-buien.js
new file mode 100644
index 0000000..1d599eb
--- /dev/null
+++ b/modules/subd-buien/subd-buien.js
@@ -0,0 +1,133 @@
+const cmn = require("../$common.js");
+const fs = require("fs");
+const express = require("express");
+const child_process = require("child_process");
+
+// NOTE:
+// This module expects a buienradar-store instance in the 'buienradar-store'
+// directory within this module's directory. The database therein should be
+// named 'db', and the background image should be stored under key 'mode'
+// (which should probably be the cmpref).
+
+
+// cb : (err, status, stdout : Buffer) -> ()
+function runDbmanage(bdir, args, cb) {
+	const options = {
+		cwd: bdir,
+		stdio: ["inherit", "pipe", "inherit"],
+		timeout: 5000,  // milliseconds
+	};
+	const proc = child_process.spawn(bdir + "/dbmanage", args, options);
+
+	const outbufs = [];
+	proc.stdout.on("data", data => {
+		outbufs.push(data);
+	});
+
+	let callbackInvoked = false;
+
+	proc.on("error", err => {
+		if (callbackInvoked) return;
+		callbackInvoked = true;
+		cb(err, null, null);
+	});
+
+	proc.on("exit", (code, sig) => {
+		if (callbackInvoked) return;
+		callbackInvoked = true;
+		if (code != null) cb(null, code, Buffer.concat(outbufs));
+		else cb("Process terminated by signal", null, null);
+	});
+}
+
+// cb : (err, frames : [String]) -> ()
+function getFramesList(bdir, cb) {
+	runDbmanage(bdir, ["list", bdir + "/db"], (err, status, out) => {
+		if (err) cb(err, null);
+		else if (status != 0) cb("Error getting list of frames from DB", null);
+		else cb(null, out.toString("utf8").trim().split("\n").filter(s => s != "mode"));
+	});
+}
+
+// cb : (err, pngs : Buffer) -> ()
+function getFramesPng(bdir, stamps, cb) {
+	runDbmanage(bdir, ["getmultiple", bdir + "/db"].concat(stamps), (err, status, out) => {
+		if (err) { cb(err, null); return; }
+		if (status != 0) { cb("Error getting frames from DB", null); return; }
+		cb(null, out);
+
+		// const pngs = [];
+
+		// let cursor = 0;
+		// while (cursor < out.length) {
+		//     let pnglen = out.readBigUInt64LE(cursor);
+		//     if (Number(pnglen) != pnglen) {
+		//         cb("PNG too large", null);
+		//         return;
+		//     }
+		//     pnglen = Number(pnglen);
+
+		//     pngs.push(out.slice(cursor + 8, cursor + 8 + pnglen));
+		//     cursor += 8 + pnglen;
+		// }
+
+		// cb(null, pngs);
+	});
+}
+
+// cb : (err, present : Bool) -> ()
+function checkFramePresent(bdir, stamp, cb) {
+	runDbmanage(bdir, ["has", bdir + "/db", stamp], (err, status, out) => {
+		if (err) { cb(err, null); return; }
+		if (status == 1) { cb("Error getting status from DB", null); return; }
+		if (status == 0) cb(null, true);
+		else cb(null, false);
+	});
+}
+
+module.exports = function (app, io, moddir) {
+	const buiendir = moddir + "/buienradar-store";
+	console.log(`buien: Using buienradar-store directory '${buiendir}'`);
+
+	const router = express.Router();
+
+	router.get("/", function (req, res) {
+		res.sendFile(moddir + "/index.html");
+	});
+
+	router.get("/api/list/json", function (req, res) {
+		getFramesList(buiendir, (err, frames) => {
+			if (err) {
+				console.log(err);
+				res.sendStatus(500);
+			} else res.json(frames);
+		});
+	});
+
+	router.get("/api/images", function (req, res) {
+		const url = new URL(req.url, `https://${req.headers.host}`);
+		const stamps = url.searchParams.getAll("s").flatMap(s => s.split(","));
+		getFramesPng(buiendir, stamps, (err, buf) => {
+			if (err) {
+				console.log(err);
+				res.sendStatus(500);
+			} else res.send(buf);
+		});
+	});
+
+	router.get("/api/has/:stamp", function (req, res) {
+		checkFramePresent(buiendir, req.params.stamp, (err, present) => {
+			if (err) {
+				console.log(err);
+				res.sendStatus(500);
+			} else if (present) res.sendStatus(200);
+			else res.sendStatus(404);
+		});
+	});
+
+	app.use(function (req, res, next) {
+		if (req.subdomains.length && req.subdomains[req.subdomains.length - 1] == "buien") { 
+			router.handle(req, res, next);
+		} else next();
+	});
+};
author	Tom Smeding <tom@tomsmeding.com>	2021-08-08 11:04:28 +0200
committer	Tom Smeding <tom@tomsmeding.com>	2021-08-08 11:04:38 +0200
commit	aadd3715f9ced40b0bf25729a9968d9c42a19cb2 (patch)
tree	c059e395c56e31ecf3b82a09308ac5d5a219c909
parent	b1955e8124e146fae1940226634bb09dd80d0935 (diff)