Initial

author: tomsmeding <tom.smeding@gmail.com> 2016-09-13 11:43:06 +0200
committer: tomsmeding <tom.smeding@gmail.com> 2016-09-13 11:44:11 +0200
commit: f00ba92ed2cc1a9c24ad783e83525d1b5a85b857 (patch)
tree: 4d1c00a47c7f3842bcf3dece83d3c00ed3ae459f /modules/changes
2 files changed, 420 insertions, 0 deletions
diff --git a/modules/changes/changes.html b/modules/changes/changes.html
new file mode 100644
index 0000000..cda3da1
--- /dev/null
+++ b/modules/changes/changes.html
@@ -0,0 +1,257 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Website change monitor</title>
+<script>
+"use strict";
+
+var urls=[];
+
+function pad(s,n,c){
+	s=s+"";
+	if(!c)c="0";
+	var fill=c,filllen=n-s.length;
+	if(filllen<=0)return s;
+	while(fill.length<filllen)fill+=c;
+	return fill+s;
+}
+
+function dateformat(date){
+	return date.getFullYear()+"/"+pad(date.getMonth(),2)+"/"+pad(date.getDate(),2)+" "+
+	       pad(date.getHours(),2)+":"+pad(date.getMinutes(),2)+":"+pad(date.getSeconds(),2);
+}
+
+function fetch(method,url,data/*?*/,cb){
+	if(!cb){
+		cb=data;
+		data=undefined;
+		if(!cb)throw new Error("No callback passed to fetch");
+	}
+	var xhr=new XMLHttpRequest();
+	xhr.onreadystatechange=function(ev){
+		if(xhr.readyState<4)return;
+		cb(xhr.status,xhr.responseText);
+	};
+	xhr.open(method,url);
+	xhr.send(data);
+}
+
+function focusData(data){
+	document.getElementById("timelinecontainer").classList.add("visible");
+	var header=document.getElementById("timelineheader");
+	var tbody=document.getElementById("timelinetbody");
+	if(!header.firstChild)header.appendChild(document.createTextNode(data.url));
+	else header.firstChild.nodeValue=data.url;
+
+	var l=tbody.children,i;
+	for(i=l.length-1;i>=0;i--)tbody.removeChild(l[i]);
+
+	var tr,td,s;
+	for(i=data.timeline.length-1;i>=0;i--){
+		tr=document.createElement("tr");
+		if(i<data.timeline.length-1&&data.timeline[i][1]==data.timeline[i+1][1]){
+			tr.classList.add("repeated");
+		}
+
+		td=document.createElement("td");
+		td.appendChild(document.createTextNode(dateformat(new Date(data.timeline[i][0]))));
+		tr.appendChild(td);
+
+		td=document.createElement("td");
+		s=data.timeline[i][1];
+		if(s==null)s="(error while retrieving page)";
+		td.appendChild(document.createTextNode(s));
+		tr.appendChild(td);
+
+		tbody.appendChild(tr);
+	}
+}
+
+function makeURLtr(url){
+	var tr=document.createElement("tr");
+	var td=document.createElement("td");
+
+	td.appendChild(document.createTextNode(url));
+
+	var e=document.createElement("input");
+	e.type="button";
+	e.classList.add("rowdeletebutton");
+	e.value="X";
+	e.setAttribute("title","Delete URL");
+	e.addEventListener("click",function(ev){
+		ev.stopPropagation();
+		fetch("DELETE","/changes/url",url,function(status,body){
+			if(document.getElementById("timelineheader").firstChild.nodeValue==url){
+				document.getElementById("timelinecontainer").classList.remove("visible");
+			}
+			if(status!=200){
+				alert("Error deleting url: ("+status+") "+body);
+				return;
+			}
+			tr.parentNode.removeChild(tr);
+			updateURLs();
+		});
+	});
+	td.appendChild(e);
+
+	tr.addEventListener("click",function(ev){
+		fetch("GET","/changes/url?url="+encodeURIComponent(url),function(status,body){
+			if(status!=200){
+				alert("Error getting data: ("+status+") "+body);
+				return;
+			}
+			var data;
+			try {
+				data=JSON.parse(body);
+			} catch(e){
+				alert("Server sent invalid data!");
+				console.log("Invalid data:",data);
+				return;
+			}
+			focusData(data);
+		});
+	});
+
+	tr.appendChild(td);
+	return tr;
+}
+
+function updateURLs(){
+	fetch("GET","/changes/urls",function(status,body){
+		urls=JSON.parse(body);
+		if(!urls){
+			urls=[];
+			alert("Error retrieving URLs!");
+			return;
+		}
+		var tbody=document.getElementById("urlstable").firstElementChild;
+		var l=tbody.children,i,tr,td;
+		for(i=l.length-1;i>=0;i--)tbody.removeChild(l[i]);
+		for(i=0;i<urls.length;i++){
+			tbody.appendChild(makeURLtr(urls[i]));
+		}
+	});
+}
+
+function addURLbutton(){
+	var e=document.getElementById("urladdinput");
+	var url=e.value;
+	fetch("POST","/changes/url",url,function(status,body){
+		if(status==200){
+			e.value="";
+		} else {
+			alert("URL submission error: ("+status+") "+body);
+		}
+		updateURLs();
+	});
+}
+
+function addURLkeypress(ev){
+	if(ev.keyCode==10||ev.keyCode==13)addURLbutton();
+}
+
+function sendRefresh(){
+	fetch("POST","/changes/refresh",function(status,body){});
+}
+
+window.addEventListener("load",function(){
+	updateURLs();
+	//setInterval(updateURLs,10*1000);
+});
+</script>
+<style>
+body{
+	font-family:Georgia,Times,serif;
+	font-size:14px;
+}
+h1{
+	/*font-size:28px;*/
+	/*margin:19px 0 19px 0;*/
+}
+#urlscontainer{
+	width:500px;
+}
+#urlstablecontainer{
+	height:130px;
+	border:1px #888 solid;
+	overflow-y:scroll;
+}
+#urlstable{
+	width:100%;
+	border-collapse:collapse;
+}
+#urlstable tbody, #urlstable tr, #urlstable td{
+	width:100%;
+}
+#urlstable tr{
+	cursor:pointer;
+}
+#urlstable tr:hover{
+	background-color:#ddd;
+}
+#urlstable tr:nth-child(2n){
+	background-color:#eee;
+}
+#urlstable td{
+	height:100%;
+	vertical-align:middle;
+}
+#urladdinput{
+	width:300px;
+}
+.rowdeletebutton{
+	color:red;
+	font-family:Monospace;
+	font-weight:bold;
+	float:right;
+	cursor:pointer;
+	padding:0 3px 0 3px;
+	background-color:rgba(255,0,0,0.1);
+	border-radius:2px;
+	border-width:0;
+}
+
+#timelinetbl{
+	border-collapse:collapse;
+}
+#timelinetbl td, #timelinetbl th{
+	border:1px #888 solid;
+}
+
+tr.repeated{
+	color:#aaa;
+	font-size:10px;
+}
+
+#timelinecontainer{
+	display:none;
+}
+#timelinecontainer.visible{
+	display:block;
+}
+</style>
+</head>
+<body>
+<h1>Website change monitor</h1>
+<b>URLs:</b>
+<input type="button" onclick="sendRefresh();" value="Redownload sites">
+<br>
+<div>
+	<div id="urlscontainer">
+		<div id="urlstablecontainer">
+			<table id="urlstable"><tbody></tbody></table>
+		</div>
+		<input type="text" id="urladdinput" onkeypress="addURLkeypress(event);">
+		<input type="button" onclick="addURLbutton();" value="Add URL" title="Add URL to watch list">
+	</div>
+</div>
+<div id="timelinecontainer">
+	<h2 id="timelineheader"></h2>
+	<table id="timelinetbl">
+		<thead><tr><th>Date</th><th>Hash</th></tr></thead>
+		<tbody id="timelinetbody"></tbody>
+	</table>
+</div>
+</body>
+</html>
diff --git a/modules/changes/changes.js b/modules/changes/changes.js
new file mode 100644
index 0000000..6a8c8f2
--- /dev/null
+++ b/modules/changes/changes.js
@@ -0,0 +1,163 @@
+"use strict";
+
+var cmn=require("../$common.js"),
+    persist=require("node-persist"),
+    crypto=require("crypto"),
+    http=require("http"),
+    https=require("https"),
+    URL=require("url");
+
+var moddir=null;
+
+persist=persist.create({
+	dir:"persist/changes",
+	continuous:false,
+	interval:false
+});
+persist.initSync();
+
+//urls: map(url => URLobject)
+//url: String
+//URLobject: {url, timeline: [[Date, hash]]}
+var urls=persist.getItemSync("urls");
+if(!urls){
+	urls={};
+	persist.setItemSync("urls",urls);
+}
+
+
+function URLobject(url){
+	if(!(this instanceof URLobject))return new URLobject(url);
+	this.url=url;
+	this.timeline=[];
+}
+
+function fetch(method,url,data/*?*/,cb){
+	var cbcalled=false;
+
+	if(!cb){
+		cb=data;
+		data=undefined;
+		if(!cb)throw new Error("No callback passed to fetch");
+	}
+
+	try {
+		url=URL.parse(url);
+	} catch(e){
+		cb(-1,null);
+		return;
+	}
+
+	var httpclass;
+	switch(url.protocol){
+		case "http:": httpclass=http; break;
+		case "https:": httpclass=https; break;
+		default:
+			cb(-1,null);
+			return;
+	}
+
+	url.method=method;
+
+	var req=httpclass.request(url,function(res){
+		var body="";
+		res.on("data",function(data){
+			body+=data;
+		});
+		res.on("end",function(){
+			if(!cbcalled)cb(res.statusCode,body);
+			cbcalled=true;
+		});
+		res.on("error",function(err){
+			if(!cbcalled)cb(-1,err);
+			cbcalled=true;
+		});
+	});
+	req.on("error",function(err){
+		if(!cbcalled)cb(-1,err);
+		cbcalled=true;
+	});
+	if(data)req.write(data);
+	req.end();
+}
+
+function refreshURLs(){
+	var hashes={};
+	var i;
+	var url;
+	var timeout=null;
+	for(url in urls){
+		console.log("Fetching <"+url+">");
+		fetch("GET",url,function(url,status,body){
+			console.log("Got <"+url+">; status = "+status);
+			if(status==-1){
+				hashes[url]=[new Date(),null];
+			} else {
+				var hash=crypto.createHash("sha256");
+				hash.update(body);
+				hashes[url]=[new Date(),hash.digest("hex")];
+			}
+
+			if(!urls[url]){
+				console.log("WARNING: url <"+url+"> from hashes not found in urls!");
+				return;
+			}
+			//var last=urls[url].timeline[urls[url].timeline.length-1];
+			//if(last==undefined||hashes[url][1]==null||hashes[url][1]!=last[1]){
+				urls[url].timeline.push(hashes[url]);
+			//}
+
+			if(timeout)clearTimeout(timeout);
+			timeout=setTimeout(function(){
+				persist.setItemSync("urls",urls);
+				console.log("(persisted after refresh)")
+			},2000);
+		}.bind(null,url));
+	}
+}
+
+
+module.exports=function(app,io,_moddir){
+	moddir=_moddir;
+	app.all(["/changes","/changes/*"],cmn.authgen());
+	app.get("/changes",function(req,res){
+		res.sendFile(moddir+"/changes.html");
+	});
+	app.get("/changes/urls",function(req,res){
+		var list=[];
+		var url;
+		for(url in urls)list.push(url);
+		res.send(JSON.stringify(list));
+	});
+	app.get("/changes/url",function(req,res){
+		var url=req.query.url;
+		if(!urls[url]){
+			res.status(404);
+			res.send("URL not found in watch list");
+			return;
+		}
+		res.send(JSON.stringify(urls[url]));
+	});
+	app.post("/changes/url",function(req,res){
+		var url=req.body;
+		urls[url]=new URLobject(url);
+		persist.setItemSync("urls",urls);
+		// refreshURLs();
+		res.send();
+	});
+	app.delete("/changes/url",function(req,res){
+		var url=req.body;
+		if(!urls[url]){
+			res.status(404);
+			res.send("URL not found in watch list");
+			return;
+		}
+		delete urls[url];
+		persist.setItemSync("urls",urls);
+		res.send();
+	});
+	app.post("/changes/refresh",function(req,res){
+		refreshURLs();
+		res.send();
+	});
+};
author	tomsmeding <tom.smeding@gmail.com>	2016-09-13 11:43:06 +0200
committer	tomsmeding <tom.smeding@gmail.com>	2016-09-13 11:44:11 +0200
commit	f00ba92ed2cc1a9c24ad783e83525d1b5a85b857 (patch)
tree	4d1c00a47c7f3842bcf3dece83d3c00ed3ae459f /modules/changes