From f00ba92ed2cc1a9c24ad783e83525d1b5a85b857 Mon Sep 17 00:00:00 2001 From: tomsmeding Date: Tue, 13 Sep 2016 11:43:06 +0200 Subject: Initial --- modules/changes/changes.html | 257 +++++++++++++++++++++++++++++++++++++++++++ modules/changes/changes.js | 163 +++++++++++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100644 modules/changes/changes.html create mode 100644 modules/changes/changes.js (limited to 'modules/changes') diff --git a/modules/changes/changes.html b/modules/changes/changes.html new file mode 100644 index 0000000..cda3da1 --- /dev/null +++ b/modules/changes/changes.html @@ -0,0 +1,257 @@ + + + + +Website change monitor + + + + +

Website change monitor

+URLs: + +
+

+ + +

+ + + +

Date	Hash

+ + diff --git a/modules/changes/changes.js b/modules/changes/changes.js new file mode 100644 index 0000000..6a8c8f2 --- /dev/null +++ b/modules/changes/changes.js @@ -0,0 +1,163 @@ +"use strict"; + +var cmn=require("../$common.js"), + persist=require("node-persist"), + crypto=require("crypto"), + http=require("http"), + https=require("https"), + URL=require("url"); + +var moddir=null; + +persist=persist.create({ + dir:"persist/changes", + continuous:false, + interval:false +}); +persist.initSync(); + +//urls: map(url => URLobject) +//url: String +//URLobject: {url, timeline: [[Date, hash]]} +var urls=persist.getItemSync("urls"); +if(!urls){ + urls={}; + persist.setItemSync("urls",urls); +} + + +function URLobject(url){ + if(!(this instanceof URLobject))return new URLobject(url); + this.url=url; + this.timeline=[]; +} + +function fetch(method,url,data/*?*/,cb){ + var cbcalled=false; + + if(!cb){ + cb=data; + data=undefined; + if(!cb)throw new Error("No callback passed to fetch"); + } + + try { + url=URL.parse(url); + } catch(e){ + cb(-1,null); + return; + } + + var httpclass; + switch(url.protocol){ + case "http:": httpclass=http; break; + case "https:": httpclass=https; break; + default: + cb(-1,null); + return; + } + + url.method=method; + + var req=httpclass.request(url,function(res){ + var body=""; + res.on("data",function(data){ + body+=data; + }); + res.on("end",function(){ + if(!cbcalled)cb(res.statusCode,body); + cbcalled=true; + }); + res.on("error",function(err){ + if(!cbcalled)cb(-1,err); + cbcalled=true; + }); + }); + req.on("error",function(err){ + if(!cbcalled)cb(-1,err); + cbcalled=true; + }); + if(data)req.write(data); + req.end(); +} + +function refreshURLs(){ + var hashes={}; + var i; + var url; + var timeout=null; + for(url in urls){ + console.log("Fetching <"+url+">"); + fetch("GET",url,function(url,status,body){ + console.log("Got <"+url+">; status = "+status); + if(status==-1){ + hashes[url]=[new Date(),null]; + } else { + var hash=crypto.createHash("sha256"); + hash.update(body); + hashes[url]=[new Date(),hash.digest("hex")]; + } + + if(!urls[url]){ + console.log("WARNING: url <"+url+"> from hashes not found in urls!"); + return; + } + //var last=urls[url].timeline[urls[url].timeline.length-1]; + //if(last==undefined||hashes[url][1]==null||hashes[url][1]!=last[1]){ + urls[url].timeline.push(hashes[url]); + //} + + if(timeout)clearTimeout(timeout); + timeout=setTimeout(function(){ + persist.setItemSync("urls",urls); + console.log("(persisted after refresh)") + },2000); + }.bind(null,url)); + } +} + + +module.exports=function(app,io,_moddir){ + moddir=_moddir; + app.all(["/changes","/changes/*"],cmn.authgen()); + app.get("/changes",function(req,res){ + res.sendFile(moddir+"/changes.html"); + }); + app.get("/changes/urls",function(req,res){ + var list=[]; + var url; + for(url in urls)list.push(url); + res.send(JSON.stringify(list)); + }); + app.get("/changes/url",function(req,res){ + var url=req.query.url; + if(!urls[url]){ + res.status(404); + res.send("URL not found in watch list"); + return; + } + res.send(JSON.stringify(urls[url])); + }); + app.post("/changes/url",function(req,res){ + var url=req.body; + urls[url]=new URLobject(url); + persist.setItemSync("urls",urls); + // refreshURLs(); + res.send(); + }); + app.delete("/changes/url",function(req,res){ + var url=req.body; + if(!urls[url]){ + res.status(404); + res.send("URL not found in watch list"); + return; + } + delete urls[url]; + persist.setItemSync("urls",urls); + res.send(); + }); + app.post("/changes/refresh",function(req,res){ + refreshURLs(); + res.send(); + }); +}; -- cgit v1.2.3-70-g09d2