commit f8bd82f33d38446b77183fd67d36f907d6c8e04c
Author: x0x7 <x0x7@tutanota.com>
Date:   Fri Jul 26 18:47:43 2024 -0515

    Added files

diff --git a/data.js b/data.js
new file mode 100644
index 0000000..39fdb52
--- /dev/null
+++ b/data.js
@@ -0,0 +1,92 @@
+var exports = module.exports;
+//var {Database} = require('sqlite3');
+//var sqldb = new Database(_data+'/data.db');
+var {Client} = require('cassandra-driver');
+const cass = new Client({
+ cloud: {secureConnectBundle:'secure-connect-todo.zip'},
+ credentials: {
+  username:'',
+  password:''
+ },
+});
+
+async function startcass() {
+ await cass.connect();
+ cass.execute('use todo');
+}
+startcass();
+
+(()=>{
+ var rankCache = new Map();
+ var rankReadTimeouts = new Map();
+ var rankWriteTimeouts = new Map();
+ var rankReadBounce = 1000*60*5;
+ var rankWriteBounce = 1000*15;
+ function rawgetrank(user,cb) {
+  cass.execute('Select json from rank where user=?',[user]).then(result=>{
+   if(!result || !result.rows || !result.rows.length) return cb(null,{});
+   cb(null,JSON.parse(result.rows[0].json));
+  }).catch(cb);
+ }
+ 
+ function updaterankreadtimeout(user) {
+  if(rankReadTimeouts.has(user)) clearTimeout(rankReadTimeouts.get(user));
+  rankReadTimeouts.set(user,setTimeout(()=>rankCache.delete(user),rankReadBounce));
+ }
+ 
+ function getrank(user,cb) {
+  updaterankreadtimeout(user);
+  if(rankCache.has(user)) return cb(null,rankCache.get(user));
+  rawgetrank(user,(err,obj)=>{
+   if(err) return cb(err);
+   rankCache.set(user,obj);
+   return cb(null,obj);
+  });
+ }
+ 
+ function rawwriterank(user,obj) {
+  cass.execute('Insert into rank (user,json) values (?,?)',[user,JSON.stringify(obj)]);
+ }
+ 
+ function writerank(user,obj) {
+  if(rankWriteTimeouts.has(user)) clearTimeout(rankWriteTimeouts.get(user));
+  rankWriteTimeouts.set(user,setTimeout(()=>rawwriterank(user,obj),rankWriteBounce));
+ }
+ exports.getrank=getrank;
+ exports.writerank=writerank;
+})();
+
+(()=>{
+ function rawgettitle(user,id,cb) {
+  cass.execute('Select title from titles where user=? and id=?',[user,id]).then(result=>{
+   if(!result || !result.rows || !result.rows.length) return cb('title not found');
+   cb(null,result.rows[0]);
+  }).catch(cb);
+ }
+ function rawwritetitle(user,id,title) {
+  cass.execute('Insert into titles (user,id,title) values (?,?,?)',[user.id,title]);
+ }
+ exports.gettitle=rawgettitle;
+ exports.writetitle=rawwritetitle;
+})();
+
+(()=>{
+ function rawgetday(user,day,cb) {
+  cass.execute('Select json from day where user=? and day=?',[user,day]).then(result=>{
+   if(!result || !result.rows || !result.rows.length) return cb(null,[]);
+   cb(null,JSON.parse(result.rows[0].json));
+  }).catch(cb);
+ }
+ function rawwriteday(user,day,list) {
+  cass.execute('Insert into day (user,day,json) values (?,?,?)',[user,day,JSON.stringify(list)]);
+ }
+ exports.getday=rawgetday;
+ exports.writeday=rawwriteday;
+})();
+
+(()=>{
+ function rawaddlog(user,title,id) {
+  cass.execute('Insert into addjournal (user,time,title,id) values (?,?,?,?)',[user,Date.now(),title,id]);
+ }
+ exports.addlogwrite=rawaddlog;
+})();
diff --git a/data2.js b/data2.js
new file mode 100644
index 0000000..24eb2ad
--- /dev/null
+++ b/data2.js
@@ -0,0 +1,93 @@
+
+var readwritecache2 = require('../readwritecache2');
+var fs=require('fs');
+var _data = __dirname+'/data';
+var sqlitedb = require('./sqlitedb');
+var sqldb = sqlitedb;
+
+//var {Client} = require('cassandra-driver');
+//const cass = new Client({
+// cloud: {secureConnectBundle:'secure-connect-todo.zip'},
+// credentials: {
+//  username:'jvonmitchell@gmail.com',
+//  password:'6e9hDX8$$'
+// },
+//});
+//
+//async function startcass() {
+// await cass.connect();
+// cass.execute('use todo');
+//}
+//startcass();
+
+module.exports.ranks=readwritecache2({
+ default:{},
+ writefreq:1000*30,
+ readfreq:1000*60*5,
+ delayclearonread:true,
+ delaywriteonwrite:true,
+ get:(key,cb)=>{
+  console.log('Reading ranks for',key);
+  sqlitedb.get("SELECT json FROM rank WHERE user=?",key,(err,json)=>{
+   if(err) return cb(err);
+   if(!json) return cb();
+   try {
+    cb(null,JSON.parse(json.json));
+   }
+   catch(e) {
+    cb(e);
+   }
+  });
+ },
+ write:(key,obj,notindb,cb)=>{
+  //console.log('Writing ranks for',key);
+  if(notindb) return sqlitedb.run('INSERT INTO rank (user,json) VALUES (?,?)',key,JSON.stringify(obj),cb);
+  sqlitedb.run('UPDATE rank SET json=? WHERE user=?',JSON.stringify(obj),key,cb);
+ }
+});
+
+module.exports.journals=readwritecache2({
+ default:{},
+ get:(key,cb)=>{
+  var user = key.split('.')[0];
+  var date=key.split('.day.')[1].replace('.json','');
+  //var date=key.toString();
+  sqldb.get('Select json from day where user=? and day=?',user,date,(err,result)=>{
+   if(err) return cb(err);
+   if(result) return cb(null,JSON.parse(result.json));
+   //cass.execute('Select json from day where user=? and day=?',[user,date]).then(result=>{
+    //if(result && result.rows && result.rows.length) {
+    // //console.log({result});
+    // return cb(null,JSON.parse(result.rows[0].json));
+    //}
+    fs.readFile(_data+'/'+key+'.json',(err,buff)=>{
+     if(err) return cb();
+     try {
+      cb(null,JSON.parse(buff));
+     }
+     catch(e) {
+      console.log({key,buff:buff.toString(),err:e,cb});
+      cb(e);
+     }
+    });
+   //});
+  });
+ },
+ write:(key,obj,notindb,cb)=>{
+  var user = key.split('.')[0];
+  var day = key.split('.day.')[1];
+  var stringified=JSON.stringify(obj);
+  //cass.execute('Insert into day (user,day,json) values (?,?,?)',[user,day,stringified]);
+  fs.writeFile(_data+'/'+key+'.json',stringified,cb);
+  if(notindb) {
+   sqldb.run('INSERT into day (user,day,json) values (?,?,?)',user,day,stringified);
+  }
+  else {
+   sqldb.run('UPDATE day SET json=? WHERE user=? AND day=?',stringified,user,day);
+  }
+ },
+ writefreq:1000*30,
+ readfreq:1000*60*5,
+ delayclearonread:true,
+ delaywriteonwrite:true
+});
diff --git a/easystart.js b/easystart.js
new file mode 100755
index 0000000..bff9736
--- /dev/null
+++ b/easystart.js
@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+
+var repl = require('repl');
+var user = process.argv[2]||'me';
+var todo=require('./nice')(user);
+var context=repl.start(user+'> ').context;
+
+context.todo=todo;
+Object.keys(todo).forEach(k=>context[k]=todo[k]);
diff --git a/nice.js b/nice.js
new file mode 100644
index 0000000..48cb0d9
--- /dev/null
+++ b/nice.js
@@ -0,0 +1,330 @@
+module.exports=exports;
+var todo=require('./todo');
+var {pullexposedtasks,randomtasks,edittitle,searchtasks,pullmirrorsuper,pullmirrortask,getjournal,decayranks,completetask,pulltasks,addtask,removetask,ranktasks,removetasks,addmany}=todo;
+Object.keys(todo).forEach(k=>exports[k]=todo[k]);
+var prompt=require('prompt-sync')();
+var fs=require('fs');
+var async=require('async');
+
+function noop() {}
+
+function exports(user) {
+ var retr={};
+ retr.exports=exports;
+ retr.random=random;
+ retr.rand=random;
+ function random(count=5) {
+  randomtasks(user,count,(err,res)=>{
+   console.log();
+   res.forEach(i=>console.log(i.id,i.title));
+  });
+ }
+ retr.add=add;
+ function add() {
+  if(arguments.length==0) {
+   var list=[];
+   function ask() {
+    var line=prompt(':');
+    if(line) {
+     list.push(line);
+     ask();
+    }
+    else { //Add list
+     addmany(user,list,console.log);
+    }
+    //if(line) addtask(user,line,ask);
+   }
+   return ask();
+  }
+  Array.prototype.forEach.bind(arguments)(item=>{
+   if(typeof(item)==='string') addtask(user,item,noop);
+   else { //Asuming list
+    item.forEach(item=>addtask(user,item,noop));
+   }
+  });
+ }
+ retr.remove=remove;
+ function remove() {
+  Array.prototype.forEach.bind(arguments)(item=>{
+   if(typeof(item)==='number') {
+    if(item<=4) {
+     var verify = prompt('Verify: Y/n');
+     if(verify[0]=='n'||verify[0]=='N') return;
+    }
+    removetask(user,item,noop);
+   }
+   else { //Asuming list
+    item.forEach(item=>removetask(user,item,noop));
+   }
+  });
+ }
+ function printtasks(tasks,stream) {
+  pastpull=tasks;
+  try {
+   tasks.sort(()=>Math.random()-0.5);
+   stream.write('\n');
+   //console.log({tasks});
+   tasks.forEach((t,i)=>{
+    //console.log(t.id+':',i,t.title);
+    stream.write(t.id+': '+i+' '+t.title+'\n');
+   });
+  }
+  catch(e) {
+   console.log({user,tasks});
+   console.log(e);
+  }
+ }
+ retr.pullexposed = pullexposed;
+ function pullexposed(count=4,stream=process.stdout) {
+  pullexposedtasks(user,count,(err,tasks)=>{
+   if(err) return console.log(err,tasks);
+   printtasks(tasks,stream);
+  });
+ }
+ var pastpull=[];
+ retr.pull=pull;
+ retr.p = pull;
+ function pull(count=4,stream=process.stdout) {
+  pulltasks(user,count,(err,tasks)=>{
+   if(err) return console.log(err,tasks);
+   printtasks(tasks,stream);
+  });
+ }
+ retr.r=r;
+ var pulllog=fs.createWriteStream(__dirname+'/data/logs/pull.log',{flags:'a'});
+ //function r() {
+ // if(arguments.length===1 && typeof(arguments[0])==='string') {
+ //  return retr.done(arguments[0]);
+ // }
+ // var expanded=[];
+ // Array.prototype.forEach.bind(arguments)(item=>{
+ //  if(typeof(item)==='number') expanded.push(item);
+ //  else { //Asuming list
+ //   expanded=expanded.concat(item);
+ //  }
+ // });
+ // var toptitle = pastpull[expanded[0]].title;
+ // expanded=expanded.map(i=>pastpull[i].id);
+ // //console.log(expanded)
+ // ranktasks(user,expanded,noop);
+ // pulllog.write(toptitle+'\n');
+ // pastrank=expanded;
+ //}
+ function r() {
+  if (arguments.length === 1 && typeof(arguments[0]) === 'string') {
+   return retr.done(arguments[0]);
+  }
+  var expanded = [];
+  Array.prototype.forEach.call(arguments, item => {
+   if (typeof(item) === 'number') {
+    expanded.push([item]); // Wrap single numbers in an array
+   } else if (Array.isArray(item)) {
+    expanded.push(item); // Use array as is
+   }
+  });
+  var toptitle = pastpull[expanded[0][0]].title; // Use the first element of the first array to get the title
+  expanded = expanded.map(group => group.map(i => pastpull[i].id)); // Replace temp ids with longhand ids
+  ranktasks(user, expanded, noop);
+  pulllog.write(toptitle + '\n');
+  pastrank = expanded.flat();
+ }
+ retr.addfromfile=addfromfile;
+ function addfromfile(path='insert') {
+  fs.readFile(path,(err,buff)=>{
+   if(err) return console.log(err);  
+   addmany(user,buff.toString().split('\n').filter(i=>i.length),console.log);
+  });
+ }
+ retr.done=done;
+ function done(taskid) {
+  if(taskid<=4) {
+   var verify = prompt('Verify: Y/n');
+   if(verify[0]=='n'||verify[0]=='N') return;
+  }
+  completetask(user,taskid,err=>{
+   if(err) return console.log(err);
+   decayranks(user,1-5/300,noop);
+   //if(typeof(taskid)=='number') setTimeout(pm,100);
+  });
+ }
+ retr.d=d;
+ function d() {
+  done(pastrank[0]);
+ }
+ var pastrank=[];
+ retr.rank=rank;
+ //function rank() {
+ // var expanded=[];
+ // Array.prototype.forEach.bind(arguments)(item=>{
+ //  if(typeof(item)==='number') expanded.push(item);
+ //  else { //Asuming list
+ //   expanded=expanded.concat(item);
+ //  }
+ // });
+ // pastrank=expanded;
+ // ranktasks(user,expanded,noop);
+ //}
+ function rank() {
+  var expanded = [];
+  if (arguments.length === 1 && Array.isArray(arguments[0])) {
+   expanded = arguments[0]; // Treat single array argument as the complete taskGroups
+  } else {
+   // Treat each argument as an element to be passed to ranktasks
+   Array.prototype.forEach.call(arguments, item => {
+    if (typeof(item) === 'number') {
+     expanded.push([item]); // Wrap single numbers in an array
+    } else {
+     expanded.push(item); // Already an array, use as is
+    }
+   });
+  }
+  pastrank = expanded.flat();
+  ranktasks(user, expanded, noop);
+ }
+ retr.pullmirror=pullmirror;
+ retr.pm=pullmirror;
+ retr.pullmirrormonth=pullmirrormonth;
+ retr.pullmirrorweek=pullmirrorweek;
+ retr.pullmirrorday=pullmirrorday;
+ function pullmirror() {
+  //async.parallel([pullmirrormonth,pullmirrorweek,pullmirrorday],(err,res)=>{
+  async.map([doublepullmirrormonth,pullmirrormonth,doublepullmirrorweek,pullmirrorweek,doublepullmirrorday,pullmirrorday],(func,cb)=>{
+   func((err,res)=>{
+    if(err == 'Journal missing - skiplimit reached') return cb();
+    cb(err,res);
+   });
+  },(err,res)=>{
+   if(err) return console.log(err);
+   res=res.filter(r=>r);
+   res.forEach(item=>{
+    if(!item) return console.log('missing');
+    console.log((new Date(item.time)).toLocaleString(),item.title,item.id||'');
+   });
+  });
+ }
+ function pullmirrorday(cb) {
+  pullmirrortask(user,cb);
+ }
+ function doublepullmirrorday(cb) {
+  getjournal(user,0,(err,journal)=>{
+   if(err) return console.log(err);
+   if(!journal.list) return cb();
+   pullmirrorsuper(user,1,journal.list.length*2,5,cb);
+  });
+ }
+ retr.lookback=lookback;
+ function lookback(days) {
+  getjournal(user,days,(err,journal)=>{
+   if(err) return console.log(err);
+   if(!journal.list) return console.log('Journal missing');
+   journal.list.forEach(item=>{
+    console.log((new Date(item.time)).toLocaleString(),item.title,item.id||'');
+   });
+  });
+ }
+ retr.pullmirrorweek=pullmirrorweek;
+ function pullmirrorweek(cb) {
+  var day=(new Date()).getDay()||7;
+  var pending=day;
+  var lookcount=0;
+  for(var i=0;i<day;++i) {
+   getjournal(user,i,(err,journal)=>{
+    --pending;
+    if(journal && journal.list) lookcount+=journal.list.length;
+    if(!pending) {
+     pullmirrorsuper(user,day,lookcount,5,cb);
+    }
+   });
+  }
+ }
+ retr.doublepullmirrorweek=doublepullmirrorweek;
+ function doublepullmirrorweek(cb) {
+  var day=(new Date()).getDay()||7;
+  var pending=day;
+  var lookcount=0;
+  for(var i=0;i<day;++i) {
+   getjournal(user,i,(err,journal)=>{
+    --pending;
+    if(journal && journal.list) lookcount+=journal.list.length;
+    if(!pending) {
+     pullmirrorsuper(user,day,lookcount*2,5,cb);
+    }
+   });
+  }
+ }
+ retr.pullmirrormonth=pullmirrormonth;
+ function pullmirrormonth(cb) {
+  var day=(new Date()).getDate()||30;
+  var pending=day;
+  var lookcount=0;
+  for(var i=0;i<day;++i) {
+   getjournal(user,i,(err,journal)=>{
+    --pending;
+    if(journal && journal.list) lookcount+=journal.list.length;
+    if(!pending) {
+     pullmirrorsuper(user,day,lookcount,15,cb);
+    }
+   });
+  }
+ }
+ retr.doublepullmirrormonth=doublepullmirrormonth;
+ function doublepullmirrormonth(cb) {
+  var day=(new Date()).getDate()||30;
+  var pending=day;
+  var lookcount=0;
+  for(var i=0;i<day;++i) {
+   getjournal(user,i,(err,journal)=>{
+    --pending;
+    if(journal && journal.list) lookcount+=journal.list.length;
+    if(!pending) {
+     pullmirrorsuper(user,day,lookcount*2,15,cb);
+    }
+   });
+  }
+ }
+ function lookup(ref) {
+  if(!ref) return;
+  if(typeof(ref)=='number') return lookupid(ref);
+  else return lookupname(ref);
+ }
+ retr.search=search;
+ function search(titlefrag) {
+  var args=[];
+  for(var i=0;arguments[i];++i) {
+   args=args.concat(arguments[i]);
+  }
+  searchtasks(user,args,console.log);
+ }
+ retr.searchres=searchres;
+ function searchres(titlefrag) {
+  var args=[];
+  for(var i=0;arguments[i];++i) {
+   args=args.concat(arguments[i]);
+  }
+  return new Promise((resolve,reject)=>{
+   searchtasks(user,args,(err,results)=>{
+    if(err) return reject(err);
+    resolve(results);
+   });
+  });
+ }
+ retr.edit = edit;
+ function edit(id,title) {
+  edittitle(user,id,title);
+ }
+ /*
+ http.createServer((req,res)=>{
+  if(req.url=='/pull') {
+   res.writeHead(200,{'Content-type':'text/plain'});
+   pulltasks(user,5,(err,tasks)=>{
+    tasks.forEach((t,i)=>{
+     //console.log(t.id+':',i,t.title);
+     res.write(t.id+': '+i+' '+t.title+'\n');
+    });
+    res.end();
+   });
+  }
+ }).listen(7000);
+ */
+ return retr;
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..683206f
--- /dev/null
+++ b/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "todo2",
+  "version": "1.0.0",
+  "description": "",
+  "main": ".eslintrc.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node server.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "prompt-sync": "^4.2.0",
+    "sqlite3": "^5.1.4"
+  }
+}
diff --git a/server.js b/server.js
new file mode 100755
index 0000000..4963f6e
--- /dev/null
+++ b/server.js
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+var async = require('async');
+var UrlResolver = require('urlresolver');
+var p = new UrlResolver();
+var user = 'me';
+exports.p = p;
+
+var todo=require('./todo');
+var {pulltasks,addtask,removetask,ranktasks,removetasks}=todo;
+Object.keys(todo).forEach(k=>exports[k]=todo[k]);
+
+exports.de=de;
+function de(res,err,code) {
+ res.writeHead(code||400,{"Content-Type":'text/plain'});
+ res.end(err.toString());
+}
+
+exports.simpcb=simpcb;
+function simpcb(res) {
+ return (err,data)=>{
+  if(err) return de(res,err);
+  if(data) return res.end(data.toString());
+  res.end('success');
+ };
+}
+
+
+p.add('/pull',(req,res)=>{
+ pulltasks(user,5,(err,tasks)=>{
+  if(err) return cb(err);
+  res.writeHead(200,{"Content-Type":'text/plain'});
+  res.end(tasks.map((i,ind)=>`${i.id}: ${ind} ${i.title}`).join('\n'));
+ });
+});
+
+
+p.add('/api/pull',(req,res)=>{
+ //Replace with req.user
+ pulltasks(user,5,(err,tasks)=>{
+  if(err) return cb(err);
+  res.writeHead(200,{"Content-Type":'application/json'});
+  res.end(JSON.stringify(tasks));
+ });
+});
+
+
+p.add('/add/task/',(req,res)=>{
+ addtask(req.user,req.post.string,simpcb(res));
+});
+
+
+p.add('/remove/task',(req,res)=>{
+ var taskid=parseInt(req.post.string);
+ if(isNaN(taskid)) return de(res,"Not a number");
+ removetask(req.user,taskid,simpcb(res));
+});
+
+
+p.add('/rank/tasks',(req,res)=>{
+ var tasks=req.post.string.split(',').map(i=>parseInt(i)).filter(i=>!isNaN(i));
+ ranktasks(req.user,tasks,simpcb(res));
+});
+
+
+p.add('/rank/and/remove/tasks',(req,res)=>{
+ var [rank,remove]=req.post.string.split('|').map(list=>list.split(',').map(i=>parseInt(i)).filter(i=>!isNaN(i)));
+ async.parallel([
+  cb=>ranktasks(req.user,rank,cb),
+  cb=>removetasks(req.user,remove,cb)
+ ],simpcb(res));
+});
+
+function exports(req,res) {
+ p.get(req.url)(req,res);
+}
+
+require('http').createServer(exports).listen(8092);
diff --git a/setupdb.js b/setupdb.js
new file mode 100755
index 0000000..32f9dc9
--- /dev/null
+++ b/setupdb.js
@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+/*
+  sqlitedb.get("SELECT json FROM rank WHERE user=?",key,(err,json)=>{
+  if(notindb) return sqlitedb.run('INSERT INTO rank (user,json) VALUES (?,?)',key,JSON.stringify(obj),cb);
+  sqlitedb.run('UPDATE rank SET json=? WHERE user=?',JSON.stringify(obj),key,cb);
+*/
+
+var db = require('./sqlitedb');
+
+db.run('CREATE TABLE rank (user text primary key,json text)',console.log);
+
+/*
+todo.js:  async.map(ids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
+todo.js:  sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
+todo.js:   sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
+todo.js:  sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,result)=>{
+todo.js:   sqldb.run('Delete from titles where user=? and id=?',user,taskid,err=>{
+todo.js:   sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,titleobj)=>{
+todo.js: sqldb.all(`Select id,title from titles where user=? and ${titlefrags}`,user,cb);
+todo.js: sqldb.run('Update titles Set title=? Where user=? and id=?',title,user,id,cb);
+todo.js:  async.map(selectedids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
+*/
+
+db.run('CREATE TABLE titles (user text,id integer,title text)');
+db.run('CREATE INDEX titles_idx ON titles(user,id)');
+
+/*
+data2.js:  sqldb.get('Select json from day where user=? and day=?',user,date,(err,result)=>{
+data2.js:   sqldb.run('INSERT into day (user,day,json) values (?,?,?)',user,day,stringified);
+data2.js:   sqldb.run('UPDATE day SET json=? WHERE user=? AND day=?',stringified,user,day);
+*/
+
+CREATE TABLE day (user text not null,day text not null,json text not null);
+CREATE UNIQUE INDEX day_idx ON day(user,day);
diff --git a/sqlitedb.js b/sqlitedb.js
new file mode 100644
index 0000000..4741741
--- /dev/null
+++ b/sqlitedb.js
@@ -0,0 +1,4 @@
+
+
+var sqlite3 = require('sqlite3');
+module.exports  = new sqlite3.Database(__dirname+'/data/data.db');
diff --git a/test.data.js b/test.data.js
new file mode 100644
index 0000000..9cc4e45
--- /dev/null
+++ b/test.data.js
@@ -0,0 +1,8 @@
+
+function test1() {
+ var a = require('./data');
+ setTimeout(()=>{
+  a.getrank('test',console.log);
+ },1000);
+}
+test1();
diff --git a/test.js b/test.js
new file mode 100644
index 0000000..103f068
--- /dev/null
+++ b/test.js
@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+
+var a = require('./server');
+
+a.pulltasks('test',5,console.log);
diff --git a/todo.js b/todo.js
new file mode 100644
index 0000000..4456f8b
--- /dev/null
+++ b/todo.js
@@ -0,0 +1,475 @@
+var exports=module.exports;
+var _data = __dirname+'/data';
+exports._data=_data;
+
+var fs=require('fs');
+var {shuffle} = require('lodash');
+var async = require('async');
+var df = require('../df');
+var {ranks,journals} = require('./data2');
+
+df((err,kb)=>{
+ if(err) return console.log('df err',err);
+ if(kb<1000) process.exit(1);
+});
+
+var db = require('../readwritecache')(dbwrite,dbwrite,dbread,{},1000,1);
+//var {Database} = require('sqlite3');
+var sqldb = require('./sqlitedb'); 
+//var {Client} = require('cassandra-driver');
+//const cass = new Client({
+// cloud: {secureConnectBundle:'secure-connect-todo.zip'},
+// credentials: {
+//  username:'jvonmitchell@gmail.com',
+//  password:'6e9hDX8$$'
+// },
+//});
+
+//async function startcass() {
+// await cass.connect();
+// cass.execute('use todo');
+//}
+//startcass();
+
+
+exports.db=db;
+
+exports.dbwrite=dbwrite;
+function dbwrite(key,obj,cb) {
+ //console.log("Writing",key);
+ var stringified=JSON.stringify(obj);
+ var user = key.split('.')[0];
+ if(key.endsWith('.rank')) {
+  sqldb.run('Update rank set json=? where user=?',stringified,user,noop);
+  cass.execute('Insert into rank (user,json) values (?,?)',[user,stringified]);
+ }
+ if(key.indexOf('.day.')!=-1) {
+  var day = key.split('.day.')[1];
+  cass.execute('Insert into day (user,day,json) values (?,?,?)',[user,day,stringified]);
+ }
+ fs.writeFile(_data+'/'+key+'.json',stringified,cb);
+}
+
+var rankCache = new Map();
+var rankCacheTimeouts = new Map();
+exports.dbread=dbread;
+function dbread(key,cb) {
+ var user = key.split('.')[0];
+ if(key.endsWith('.rank')) {
+  if(rankCache.has(user)) {
+   clearTimeout(rankCacheTimeouts.get(user));
+   rankCacheTimeouts.set(user,setTimeout(()=>rankCache.delete(user)));
+   return cb(null,rankCache.get(user));
+  }
+  return sqldb.get('Select json from rank where user=?',user,(err,json)=>{
+   if(err) return cb(err);
+   //if(!json) return cb('User not found in rank');
+   if(!json) return cb();
+   json=JSON.parse(json.json);
+   rankCache.set(user,json);
+   rankCacheTimeouts.set(user,setTimeout(()=>rankCache.delete(user)));
+   cb(null,json);
+  });
+ }
+ if(key.indexOf('.day.')!=-1) {
+  //var indexOf = key.indexOf('.day.');
+  //console.log({key,indexOf});
+  var date=key.split('.day.')[1].replace('.json','');
+  return cass.execute('Select json from day where user=? and day=?',[user,date]).then(result=>{
+   if(result && result.rows && result.rows.length) {
+    //console.log({result});
+    return cb(null,JSON.parse(result.rows[0].json));
+   }
+   fs.readFile(_data+'/'+key+'.json',(err,buff)=>{
+    if(err) return cb();
+    try {
+     cb(null,JSON.parse(buff));
+    }
+    catch(e) {
+     console.log({key,buff:buff.toString(),err:e,cb});
+     cb(e);
+    }
+   });
+  });
+ }
+ fs.readFile(_data+'/'+key+'.json',(err,buff)=>{
+  if(err) return cb();
+  try {
+   cb(null,JSON.parse(buff));
+  }
+  catch(e) {
+   console.log({key,buff:buff.toString(),err:e,cb});
+   cb(e);
+  }
+ });
+}
+
+
+exports.minindexes=minindexes;
+function minindexes(list) {
+ if(!list.length) return [];
+ var minvalue=list.reduce((a,b)=>a<b?a:b);
+ var retr=[];
+ for(var i=0;i<list.length;++i) {
+  if(list[i]==minvalue) retr.push(i);
+ }
+ //console.log({minvalue,idexes:retr});
+ return retr;
+}
+
+
+
+
+
+
+exports.removetasks=removetasks;
+function removetasks(user,tasks,cb) {
+ async.map(tasks,(task,cb)=>removetask(user,cb),cb);
+}
+
+function noop () {}
+
+exports.addtask=addtask;
+function addtask(user,tasktext,cb) {
+ //var rankkey=user+'.rank';
+ var addhistkey=user+'.addhist';
+ //async.map([rankkey,addhistkey],db.get,(err,objs)=>{
+ async.parallel([
+  cb=>ranks.get(user,cb),
+  cb=>db.get(addhistkey,cb)
+ ],(err,objs)=>{
+  if(err) return cb(err);
+  var [rankobj,addhistobj]=objs;
+  var i=0;
+  while(rankobj[i]) ++i;
+  sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
+  rankobj[i]={};
+  if(!addhistobj.hist) addhistobj.hist=[];
+  addhistobj.hist.push(tasktext);
+  //console.log({rankkey,rankobj,i,tasktext});
+  //db.markWrite(rankkey);
+  ranks.markWrite(user);
+  db.markWrite(addhistkey);
+  cb(null,i);
+ });
+}
+
+exports.addmany=addmany;
+function addmany(user,tasktexts,cb) {
+ //var rankkey=user+'.rank';
+ //db.get(rankkey,(err,rankobj)=>{
+ ranks.get(user,(err,rankobj)=>{
+  if(err) return cb(err);
+  var retr=[];
+  var i=0;
+  tasktexts.forEach(tasktext=>{
+   if(!tasktext) return;
+   while(rankobj[i]) ++i;
+   retr.push(i);
+   //sqldb.run('BEGIN TRANSACTION',noop);
+   sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
+   //sqldb.run('COMMIT',noop);
+   rankobj[i]={};
+  });
+  //db.markWrite(rankkey);
+  ranks.markWrite(user);
+  cb(null,retr);
+ });
+}
+
+exports.removetask=removetask;
+function removetask(user,taskid,cb) {
+ //var rankkey=user+'.rank';
+ //db.get(rankkey,(err,rankobj)=>{
+ ranks.get(user,(err,rankobj)=>{
+  if(err) return cb(err);
+  sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,result)=>{
+   if(err) return cb(err);
+   if(result) console.log('Removed:',result.title);
+   sqldb.run('Delete from titles where user=? and id=?',user,taskid,err=>{
+    if(err) return cb(err);
+    console.log('removing',taskid);
+    delete rankobj[taskid];
+    Object.values(rankobj).forEach(peerobj=>{
+     delete peerobj[taskid];
+    });
+    //db.markWrite(rankkey);
+    ranks.markWrite(user);
+    cb();
+   });
+  });
+ });
+}
+
+exports.completetask=completetask;
+function completetask(user,taskid,cb) {
+ var now=Date.now();
+ var journalkey=user+'.day.'+(new Date()).toLocaleDateString().replace(/\//g,'.');
+ //db.get(journalkey,(err,journalobj)=>{
+ journals.get(journalkey,(err,journalobj)=>{
+  if(err) return cb(err);
+  var title;
+  journalobj.list=journalobj.list||[];
+  if(typeof(taskid)==='string') {
+   title=taskid;
+   journalobj.list.push({time:now,title});
+   journals.markWrite(journalkey);
+   //db.markWrite(journalkey);
+  }
+  else {
+   sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,titleobj)=>{
+    if(err) return cb(err);
+    title=titleobj.title;
+    journalobj.list.push({time:now,title,id:taskid});
+    removetask(user,taskid,cb);
+    journals.markWrite(journalkey);
+    //db.markWrite(journalkey);
+   });
+  }
+ });
+}
+
+exports.undercount=undercount;
+function undercount(id,obj,blockset) {
+ if(blockset.has(id)) return 1000;
+ var count=0;
+ var keys=Object.keys(obj);
+ if(!keys.length) return 0;
+ for(var i=0;i<keys.length;++i) {
+  if(blockset.has(keys[i])) continue;
+  if(obj[keys[i]]<0) count-=obj[keys[i]];
+ }
+ //console.log({id,count});
+ return count;
+}
+
+
+exports.decayranks=decayranks;
+const DECAY=5/300;
+function decayranks(user,decay,cb) {
+ //var key=user+'.rank';
+ decay=decay||DECAY;
+ //db.get(key,(err,obj)=>{
+ ranks.get(user,(err,obj)=>{
+  if(err) return cb(err);
+  Object.values(obj).forEach(subobj=>{
+   Object.keys(subobj).forEach(subkey=>{
+    if(subkey=='block') return;
+    subobj[subkey]*=decay;
+   });
+  });
+  //db.markWrite(key);
+  ranks.markWrite(user);
+  cb();
+ });
+}
+
+exports.getjournal=getjournal;
+function getjournal(user,days,cb) {
+ days=days||0;
+ days=Math.abs(days);
+ var journalkey=user+'.day.'+(new Date(Date.now()-1000*60*60*24*days)).toLocaleDateString().replace(/\//g,'.');
+ //db.get(journalkey,cb);
+ journals.get(journalkey,cb);
+}
+
+//exports.ranktasks=ranktasks;
+//function ranktasks(user,tasklist,cb) {
+// //var key=user+'.rank';
+// //db.get(key,(err,obj)=>{
+// ranks.get(user,(err,obj)=>{
+//  if(err) return cb(err);
+//  var i;
+//  for(i=0;i<tasklist.length;++i) {
+//   for(var j=i+1;j<tasklist.length;++j) {
+//    obj[tasklist[i]][tasklist[j]]=1;
+//    obj[tasklist[j]][tasklist[i]]=-1;
+//   }
+//  }
+//  for(i=1;i<tasklist.length;++i) {
+//   obj[tasklist[i]].block=i; //We don't want to see it for a bit, add a block that will count down
+//   //console.log("Block",tasklist[i],i);
+//  }
+//  //db.markWrite(key);
+//  ranks.markWrite(user);
+//  cb();
+// });
+//}
+
+exports.ranktasks=ranktasks;
+function ranktasks(user,taskGroups,cb) {
+ ranks.get(user,(err,obj)=>{
+  if(err) return cb(err);
+  var i;
+  for(i=0;i<taskGroups.length;++i) {
+   for(var j=i+1;j<taskGroups.length;++j) {
+    for(var taskA of [].concat(taskGroups[i])) {
+     for(var taskB of [].concat(taskGroups[j])) {
+      obj[taskA][taskB]=1;
+      obj[taskB][taskA]=-1;
+     }
+    }
+   }
+  }
+  for (let i = 1; i < taskGroups.length; ++i) {
+   [].concat(taskGroups[i]).forEach(task => obj[task].block = i);
+  }
+  ranks.markWrite(user);
+  cb();
+ });
+}
+
+exports.searchtasks=searchtasks;
+function searchtasks(user,titlefrags,cb) {
+ //This is not safe
+ if(typeof(titlefrags)=='string') titlefrags = [titlefrags];
+ titlefrags = titlefrags.map(i=>i.toLowerCase());
+ titlefrags = titlefrags.map(i=>`title like '%${i}%'`);
+ titlefrags = titlefrags.join(' and ');
+ sqldb.all(`Select id,title from titles where user=? and ${titlefrags}`,user,cb);
+}
+
+exports.edittitle=edittitle;
+function edittitle(user,id,title,cb) {
+ sqldb.run('Update titles Set title=? Where user=? and id=?',title,user,id,cb);
+}
+
+exports.pullmirrorsuper=pullmirrorsuper;
+function pullmirrorsuper(user,daysback,donecount,skiplimit,cb) {
+ getjournal(user,daysback,(err,backjournal)=>{
+  if(err) return cb(err);
+  if(!backjournal.list) {
+   if(skiplimit) pullmirrorsuper(user,daysback+1,donecount,skiplimit-1,cb);
+   else return cb("Journal missing - skiplimit reached");
+  }
+  else if(backjournal.list.length<donecount) pullmirrorsuper(user,daysback+1,donecount-=backjournal.list.length,skiplimit,cb);
+  else cb(null,backjournal.list[backjournal.list.length-donecount]);
+ });
+}
+ 
+
+exports.pullmirrortask=pullmirrortask;
+function pullmirrortask(user,cb) {
+ var skiplimit=5;
+ getjournal(user,0,(err,journal0)=>{
+  if(err) return cb(err);
+  if(!journal0.list) return cb("Journal missing");
+  var donecount=journal0.list.length;
+  var daysback=1;
+  function getone() {
+   getjournal(user,daysback,(err,backjournal)=>{
+    if(err) return cb(err);
+    if(!backjournal.list) {
+     if(skiplimit) {
+      --skiplimit;
+      ++daysback;
+      //console.log('One journal missing');
+      return getone();
+     }
+     return cb("Journal missing");
+    }
+    if(backjournal.list.length<donecount) {
+     donecount-=backjournal.list.length;
+     ++daysback;
+     return getone();
+    }
+    cb(null,backjournal.list[backjournal.list.length-donecount]);
+   });
+  }
+  getone(1);
+ });
+}
+
+function filter(list,filterSet,maxcount) {
+ maxcount = maxcount || Infinity;
+ return list.filter(i=>!filterSet.has(i)).slice(0,maxcount);
+}
+
+exports.randomtasks=randomtasks;
+function randomtasks(user,count,cb) {
+ ranks.get(user,(err,rank)=>{
+  var allids = Object.keys(rank);
+  var selectedids;
+  if(allids<count*count) {
+   selectedids=allids.sort(()=>Math.random()-0.5).slice(0,5);
+  }
+  else {
+   selectedids=new Set();
+   while(selectedids.size<count) {
+    selectedids.add(allids[Math.floor(allids.length*Math.random())]);
+   }
+   selectedids=Array.from(selectedids);
+  }
+  async.map(selectedids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
+ });
+}
+
+function getRanksButOnlyExposed(user,cb) {
+ ranks.get(user,(err,rank)=>{
+  if(err) return cb(err);
+  var out={};
+  var ids=Object.keys(rank);
+  var validids = ids.filter(id=>Object.keys(rank[id]).length>0);
+  validids.forEach(id=>out[id]=rank[id]);
+  cb(null,out);
+ });
+}
+
+
+exports.pulltaskids=pulltaskids;
+function pulltaskids(user,count,cb,getter=ranks.get) {
+ //var key=user+'.rank';
+ var pulledids=[];
+ var pulledidsSet=new Set();
+ var filterSet=new Set();
+ //db.get(key,(err,rank)=>{
+ getter(user,(err,rank)=>{
+  if(err) return cb(err);
+  var rankkeys=Object.keys(rank);
+  var topcohort;
+  //console.log({rank,rankkeys});
+  //For blocking items with temp block.
+  rankkeys.forEach(i=>{ //If it currently has a block don't display it
+   //console.log('Block level',i,rank[i].block);
+   if(rank[i].block>0) {
+    //console.log('Blocking',i,rank[i].block);
+    rank[i].block=Math.floor(rank[i].block-1);
+    //console.log('Blocking',i,rank[i].block);
+    if(rank[i].block<=0) {
+     delete rank[i].block;
+     //console.log('Delete rank',i);
+    }
+    filterSet.add(i);
+   }
+  });
+  //db.markWrite(key);
+  ranks.markWrite(user);
+  while(pulledids.length<count) {
+   topcohort=minindexes(rankkeys.map(id=>undercount(id,rank[id],pulledidsSet))).map(i=>rankkeys[i]);
+   //console.log({rankkeys,topcohort});
+   if(topcohort.length==0) return cb(null,pulledids); //No more items to add.  Return.
+   topcohort.forEach(id=>pulledidsSet.add(id));
+   topcohort=filter(shuffle(topcohort),filterSet,count-pulledids.length);
+   pulledids=pulledids.concat(topcohort);
+   if(pulledids.length==count) return cb(null,pulledids);
+  }
+ });
+}
+
+exports.pulltasks=pulltasks;
+function pulltasks(user,count,cb) {
+ pulltaskids(user,count,(err,ids)=>{
+  if(err) return cb(err);
+  console.log({ids});
+  async.map(ids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
+ });
+}
+
+module.exports.pullexposedtasks=pullexposedtasks;
+function pullexposedtasks(user,count,cb) {
+ pulltaskids(user,count,(err,ids)=>{
+  if(err) return cb(err);
+  console.log({ids});
+  async.map(ids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
+ },getRanksButOnlyExposed);
+};