todo4/todo.js

var exports=module.exports;
var _data = __dirname+'/data';
exports._data=_data;

var fs=require('fs');
var {shuffle} = require('lodash');
var async = require('async');
var df = require('../df');
var {ranks,journals} = require('./data2');

df((err,kb)=>{
 if(err) return console.log('df err',err);
 if(kb<1000) process.exit(1);
});

var db = require('../readwritecache')(dbwrite,dbwrite,dbread,{},1000,1);
//var {Database} = require('sqlite3');
var sqldb = require('./sqlitedb'); 
//var {Client} = require('cassandra-driver');
//const cass = new Client({
// cloud: {secureConnectBundle:'secure-connect-todo.zip'},
// credentials: {
//  username:'jvonmitchell@gmail.com',
//  password:'6e9hDX8$$'
// },
//});

//async function startcass() {
// await cass.connect();
// cass.execute('use todo');
//}
//startcass();


exports.db=db;

exports.dbwrite=dbwrite;
function dbwrite(key,obj,cb) {
 //console.log("Writing",key);
 var stringified=JSON.stringify(obj);
 var user = key.split('.')[0];
 if(key.endsWith('.rank')) {
  sqldb.run('Update rank set json=? where user=?',stringified,user,noop);
  cass.execute('Insert into rank (user,json) values (?,?)',[user,stringified]);
 }
 if(key.indexOf('.day.')!=-1) {
  var day = key.split('.day.')[1];
  cass.execute('Insert into day (user,day,json) values (?,?,?)',[user,day,stringified]);
 }
 fs.writeFile(_data+'/'+key+'.json',stringified,cb);
}

var rankCache = new Map();
var rankCacheTimeouts = new Map();
exports.dbread=dbread;
function dbread(key,cb) {
 var user = key.split('.')[0];
 if(key.endsWith('.rank')) {
  if(rankCache.has(user)) {
   clearTimeout(rankCacheTimeouts.get(user));
   rankCacheTimeouts.set(user,setTimeout(()=>rankCache.delete(user)));
   return cb(null,rankCache.get(user));
  }
  return sqldb.get('Select json from rank where user=?',user,(err,json)=>{
   if(err) return cb(err);
   //if(!json) return cb('User not found in rank');
   if(!json) return cb();
   json=JSON.parse(json.json);
   rankCache.set(user,json);
   rankCacheTimeouts.set(user,setTimeout(()=>rankCache.delete(user)));
   cb(null,json);
  });
 }
 if(key.indexOf('.day.')!=-1) {
  //var indexOf = key.indexOf('.day.');
  //console.log({key,indexOf});
  var date=key.split('.day.')[1].replace('.json','');
  return cass.execute('Select json from day where user=? and day=?',[user,date]).then(result=>{
   if(result && result.rows && result.rows.length) {
    //console.log({result});
    return cb(null,JSON.parse(result.rows[0].json));
   }
   fs.readFile(_data+'/'+key+'.json',(err,buff)=>{
    if(err) return cb();
    try {
     cb(null,JSON.parse(buff));
    }
    catch(e) {
     console.log({key,buff:buff.toString(),err:e,cb});
     cb(e);
    }
   });
  });
 }
 fs.readFile(_data+'/'+key+'.json',(err,buff)=>{
  if(err) return cb();
  try {
   cb(null,JSON.parse(buff));
  }
  catch(e) {
   console.log({key,buff:buff.toString(),err:e,cb});
   cb(e);
  }
 });
}


exports.minindexes=minindexes;
function minindexes(list) {
 if(!list.length) return [];
 var minvalue=list.reduce((a,b)=>a<b?a:b);
 var retr=[];
 for(var i=0;i<list.length;++i) {
  if(list[i]==minvalue) retr.push(i);
 }
 //console.log({minvalue,idexes:retr});
 return retr;
}






exports.removetasks=removetasks;
function removetasks(user,tasks,cb) {
 async.map(tasks,(task,cb)=>removetask(user,cb),cb);
}

function noop () {}

exports.addtask=addtask;
function addtask(user,tasktext,cb) {
 //var rankkey=user+'.rank';
 var addhistkey=user+'.addhist';
 //async.map([rankkey,addhistkey],db.get,(err,objs)=>{
 async.parallel([
  cb=>ranks.get(user,cb),
  cb=>db.get(addhistkey,cb)
 ],(err,objs)=>{
  if(err) return cb(err);
  var [rankobj,addhistobj]=objs;
  var i=0;
  while(rankobj[i]) ++i;
  sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
  rankobj[i]={};
  if(!addhistobj.hist) addhistobj.hist=[];
  addhistobj.hist.push(tasktext);
  //console.log({rankkey,rankobj,i,tasktext});
  //db.markWrite(rankkey);
  ranks.markWrite(user);
  db.markWrite(addhistkey);
  cb(null,i);
 });
}

exports.addmany=addmany;
function addmany(user,tasktexts,cb) {
 //var rankkey=user+'.rank';
 //db.get(rankkey,(err,rankobj)=>{
 ranks.get(user,(err,rankobj)=>{
  if(err) return cb(err);
  var retr=[];
  var i=0;
  tasktexts.forEach(tasktext=>{
   if(!tasktext) return;
   while(rankobj[i]) ++i;
   retr.push(i);
   //sqldb.run('BEGIN TRANSACTION',noop);
   sqldb.run('Insert into titles values (?,?,?)',user,i,tasktext,noop);
   //sqldb.run('COMMIT',noop);
   rankobj[i]={};
  });
  //db.markWrite(rankkey);
  ranks.markWrite(user);
  cb(null,retr);
 });
}

exports.removetask=removetask;
function removetask(user,taskid,cb) {
 //var rankkey=user+'.rank';
 //db.get(rankkey,(err,rankobj)=>{
 ranks.get(user,(err,rankobj)=>{
  if(err) return cb(err);
  sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,result)=>{
   if(err) return cb(err);
   if(result) console.log('Removed:',result.title);
   sqldb.run('Delete from titles where user=? and id=?',user,taskid,err=>{
    if(err) return cb(err);
    console.log('removing',taskid);
    delete rankobj[taskid];
    Object.values(rankobj).forEach(peerobj=>{
     delete peerobj[taskid];
    });
    //db.markWrite(rankkey);
    ranks.markWrite(user);
    cb();
   });
  });
 });
}

exports.completetask=completetask;
function completetask(user,taskid,cb) {
 var now=Date.now();
 var journalkey=user+'.day.'+(new Date()).toLocaleDateString().replace(/\//g,'.');
 //db.get(journalkey,(err,journalobj)=>{
 journals.get(journalkey,(err,journalobj)=>{
  if(err) return cb(err);
  var title;
  journalobj.list=journalobj.list||[];
  if(typeof(taskid)==='string') {
   title=taskid;
   journalobj.list.push({time:now,title});
   journals.markWrite(journalkey);
   //db.markWrite(journalkey);
  }
  else {
   sqldb.get('Select title from titles where user=? and id=?',user,taskid,(err,titleobj)=>{
    if(err) return cb(err);
    title=titleobj.title;
    journalobj.list.push({time:now,title,id:taskid});
    removetask(user,taskid,cb);
    journals.markWrite(journalkey);
    //db.markWrite(journalkey);
   });
  }
 });
}

exports.undercount=undercount;
function undercount(id,obj,blockset) {
 if(blockset.has(id)) return 1000;
 var count=0;
 var keys=Object.keys(obj);
 if(!keys.length) return 0;
 for(var i=0;i<keys.length;++i) {
  if(blockset.has(keys[i])) continue;
  if(obj[keys[i]]<0) count-=obj[keys[i]];
 }
 //console.log({id,count});
 return count;
}


exports.decayranks=decayranks;
const DECAY=5/300;
function decayranks(user,decay,cb) {
 //var key=user+'.rank';
 decay=decay||DECAY;
 //db.get(key,(err,obj)=>{
 ranks.get(user,(err,obj)=>{
  if(err) return cb(err);
  Object.values(obj).forEach(subobj=>{
   Object.keys(subobj).forEach(subkey=>{
    if(subkey=='block') return;
    subobj[subkey]*=decay;
   });
  });
  //db.markWrite(key);
  ranks.markWrite(user);
  cb();
 });
}

exports.getjournal=getjournal;
function getjournal(user,days,cb) {
 days=days||0;
 days=Math.abs(days);
 var journalkey=user+'.day.'+(new Date(Date.now()-1000*60*60*24*days)).toLocaleDateString().replace(/\//g,'.');
 //db.get(journalkey,cb);
 journals.get(journalkey,cb);
}

//exports.ranktasks=ranktasks;
//function ranktasks(user,tasklist,cb) {
// //var key=user+'.rank';
// //db.get(key,(err,obj)=>{
// ranks.get(user,(err,obj)=>{
//  if(err) return cb(err);
//  var i;
//  for(i=0;i<tasklist.length;++i) {
//   for(var j=i+1;j<tasklist.length;++j) {
//    obj[tasklist[i]][tasklist[j]]=1;
//    obj[tasklist[j]][tasklist[i]]=-1;
//   }
//  }
//  for(i=1;i<tasklist.length;++i) {
//   obj[tasklist[i]].block=i; //We don't want to see it for a bit, add a block that will count down
//   //console.log("Block",tasklist[i],i);
//  }
//  //db.markWrite(key);
//  ranks.markWrite(user);
//  cb();
// });
//}

exports.ranktasks=ranktasks;
function ranktasks(user,taskGroups,cb) {
 ranks.get(user,(err,obj)=>{
  if(err) return cb(err);
  var i;
  for(i=0;i<taskGroups.length;++i) {
   for(var j=i+1;j<taskGroups.length;++j) {
    for(var taskA of [].concat(taskGroups[i])) {
     for(var taskB of [].concat(taskGroups[j])) {
      obj[taskA][taskB]=1;
      obj[taskB][taskA]=-1;
     }
    }
   }
  }
  for (let i = 1; i < taskGroups.length; ++i) {
   [].concat(taskGroups[i]).forEach(task => obj[task].block = i);
  }
  ranks.markWrite(user);
  cb();
 });
}

exports.searchtasks=searchtasks;
function searchtasks(user,titlefrags,cb) {
 //This is not safe
 if(typeof(titlefrags)=='string') titlefrags = [titlefrags];
 titlefrags = titlefrags.map(i=>i.toLowerCase());
 titlefrags = titlefrags.map(i=>`title like '%${i}%'`);
 titlefrags = titlefrags.join(' and ');
 sqldb.all(`Select id,title from titles where user=? and ${titlefrags}`,user,cb);
}

exports.edittitle=edittitle;
function edittitle(user,id,title,cb) {
 sqldb.run('Update titles Set title=? Where user=? and id=?',title,user,id,cb);
}

exports.pullmirrorsuper=pullmirrorsuper;
function pullmirrorsuper(user,daysback,donecount,skiplimit,cb) {
 getjournal(user,daysback,(err,backjournal)=>{
  if(err) return cb(err);
  if(!backjournal.list) {
   if(skiplimit) pullmirrorsuper(user,daysback+1,donecount,skiplimit-1,cb);
   else return cb("Journal missing - skiplimit reached");
  }
  else if(backjournal.list.length<donecount) pullmirrorsuper(user,daysback+1,donecount-=backjournal.list.length,skiplimit,cb);
  else cb(null,backjournal.list[backjournal.list.length-donecount]);
 });
}
 

exports.pullmirrortask=pullmirrortask;
function pullmirrortask(user,cb) {
 var skiplimit=5;
 getjournal(user,0,(err,journal0)=>{
  if(err) return cb(err);
  if(!journal0.list) return cb("Journal missing");
  var donecount=journal0.list.length;
  var daysback=1;
  function getone() {
   getjournal(user,daysback,(err,backjournal)=>{
    if(err) return cb(err);
    if(!backjournal.list) {
     if(skiplimit) {
      --skiplimit;
      ++daysback;
      //console.log('One journal missing');
      return getone();
     }
     return cb("Journal missing");
    }
    if(backjournal.list.length<donecount) {
     donecount-=backjournal.list.length;
     ++daysback;
     return getone();
    }
    cb(null,backjournal.list[backjournal.list.length-donecount]);
   });
  }
  getone(1);
 });
}

function filter(list,filterSet,maxcount) {
 maxcount = maxcount || Infinity;
 return list.filter(i=>!filterSet.has(i)).slice(0,maxcount);
}

exports.randomtasks=randomtasks;
function randomtasks(user,count,cb) {
 ranks.get(user,(err,rank)=>{
  var allids = Object.keys(rank);
  var selectedids;
  if(allids<count*count) {
   selectedids=allids.sort(()=>Math.random()-0.5).slice(0,5);
  }
  else {
   selectedids=new Set();
   while(selectedids.size<count) {
    selectedids.add(allids[Math.floor(allids.length*Math.random())]);
   }
   selectedids=Array.from(selectedids);
  }
  async.map(selectedids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
 });
}

function getRanksButOnlyExposed(user,cb) {
 ranks.get(user,(err,rank)=>{
  if(err) return cb(err);
  var out={};
  var ids=Object.keys(rank);
  var validids = ids.filter(id=>Object.keys(rank[id]).length>0);
  validids.forEach(id=>out[id]=rank[id]);
  cb(null,out);
 });
}


exports.pulltaskids=pulltaskids;
function pulltaskids(user,count,cb,getter=ranks.get) {
 //var key=user+'.rank';
 var pulledids=[];
 var pulledidsSet=new Set();
 var filterSet=new Set();
 //db.get(key,(err,rank)=>{
 getter(user,(err,rank)=>{
  if(err) return cb(err);
  var rankkeys=Object.keys(rank);
  var topcohort;
  //console.log({rank,rankkeys});
  //For blocking items with temp block.
  rankkeys.forEach(i=>{ //If it currently has a block don't display it
   //console.log('Block level',i,rank[i].block);
   if(rank[i].block>0) {
    //console.log('Blocking',i,rank[i].block);
    rank[i].block=Math.floor(rank[i].block-1);
    //console.log('Blocking',i,rank[i].block);
    if(rank[i].block<=0) {
     delete rank[i].block;
     //console.log('Delete rank',i);
    }
    filterSet.add(i);
   }
  });
  //db.markWrite(key);
  ranks.markWrite(user);
  while(pulledids.length<count) {
   topcohort=minindexes(rankkeys.map(id=>undercount(id,rank[id],pulledidsSet))).map(i=>rankkeys[i]);
   //console.log({rankkeys,topcohort});
   if(topcohort.length==0) return cb(null,pulledids); //No more items to add.  Return.
   topcohort.forEach(id=>pulledidsSet.add(id));
   topcohort=filter(shuffle(topcohort),filterSet,count-pulledids.length);
   pulledids=pulledids.concat(topcohort);
   if(pulledids.length==count) return cb(null,pulledids);
  }
 });
}

exports.pulltasks=pulltasks;
function pulltasks(user,count,cb) {
 pulltaskids(user,count,(err,ids)=>{
  if(err) return cb(err);
  console.log({ids});
  async.map(ids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
 });
}

module.exports.pullexposedtasks=pullexposedtasks;
function pullexposedtasks(user,count,cb) {
 pulltaskids(user,count,(err,ids)=>{
  if(err) return cb(err);
  console.log({ids});
  async.map(ids,(id,cb)=>sqldb.get('Select id,title from titles where user=? and id=?',user,id,cb),cb);
 },getRanksButOnlyExposed);
};