webtrack-server / module / Download.js
Download.js
Raw
const OPTIONS = {
  REGEX: 'Regex',
  CONTAINS: 'Contains',
  SLIDERNUMBER: 'SliderNumber',
  SLIDERDATE: 'SliderDate'
}

const MAX_RUN_COUNT = 3;
const PAGE_ATTR = ['ID', 'PRECURSOR_ID', 'PROJECT_ID', 'URL', 'TITLE', 'DURATION', 'STARTTIME', 'TYPEVERSION', 'EVENTS', 'DESCRIPTION', 'KEYWORDS', 'CONTENT'];

var filtergroup = require('../module/Filtergroup.js');
var filter = new (require('../module/Filter.js'))(filtergroup);
var dataPage = require('../module/DataPage.js');
var project = require('../module/Project.js');
var events2page = require('../module/Events2Page.js');
var DownloadTableClass = require('./sql/DownloadTableClass.js')
const Archive = require('./lib/Archive.js');



class Download extends DownloadTableClass{

  /**
   * [getMAXCount return const MAX_RUN_COUNT]
   * @return {[type]} [description]
   */
  getMAXCount(){
    return MAX_RUN_COUNT;
  }

  /**
   * [_filter run the filter and check the value]
   * @param  {Object} f
   * @param  {Unknown} value
   * @return {Boolean}
   */
  _filter(f, value){
    switch (f.TYPE) {
      case OPTIONS.CONTAINS:
          for (let toCompare of JSON.parse(f.VALUE)) {
            if(value.indexOf(toCompare)>=0) return true;
          }
          return false;
        break;
      case OPTIONS.REGEX:
          try {
            let flags = f.VALUE.replace(/.*\/([gimy]*)$/, '$1');
            let pattern = f.VALUE.replace(new RegExp('^/(.*?)/'+flags+'$'), '$1');
            let regex = new RegExp(pattern, flags);
            return regex.test(value);
          } catch (e) {
            return false
          }
        break;
      case OPTIONS.SLIDERNUMBER:
          if(typeof f.VALUE === 'string') f.VALUE = JSON.parse(f.VALUE);
          return f.VALUE[0] <= value && value <= f.VALUE[1];
        break;
      case OPTIONS.SLIDERDATE:
          if(typeof f.VALUE === 'string') f.VALUE = JSON.parse(f.VALUE);
          return f.VALUE[0] <= value.getTime() && value.getTime() <= f.VALUE[1];
        break;

      default:

    }
    return false;
  }

  /**
   * [_filterPages filter the attributes of pages and return the filter object]
   * @param  {Array} filters
   * @param  {Object} client2pages
   * @param  {Object} page2content [data of pages]
   * @param  {Archive} archive
   * @return {Promise} Object
   */
  _filterPages(filters, client2pages, page2content, archive){
    return new Promise(async (resolve, reject) => {
      try {
        let toDel = {};

        for (let client_id in client2pages) {
          for (let index in client2pages[client_id]){
              let p = client2pages[client_id][index];


              for (let attr in p) {
                for (let f of filters) {
                  if(attr===f.COLUME){
                    if(!this._filter(f, p[attr])){
                      if(!toDel.hasOwnProperty(client_id)) toDel[client_id] = [];
                      if(!toDel[client_id].includes(index)) toDel[client_id].push(index);
                    }// if false
                  }//attr===f.COLUME
                }//for filters
              }//for page attr


              for (let version in page2content[p.ID]) {
                let v = page2content[p.ID][version];
                console.log('v.filename', v);
                if(v.hasOwnProperty('filename')){
                  let text = await archive.getAttachment(v.filename);

                  for (let f of filters) {
                    if('CONTENT'===f.COLUME){
                      if(!this._filter(f, text)){
                        if(!toDel.hasOwnProperty(client_id)) toDel[client_id] = [];
                        if(!toDel[client_id].includes(index)){
                          toDel[client_id].push(index);
                          console.log('delete', v.filename);
                          await archive.deleteAttachment(v.filename);
                        }
                      }
                    }
                  }//for filters
                }

              }//for html version



          }//for client pages
        }//for client

        for (let client_id in client2pages) {
          if(toDel.hasOwnProperty(client_id)){
            for (let index of toDel[client_id]) {
               if(client2pages[client_id].hasOwnProperty(index)){
                  delete client2pages[client_id][index];
               }else{
                 console.log('index ' + index + ' not found in ' +client_id);
               }
            }
            client2pages[client_id] = client2pages[client_id].filter(v=>v!==undefined || v!==null);
          }
          if(client2pages[client_id].length===0) delete client2pages[client_id]
        }

        resolve(client2pages);
      } catch (e) {
        console.log(e);
        reject(e);
      }

    });
  }

  /**
   * [_getClientsPages return object of pages from client_ids]
   * @param  {Integer} user_id
   * @param  {Integer} project_id
   * @param  {Array} client_ids
   * @return {Promise} Object
   */
  _getClientsPages(user_id, project_id, client_ids){
    return new Promise(async (resolve, reject) => {
      try {
        let r = {};
        for (let client_id of client_ids) {
          r[client_id] = [];
          let data = await project.getClientPages(user_id, project_id, client_id);
          r[client_id] = r[client_id].concat(data);
        }
        resolve(r)
      } catch (e) {
        reject(e)
      }

    });
  }


  /**
   * [_fillArchive fill the archive with the pages]
   * @param  {Archive} archive
   * @param  {Array} page_ids
   * @param  {Number} level
   * @return {Promise}
   */
  _fillArchive(archive, page_ids, level){
    return new Promise(async (resolve, reject) => {
      try {
        let data = {};
        const chunkSize = 50;
        let count = 0;
        let chunks = [].concat.apply([], page_ids.map((elem,i) => i%chunkSize ? [] : [page_ids.slice(i,i+chunkSize)] ));

        for (let chunk of chunks) {
          let content = await dataPage.get(chunk, level);

          for (let id of Object.keys(content)) {
            count += 1;
            if(!data.hasOwnProperty(id)){
              data[id] = [];
            }
            let d = { createdate: content[id][id].createdate };
            if(content[id][id].hasOwnProperty('html')){
              d.filename = await archive.appendAttachment(content[id][id].html, 'UTF-8')
            }else if(content[id][id].hasOwnProperty('text')){
              d.filename = await archive.appendAttachment(content[id][id].text, 'UTF-8')
            }
            data[id].push(d);
          }
        }//for
        resolve(data);
      } catch (err) {
        reject(err)
      }
    });
  }


  /**
   * [_createResult create result for download file]
   * @param  {Object} clients
   * @param  {Object} client2pages
   * @param  {Object} page2content
   * @return {Promise} Object
   */
  _createResult(clients, client2pages, page2content){
    return new Promise(async (resolve, reject) => {
      try {
        let result = [];
        for (let client of clients) {
          let client_r = {name: client.CLIENT_HASH, pages: []};
          if(client2pages[client.ID]===undefined) continue;
          for (let page of client2pages[client.ID]) {
              page.EVENTS = [];
              //push Events
              for (let event of await events2page.get(page.ID)) {
                let r_event = {
                  type: event.TYPE,
                  timestamp: event.TIMESTAMP,
                  values: []
                };
                for (let v of event.values) {
                  r_event.values.push({
                    name: v.NAME,
                    value: v.VALUE
                  })
                }
                page.EVENTS.push(r_event)
              }

              let r_page = {}
              for (let attr in page) {
                if (PAGE_ATTR.includes(attr)) {
                  r_page[attr.toLowerCase()] = page[attr];
                }
              }
              if(page2content.hasOwnProperty(page.ID)){
                r_page.content = page2content[page.ID]
              }
              client_r.pages.push(r_page);
          }
          result.push(client_r)
        }//for
        resolve(result)
      } catch (err) {
        reject(err)
      }
    });
  }


  /**
   * [start start filter and return data of page]
   * @param  {Integer} entry_id
   * @param  {Integer} user_id
   * @param  {Integer} project_id
   * @param  {Array} filter_ids   [default: []]
   * @param  {Integer} level      [Number of level to get text or html code]
   * @return {Promise} Array
   */
  _start(entry_id, user_id, project_id, filter_ids=[], level){
    return new Promise(async (resolve, reject) => {
      const archive = new Archive();
      try {
        let f = await filter.getAll(filter_ids);
        console.log('level', level);

        let clients = await project.getClients(user_id, project_id);
        let client2pages = await this._getClientsPages(user_id, project_id, clients.map(v=>v.ID));
        let page_ids = [];
        for (let client_id in client2pages){
          page_ids = page_ids.concat(client2pages[client_id].map(v=>v.ID)); //get all page ids
        }
        // page_ids = page_ids.slice(0, 2);

        let page2content = await this._fillArchive(archive, page_ids, level);
        client2pages = await this._filterPages(f, client2pages, page2content, archive);
        let result = await this._createResult(clients, client2pages, page2content);
        archive.appendFile(JSON.stringify(result), 'UTF-8', 'data.json')
        console.log('ID', entry_id, '"result finish"');
        let buf = await archive.createZip();
        console.log('ID', entry_id, '"create zip finish"');
        await this.updateFile(entry_id, buf);
        console.log('ID', entry_id, '"download.updateFile"');
        archive.clean();
        resolve()
      } catch (e) {
        console.log(e);
        archive.clean();
        reject(e)
      }
    });
  }

  /**
   * [get check the id and return the entry by id]
   * @param  {Integer} id
   * @return {Promise}
   */
  get(id){
    return new Promise(async (resolve, reject) => {
      try {
        if(await this.isId(id)){
          resolve((await super.get(id))[0])
        } else {
          throw new Error('Id '+id+' not found');
        }
      } catch (e) {
        reject(e)
      }
    });
  }

  /**
   * [updateCount increase the count of entry is startet]
   * @param  {Integer} id
   * @return {Promise}
   */
  updateCount(id){
    return new Promise(async (resolve, reject) => {
      try {
        let count = await this.getCount(id);
        await this.setCount(id, count+1);
        resolve();
      } catch (e) {
        reject(e)
      }
    })
    return db.promiseQuery("UPDATE `"+this.table+"` SET `COUNT` = '"+count+"' WHERE `ID` = "+id+";");
  }

  /**
   * [getCount return count of entry is started]
   * @param  {Integer} id
   * @return {Promise}
   */
  getCount(id){
    return new Promise(async (resolve, reject) => {
      try {
        let count = (await super.getCount(id))[0].COUNT;
        resolve(parseInt(count, 10));
      } catch (e) {
        reject(e)
      }
    });
  }

  /**
   * [getList check permisson and return list of filtered entrys]
   * @param  {Integer} user_id
   * @param  {Integer} project_id
   * @return {Promise}
   */
  getList(user_id, project_id){
    return new Promise(async (resolve, reject) => {
      try {
        await project._checkPermission(user_id, project_id)
        resolve(await super.getList(project_id));
      } catch (e) {
        reject(e)
      }
    })
  }

  /**
   * [delete // check permisson from user and delete download entry]
   * @param  {Integer} user_id
   * @param  {Integer} project_id
   * @param  {Integer} id
   * @return {Promise}
   */
  delete(user_id, project_id, id){
    return new Promise(async (resolve, reject) => {
      try {
        await project._checkPermission(user_id, project_id)
        resolve(await super.delete(id));
      } catch (e) {
        reject(e)
      }
    })
  }

  /**
   * [getFile // check permisson from user and return buffer file by id]
   * @param  {Integer} user_id
   * @param  {Integer} project_id
   * @param  {Integer} id
   * @return {Promise} Buffer
   */
  getFile(user_id, project_id, id){
    return new Promise(async (resolve, reject) => {
      try {
        await project._checkPermission(user_id, project_id)
        resolve((await super.getFile(id))[0].FILE);
      } catch (e) {
        reject(e)
      }
    })
  }



}

module.exports = Download;