Learn-to-Compress / headers / piecewise_double.h
piecewise_double.h
Raw
#ifndef PIECEWISE_DOUBLE_H_
#define PIECEWISE_DOUBLE_H_

#include "common.h"
#include "codecs.h"
#include "time.h"
#include "bit_read.h"
#include "bit_write.h"
#include "caltime.h"
#define INF 0x7f7fffff

namespace Codecset {



    
class piecewise_double : public IntegerCODEC {
public:
  using IntegerCODEC::encodeArray;
  using IntegerCODEC::decodeArray;
  using IntegerCODEC::randomdecodeArray;
  using IntegerCODEC::encodeArray8;
  using IntegerCODEC::decodeArray8;
  using IntegerCODEC::randomdecodeArray8;
  using IntegerCODEC::init;
   
  std::vector<uint8_t*> block_start_vec;
  std::vector<uint32_t> segment_index;
  uint32_t total_byte =0;
  int maxerror =(1U<<10)-1;
  int block_num;
  int block_size;

//start_index + bit + theta0 + theta1 + numbers + delta
void init( int blocks,  int blocksize,int delta){
      block_num=blocks;
      block_size=blocksize;
      maxerror = delta;
    
}   

float epsilon = 0.;
bool gt(float a, float b){
// greater than
    return (a > b + epsilon);
}
uint32_t lower_bound( uint32_t v,uint32_t len)
{
    uint32_t m;
    uint32_t x=0;
    uint32_t y=len-1;
    while(x <= y)
    {
    
        m = x+(y-x)/2;
        if(v<segment_index[m]) y = m-1;
        else x = m+1;
    }
    return y;
   
}

uint8_t * encodeArray8(uint32_t *in, const size_t length,uint8_t *res, size_t nvalue) {

    std::vector<uint32_t> indexes;
    for(uint32_t i = 0; i < nvalue; i++){
        indexes.push_back(i);
    }   
    double high_slope = (double)INF;
    double low_slope = -0.001;
    long long origin_key = in[0];
    int origin_index = indexes[0];
    int end_index = indexes[0];
    int total_index =0;
    for (int i = 1; i < (long long)nvalue; i++){
        long long key = in[i];
        int id = indexes[i];
        double tmp_point_slope = ((key - origin_key)+0.0) / ((id - origin_index)+0.0);
        if (tmp_point_slope >= low_slope && tmp_point_slope <= high_slope){
        //if (gt(tmp_point_slope, low_slope) && gt(high_slope, tmp_point_slope)){
            double tmp_high_slope = ((key + maxerror - origin_key)+0.0) / ((id - origin_index)+0.0);
            double tmp_low_slope = ((key - maxerror - origin_key)+0.0) /((id - origin_index)+0.0);
            if (tmp_low_slope<0.0){
                //std::cout<<"zero!"<<std::endl;
                tmp_low_slope=0.0;
            }
            
            
           if(tmp_high_slope<=high_slope){
           //if(gt(high_slope, tmp_high_slope)){
                high_slope = tmp_high_slope;
            }
            if(low_slope<=tmp_low_slope){
            //if(gt(tmp_low_slope, low_slope)){
                low_slope = tmp_low_slope;
            }
            end_index = id;
            
        }
        else{
            
            double slope = (high_slope + low_slope) / 2.;
            int max_error = 0;

            if (end_index == origin_index){
                slope = 1.;
            }
            int seg_len = end_index - origin_index + 1;
            int * delta = static_cast<int *>(malloc(seg_len * sizeof(int)));
            
            
            for (int j = origin_index; j <= end_index; j++ ){
                long long  predict = (long long)in[origin_index] + (long long)(slope*(double)(j-origin_index));
                long long truth = (long long)in[j];
                int tmp = abs(predict-truth);
                 
                delta[j-origin_index] = (int) truth-predict;
                total_index++;

                if (tmp > max_error){
                    max_error = tmp;
                    
                }
            }

            int tmp_bit = bits(max_error)+1;
            /*
            uint8_t * delta_pos = (uint8_t*)malloc((end_index - origin_index + 1) * sizeof(uint64_t));
            uint8_t* delta_write =delta_pos;
            delta_write=write_delta(delta, delta_write, tmp_bit, (end_index - origin_index + 1));
            int delta_size = delta_write - delta_pos;
           */

            uint8_t * descriptor = (uint8_t*)malloc((end_index - origin_index + 1) * sizeof(uint64_t)+30);
            uint8_t *out = descriptor;
            
            memcpy(out,&origin_index,sizeof(origin_index));
            out+=sizeof(origin_index);
            
            out[0]=(uint8_t)tmp_bit;
            out++;
            
            uint32_t theta0 =  (long long)in[origin_index] ;
            uint32_t numbers = (end_index - origin_index + 1);
            memcpy(out,&theta0,sizeof(theta0));

            out+=sizeof(theta0);
            memcpy(out,&slope,sizeof(slope));
            out+=sizeof(slope);
            memcpy(out,&numbers,sizeof(numbers));
            out+=sizeof(numbers);

            //std::cout<<"bit_length: "<<tmp_bit<<" start: "<<origin_index<<" end: "<<end_index<<" slope: "<<slope<<std::endl;
            
            out=write_delta(delta, out, tmp_bit, numbers);
            free(delta);
            
            
            //std::cout<<"bit_length: "<<tmp_bit<<" start: "<<origin_index<<" end: "<<end_index<<" slope: "<<slope<<std::endl;
            /*
            if(origin_index==66979987){
                uint8_t * tmpin=descriptor;
                uint32_t theta0_rec;
                float theta1_rec;
                uint8_t maxerror_rec;
                uint32_t start_ind_rec;
                uint32_t numbers_rec;
        
                memcpy(&start_ind_rec,tmpin,4);
                tmpin+=4;
                memcpy(&maxerror_rec,tmpin,1);
                tmpin++;
                memcpy(&theta0_rec,tmpin,4);
                tmpin+=4;
                memcpy(&theta1_rec,tmpin,sizeof(float));
                tmpin+=sizeof(float);
                memcpy(&numbers_rec,tmpin,4);
                tmpin +=4;
                 std::cout<<" slope_rec "<<theta1_rec<<" theta0_rec "<<theta0_rec<<"start_ind_rec"<<start_ind_rec<<"maxerror_rec"<<unsigned(maxerror_rec) <<std::endl;
             }
            */
           
            descriptor=(uint8_t*)realloc(descriptor, (out-descriptor));
            block_start_vec.push_back(descriptor);
            segment_index.push_back(origin_index);
            
            total_byte +=(out - descriptor);
            
            
            //std::cout<<"bit_length: "<<tmp_bit<<" start: "<<origin_index<<" end: "<<end_index<<" slope: "<<slope<<std::endl;
            high_slope = (double)INF;
            low_slope = 0.0;
            origin_index = id;
            origin_key = key;
            end_index = id;   
            
            
        }

    }
    
    double slope = (high_slope + low_slope) / 2.;
    if (end_index == origin_index){
        slope = 1.;
    }
    
    int seg_len = end_index - origin_index + 1;
    int * delta = static_cast<int *>(malloc(seg_len * sizeof(int)));
    int max_error = 0;
    for (int j = origin_index; j <= end_index; j++ ){
        long long  predict = (long long)in[origin_index] + (long long)(slope*(double)(j-origin_index));
        long long truth = (long long)in[j];
        int tmp = abs(predict-truth);     
        delta[j-origin_index] = (int) truth-predict;

        total_index++;
        if (tmp > max_error){
            max_error = tmp;   
        }
    }

    uint8_t tmp_bit = 0;
    if(max_error > 0.01){
        tmp_bit = ceil(log2(max_error))+2;
    }
    else{
        tmp_bit = 2;
    }

    
    uint8_t * descriptor = (uint8_t*)malloc((end_index - origin_index + 1) * sizeof(uint64_t)+30);
    uint8_t *out = descriptor;
    uint32_t start_ind = origin_index;
    memcpy(out,&start_ind,sizeof(start_ind));
    out+=sizeof(start_ind);
            
    out[0]=(uint8_t)tmp_bit;
    out++;
            
    uint32_t theta0 = (long long)in[origin_index] ;
    uint32_t numbers = (end_index - origin_index + 1);
    memcpy(out,&theta0,sizeof(theta0));
    out+=sizeof(theta0);
    memcpy(out,&slope,sizeof(slope));
    out+=sizeof(slope);
    memcpy(out,&numbers,sizeof(numbers));
    out+=sizeof(numbers);
    out=write_delta(delta, out, tmp_bit, numbers);
    free(delta);

    descriptor=(uint8_t*)realloc(descriptor, (out-descriptor));
    block_start_vec.push_back(descriptor);
    segment_index.push_back(start_ind);
    total_byte +=(out - descriptor);


    return res;
    
}
    
uint32_t *decodeArray8( uint8_t *in, const size_t length, uint32_t *out, size_t nvalue) {
//start_index + bit + theta0 + theta1 + numbers + delta   
    uint32_t * tmpout = out;
    for(int i=0;i<(int)block_start_vec.size();i++){
        uint8_t * this_block = block_start_vec[i];
        uint8_t * tmpin=this_block;
        uint32_t * theta0;
        double * theta1;
        uint8_t maxerror;
        uint32_t * start_ind;
        uint32_t * numbers;
        
        start_ind = reinterpret_cast<uint32_t*>(tmpin);
        tmpin+=4;
        maxerror = tmpin[0];
        tmpin++;
        theta0 = reinterpret_cast<uint32_t*>(tmpin);
        tmpin+=4;
        theta1 = reinterpret_cast<double*>(tmpin);
        tmpin+=sizeof(double);
        numbers = reinterpret_cast<uint32_t*>(tmpin);
        tmpin +=4;
        /*
        if(start_ind <=134956){
        std::cout<<"start_ind "<<start_ind<<" maxerror "<<unsigned(maxerror)<<" theta0 "<<theta0<<" theta1 "<<theta1<<" numbers "<<numbers<<std::endl;
        }
        */
        if(numbers[0] ==1){
            tmpout[0]=theta0[0];
            tmpout++;
        }
        else{
            tmpout=read_all_bit_double(tmpin ,0,start_ind[0],numbers[0],maxerror,theta1[0],theta0[0], tmpout);
        }
    }
    
        
      
    

    return out;
}
uint32_t randomdecodeArray8( uint8_t *in, const size_t l, uint32_t *out, size_t nvalue){

    uint32_t length=segment_index.size();
    uint8_t * this_block = block_start_vec[lower_bound(l,length)];

    uint8_t * tmpin = this_block;
    uint32_t * theta0;
    double * theta1;
    uint8_t maxerror;
    uint32_t * start_ind;
    //uint32_t * numbers;
        
    start_ind = reinterpret_cast<uint32_t*>(tmpin);
    tmpin+=4;
    maxerror = tmpin[0];
    tmpin++;
    theta0 = reinterpret_cast<uint32_t*>(tmpin);
    tmpin+=4;
    theta1 = reinterpret_cast<double*>(tmpin);
    tmpin+=sizeof(double);
    //numbers = reinterpret_cast<uint32_t*>(tmpin);
    tmpin +=4;
    
    uint32_t tmp = read_bit_double(tmpin ,maxerror , l-start_ind[0],theta1[0],theta0[0],0);
    
    return tmp;

}

uint32_t* encodeArray( uint32_t *in, const size_t length, uint32_t *out,
                   size_t nvalue) {
    std::cout<<"Haven't implement. Please try uint8_t one..."<<std::endl;
    return out;
}
uint32_t *decodeArray( uint32_t *in, const size_t length,
                              uint32_t *out, size_t nvalue) {
    std::cout<<"Haven't implement. Please try uint8_t one..."<<std::endl;
    return out;
}
uint32_t randomdecodeArray(uint32_t *in, const size_t l,uint32_t *out, size_t nvalue){
    std::cout<<"Haven't implement. Please try uint8_t one..."<<std::endl;
    return 1;
}
uint32_t get_block_nums(){
    std::cout<<"Total block num is "<<block_start_vec.size()<<std::endl;
     return total_byte;
 }    
void destroy(){
for(int i=0;i<(int)block_start_vec.size();i++){
       free(block_start_vec[i]);
   }
    
}
std::string name() const {
    return "piecewise_double"; 
}    
  
};



} // namespace FastPFor

#endif /* SIMDFASTPFOR_H_ */