CanalTP::Parser< Visitor > Struct Template Reference

#include <osmpbfreader.hpp>

Public Member Functions
void	parse ()
	Parser (const std::string &filename, Visitor &visitor, bool _wayOnly)
	~Parser ()

Private Member Functions
OSMPBF::BlobHeader	read_header ()
int32_t	read_blob (const OSMPBF::BlobHeader &header)
void	parse_primitiveblock (int32_t sz)

Private Attributes
std::ifstream	file
Visitor &	visitor
bool	wayOnly
bool	finished
char *	buffer
char *	unpack_buffer
std::streampos	file_size

Constructor & Destructor Documentation

Parser()

template<typename Visitor >

CanalTP::Parser< Visitor >::Parser ( const std::string &  filename, Visitor &  visitor, bool  _wayOnly  )

inline

                                                                         : visitor(visitor) {
 
        wayOnly = _wayOnly;
        finished = false;
        file = std::ifstream(filename.c_str(), std::ios::binary );
 
        file_size = fileSize(filename.c_str());
        if(!file.is_open())
            fatal() << "Unable to open the file " << filename;
        buffer = new char[max_uncompressed_blob_size];
        unpack_buffer = new char[max_uncompressed_blob_size];
        info() << "#Reading the file " << filename;
        info() << "#" << file_size/1000000 << " MB to do";
    }

~Parser()

template<typename Visitor >

CanalTP::Parser< Visitor >::~Parser ( )

inline

             {
        delete[] buffer;
        delete[] unpack_buffer;
    }

Member Function Documentation

parse()

template<typename Visitor >

void CanalTP::Parser< Visitor >::parse ( )

inline

                {
#ifdef DEBUG
        std::streampos todo = file_size;
        std::streampos last_todo = file_size;
#endif
 
        while(!this->file.eof() && !finished) {
            OSMPBF::BlobHeader header = this->read_header();
            if(!this->finished){
                int32_t sz = this->read_blob(header);
                if (sz > 0) {
                    if(header.type() == "OSMData") {
                        this->parse_primitiveblock(sz);
                    }
                    else if(header.type() == "OSMHeader"){
                    }
                    else {
                        warn() << "  unknown blob type: " << header.type();
                    }
#ifdef DEBUG
                    todo -= max_blob_header_size;
                    if ((todo+10000000) < last_todo) {
                        last_todo = todo;
                        info() << 100*(todo/(float)file_size) << " \% to do";
                    }
#endif
                }
            }
        }
    }

parse_primitiveblock()

template<typename Visitor >

void CanalTP::Parser< Visitor >::parse_primitiveblock ( int32_t  sz )

inlineprivate

                                          {
        OSMPBF::PrimitiveBlock primblock;
        if(!primblock.ParseFromArray(this->unpack_buffer, sz))
            fatal() << "unable to parse primitive block";
 
        for(int i = 0, l = primblock.primitivegroup_size(); i < l; i++) {
            const OSMPBF::PrimitiveGroup& pg = primblock.primitivegroup(i);
 
            if (wayOnly) {
                //if (pg.ways_size()>0) info() << "ways " << pg.ways_size();
                for(int i = 0; i < pg.ways_size(); ++i) {
                    const OSMPBF::Way& w = pg.ways(i);
 
                    uint64_t ref = 0;
                    std::vector<uint64_t> refs;
                    for(int j = 0; j < w.refs_size(); ++j){
                        ref += w.refs(j);
                        refs.push_back(ref);
                    }
                    uint64_t id = w.id();
                    visitor.way_callback(id, get_tags(w, primblock), refs);
                }
 
            } else {
                //if (pg.nodes_size()>0) info() << "simple nodes " << pg.nodes_size();
 
                // Simple Nodes
                for(int i = 0; i < pg.nodes_size(); ++i) {
                    const OSMPBF::Node& n = pg.nodes(i);
 
                    double lon = 0.000000001 * (primblock.lon_offset() + (primblock.granularity() * n.lon())) ;
                    double lat = 0.000000001 * (primblock.lat_offset() + (primblock.granularity() * n.lat())) ;
                    visitor.node_callback(n.id(), lon, lat, get_tags(n, primblock));
                }
 
                // Dense Nodes
                if(pg.has_dense()) {
                    const OSMPBF::DenseNodes& dn = pg.dense();
                    //info() << "dense nodes " << dn.id_size();
 
                    uint64_t id = 0;
                    double lon = 0;
                    double lat = 0;
 
                    int current_kv = 0;
 
                    for(int i = 0; i < dn.id_size(); ++i) {
                        id += dn.id(i);
                        lon +=  0.000000001 * (primblock.lon_offset() + (primblock.granularity() * dn.lon(i)));
                        lat +=  0.000000001 * (primblock.lat_offset() + (primblock.granularity() * dn.lat(i)));
 
                        Tags tags;
                        while (current_kv < dn.keys_vals_size() && dn.keys_vals(current_kv) != 0){
                            uint64_t key = dn.keys_vals(current_kv);
                            uint64_t val = dn.keys_vals(current_kv + 1);
                            std::string key_string = primblock.stringtable().s(key);
                            std::string val_string = primblock.stringtable().s(val);
                            current_kv += 2;
                            tags[key_string] = val_string;
                        }
                        ++current_kv;
                        visitor.node_callback(id, lon, lat, tags);
                    }
                }
 
                //if (pg.relations_size()>0) info() << "relations " << pg.relations_size();
                /*
                for(int i=0; i < pg.relations_size(); ++i){
                    const OSMPBF::Relation& rel = pg.relations(i);
                    uint64_t id = 0;
                    References refs;
 
                    for(int l = 0; l < rel.memids_size(); ++l){
                        id += rel.memids(l);
                        refs.push_back(Reference(rel.types(l), id, primblock.stringtable().s(rel.roles_sid(l))));
                    }
                    visitor.relation_callback(rel.id(), get_tags(rel, primblock), refs);
                }*/
            }
 
        }
    }

read_blob()

template<typename Visitor >

int32_t CanalTP::Parser< Visitor >::read_blob ( const OSMPBF::BlobHeader &  header )

inlineprivate

                                                      {
        OSMPBF::Blob blob;
        // size of the following blob
        int32_t sz = header.datasize();
 
        if(sz > max_uncompressed_blob_size)
            fatal() << "blob-size is bigger then allowed";
 
        if(!this->file.read(buffer, sz))
            fatal() << "unable to read blob from file";
        if(!blob.ParseFromArray(this->buffer, sz))
            fatal() << "unable to parse blob";
 
        // if the blob has uncompressed data
        if(blob.has_raw()) {
            // size of the blob-data
            sz = blob.raw().size();
 
            // check that raw_size is set correctly
            if(sz != blob.raw_size())
                warn() << "  reports wrong raw_size: " << blob.raw_size() << " bytes";
 
            memcpy(unpack_buffer, blob.raw().c_str(), sz);
            return sz;
        }
 
        if(blob.has_zlib_data()) {
            //info() << "blob has zlib stream";
            sz = blob.zlib_data().size();
 
            z_stream z;
            z.next_in   = (unsigned char*) blob.zlib_data().c_str();
            z.avail_in  = sz;
            z.next_out  = (unsigned char*) unpack_buffer;
            z.avail_out = blob.raw_size();
            z.zalloc    = Z_NULL;
            z.zfree     = Z_NULL;
            z.opaque    = Z_NULL;
 
            if(inflateInit(&z) != Z_OK) {
                fatal() << "failed to init zlib stream";
            }
            if(inflate(&z, Z_FINISH) != Z_STREAM_END) {
                fatal() << "failed to inflate zlib stream";
            }
            if(inflateEnd(&z) != Z_OK) {
                fatal() << "failed to deinit zlib stream";
            }
            return z.total_out;
        }
 
        if(blob.has_lzma_data()) {
            fatal() << "lzma-decompression is not supported";
        }
        return 0;
    }

read_header()

template<typename Visitor >

OSMPBF::BlobHeader CanalTP::Parser< Visitor >::read_header ( )

inlineprivate

                                  {
        int32_t sz;
        OSMPBF::BlobHeader result;
 
        // read the first 4 bytes of the file, this is the size of the blob-header
        if( !file.read((char*)&sz, 4) ){
            info() << "# We finished reading the file";
            this->finished = true;
            return result;
        }
 
        sz = ntohl(sz);// convert the size from network byte-order to host byte-order
 
        if(sz > max_blob_header_size)
            fatal() << "blob-header-size is bigger then allowed " << sz << " > " << max_blob_header_size;
 
        this->file.read(this->buffer, sz);
        if(!this->file.good())
            fatal() << "unable to read blob-header from file";
 
        // parse the blob-header from the read-buffer
        if(!result.ParseFromArray(this->buffer, sz))
            fatal() << "unable to parse blob header";
        return result;
    }

Member Data Documentation

buffer

template<typename Visitor >

char* CanalTP::Parser< Visitor >::buffer

private

file

template<typename Visitor >

std::ifstream CanalTP::Parser< Visitor >::file

private

file_size

template<typename Visitor >

std::streampos CanalTP::Parser< Visitor >::file_size

private

finished

template<typename Visitor >

bool CanalTP::Parser< Visitor >::finished

private

unpack_buffer

template<typename Visitor >

char* CanalTP::Parser< Visitor >::unpack_buffer

private

visitor

template<typename Visitor >

Visitor& CanalTP::Parser< Visitor >::visitor

private

wayOnly

template<typename Visitor >

bool CanalTP::Parser< Visitor >::wayOnly

private

---

The documentation for this struct was generated from the following file:

• examples/osmpbfreader.hpp