The SwappyItems Key-Values Store
tools.hpp
Go to the documentation of this file.
1 // we need it for PRId32 in snprintf
2 #define __STDC_FORMAT_MACROS
3 
4 // -------------------------------------------------------------------------
5 string replacer(string& s, const string& toReplace, const string& replaceWith) {
6  size_t pos = 0;
7  bool endless = true;
8  while(endless) {
9  pos = s.find(toReplace, pos);
10  if (pos == std::string::npos) {
11  endless = false;
12  } else {
13  s = s.replace(pos, toReplace.length(), replaceWith);
14  }
15  }
16 
17  return s;
18 }
19 
20 // -------------------------------------------------------------------------
21 int parseLine(char* line) {
22  // This assumes that a digit will be found and the line ends in " Kb".
23  int i = strlen(line);
24  const char* p = line;
25  while (*p <'0' || *p > '9') p++;
26  line[i-3] = '\0';
27  i = atoi(p);
28  return i;
29 }
30 
31 // -------------------------------------------------------------------------
32 int getUsedKB() {
33  FILE* fi = fopen("/proc/self/status", "r");
34  int result = -1;
35  char line[128];
36 
37  while (fgets(line, 128, fi) != NULL){
38  if (strncmp(line, "VmRSS:", 6) == 0){
39  result = parseLine(line);
40  break;
41  }
42  }
43  fclose(fi);
44  return result;
45 }
46 
47 // -------------------------------------------------------------------------
48 void my_handler(int s) {
49  printf("# Caught signal %d\n", s);
50  auto now = std::chrono::high_resolution_clock::now();
51  mseconds = std::chrono::duration<double, std::milli>(now-start).count();
52  printf("#end (before hibernate): %.f ms\n", mseconds);
53 
54  delete places;
55  delete ways;
56  delete nodes;
57 
58  exit(1);
59 }
60 
61 // -------------------------------------------------------------------------
62 void catchSig() {
63  struct sigaction sigIntHandler;
64  sigIntHandler.sa_handler = my_handler;
65  sigemptyset(&sigIntHandler.sa_mask);
66  sigIntHandler.sa_flags = 0;
67  sigaction(SIGINT, &sigIntHandler, NULL);
68 }
69 
70 // -------------------------------------------------------------------------
71 void logHead() {
72  printf("# cores: %12d\n", thread::hardware_concurrency());
73  printf("# items in a single file: %12d\n", FILE_ITEMS);
74  printf("# swap if more than %12d items in RAM are reached\n", RAM_MULTI * FILE_MULTI * FILE_ITEMS);
75  printf("# swap into %12d files\n", FILE_MULTI);
76  printf("# use %12d Bloom bits for a key in a file bitmask\n", BBITS);
77  printf("# use a bitmask with %12d bits for each file\n", BMASK);
78 }
79 
80 // -------------------------------------------------------------------------
81 void logEntry(double & msec, std::chrono::time_point<std::chrono::system_clock> & old, atomic<bool> & isprnt) {
82  isprnt = true;
83  auto stat = nodes->getStatistic();
84  auto now = std::chrono::high_resolution_clock::now();
85  msec = std::chrono::duration<double, std::milli>(now-old).count();
86  printf(
87  "%10.f ms "
88  "%10ld i "
89  "%10ld q "
90 
91  "%10" PRId64 " u "
92  "%10" PRId64 " d "
93 
94  "%10" PRId64 " r "
95  "%10" PRId64 " b "
96  "%10" PRId64 " e "
97 
98  "%10" PRId64 " s "
99  "%10" PRId64 " l "
100  "%10d kB "
101  "%10ld filekB "
102 
103  "%10" PRId64 " "
104  "%10" PRId64 "\n",
105 
106  msec,
107  stat.size,
108  stat.queue,
109 
110  stat.updates,
111  stat.deletes,
112 
113  stat.rangeSaysNo,
114  stat.bloomSaysNotIn,
115  stat.rangeFails,
116 
117  stat.swaps,
118  stat.fileLoads,
119  getUsedKB(),
120  stat.fileKB,
121 
122  stat.minKey,
123  stat.maxKey
124  );
125 }
126 
127 // -------------------------------------------------------------------------
128 bool catchTown(PlaceData & dummy, const uint64_t & osmid, double & lon, double & lat, const Tags & tags) {
129  if (tags.find("place") == tags.end()) return false;
130 
131  string pt = tags.at("place");
132  dummy._type = 1;
133 
134  if (pt.compare("village") == 0) {
135  dummy._type = 2;
136  } else if (pt.compare("town") == 0) {
137  dummy._type = 3;
138  } else if (pt.compare("city") == 0) {
139  dummy._type = 4;
140  }
141  dummy._lon = lon;
142  dummy._lat = lat;
143 
144  if (tags.find("name") != tags.end()) {
145  pt = tags.at("name");
146  pt = replacer(pt, "&", "und");
147  pt = replacer(pt, "\"", "'");
148  pt = replacer(pt, "<", "");
149  pt = replacer(pt, ">", "");
150  } else {
151  pt = (string) "Dingenskirchen";
152  }
153  snprintf(dummy._name, 256, "%s", pt.c_str());
154  return true;
155 }
156 
157 // -------------------------------------------------------------------------
158 bool catchWay(WayData & dummy, const uint64_t & osmid, const Tags & tags) {
159  if (tags.find("highway") == tags.end()) return false;
160 
161  dummy._type = 0;
162  string wt = tags.at("highway");
163 
164  if (wt.compare("motorway") == 0) {
165  dummy._type = 10;
166  } else if (wt.compare("motorway_link") == 0) {
167  dummy._type = 10;
168  } else if (wt.compare("trunk") == 0) {
169  dummy._type = 20;
170  } else if (wt.compare("trunk_link") == 0) {
171  dummy._type = 20;
172  } else if (wt.compare("primary") == 0) {
173  dummy._type = 30;
174  } else if (wt.compare("primary_link") == 0) {
175  dummy._type = 30;
176  } else if (wt.compare("secondary") == 0) {
177  dummy._type = 40;
178  } else if (wt.compare("unclassified") == 0) {
179  dummy._type = 50;
180  } else if (wt.compare("tertiary") == 0) {
181  dummy._type = 50;
182  } else if (wt.compare("residential") == 0) {
183  dummy._type = 60;
184  }
185 
186  if (dummy._type == 0) return false;
187 
188  string wayName = "";
189  if(tags.find("name") != tags.end()){
190  wayName = tags.at("name");
191  } else if (tags.find("ref") != tags.end()) {
192  wayName = tags.at("ref");
193  } else if (tags.find("destination:ref") != tags.end() && tags.find("destination") != tags.end()) {
194  wayName = tags.at("destination:ref") + (string) " Richtung " + tags.at("destination");
195  }
196  wayName = replacer(wayName, "&", "und");
197  wayName = replacer(wayName, "\"", "'");
198  wayName = replacer(wayName, "<", "");
199  wayName = replacer(wayName, ">", "");
200 
201  snprintf(dummy._name, 256, "%s", wayName.c_str());
202 
203  return true;
204 }
205 
206 
207 
SwappyItemsPLACES * places
Definition: ReadPbfData.cpp:56
double mseconds
Definition: ReadPbfData.cpp:61
std::chrono::time_point< std::chrono::system_clock > start
Definition: ReadPbfData.cpp:60
struct statistic_s getStatistic(void)
Definition: SwappyItems.hpp:288
std::map< std::string, std::string > Tags
Definition: osmpbfreader.hpp:51
#define RAM_MULTI
Definition: osm2graph.cpp:15
Ways_t * ways
Definition: osm2graph.cpp:84
#define BBITS
Definition: osm2graph.cpp:16
#define FILE_ITEMS
Definition: osm2graph.cpp:13
#define FILE_MULTI
Definition: osm2graph.cpp:14
#define BMASK
Definition: osm2graph.cpp:17
Nodes_t * nodes
Definition: osm2graph.cpp:85
double pos
Definition: osm2graph_bitmap.cpp:125
Definition: ReadPbfData.cpp:44
double _lat
Definition: ReadPbfData.cpp:47
uint8_t _type
Definition: ReadPbfData.cpp:45
char _name[256]
Definition: ReadPbfData.cpp:48
double _lon
Definition: ReadPbfData.cpp:46
Definition: osm2graph.cpp:60
uint8_t _type
Definition: ReadPbfData.cpp:40
char _name[256]
Definition: ReadPbfData.cpp:41
void my_handler(int s)
Definition: tools.hpp:48
int parseLine(char *line)
Definition: tools.hpp:21
void logHead()
Definition: tools.hpp:71
string replacer(string &s, const string &toReplace, const string &replaceWith)
Definition: tools.hpp:5
void logEntry(double &msec, std::chrono::time_point< std::chrono::system_clock > &old, atomic< bool > &isprnt)
Definition: tools.hpp:81
bool catchWay(WayData &dummy, const uint64_t &osmid, const Tags &tags)
Definition: tools.hpp:158
bool catchTown(PlaceData &dummy, const uint64_t &osmid, double &lon, double &lat, const Tags &tags)
Definition: tools.hpp:128
int getUsedKB()
Definition: tools.hpp:32
void catchSig()
Definition: tools.hpp:62