Libosmium  2.11.0
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <cstddef>
39 #include <stdexcept>
40 #include <system_error>
41 
43 #include <osmium/util/file.hpp>
44 
45 #ifndef _WIN32
46 # include <sys/mman.h>
47 #else
48 # include <fcntl.h>
49 # include <io.h>
50 # include <windows.h>
51 # include <sys/types.h>
52 #endif
53 
54 namespace osmium {
55 
56  namespace util {
57 
95  class MemoryMapping {
96 
97  public:
98 
99  enum class mapping_mode {
100  readonly = 0,
101  write_private = 1,
102  write_shared = 2
103  };
104 
105  private:
106 
108  size_t m_size;
109 
111  off_t m_offset;
112 
114  int m_fd;
115 
118 
119 #ifdef _WIN32
120  HANDLE m_handle;
121 #endif
122 
124  void* m_addr;
125 
126  bool is_valid() const noexcept;
127 
128  void make_invalid() noexcept;
129 
130 #ifdef _WIN32
131  using flag_type = DWORD;
132 #else
133  using flag_type = int;
134 #endif
135 
136  flag_type get_protection() const noexcept;
137 
138  flag_type get_flags() const noexcept;
139 
140  static size_t check_size(size_t size) {
141  if (size == 0) {
142  throw std::runtime_error("Zero-sized mapping is not allowed.");
143  }
144  return size;
145  }
146 
147 #ifdef _WIN32
148  HANDLE get_handle() const noexcept;
149  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
150  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
151 #endif
152 
153  int resize_fd(int fd) {
154  // Anonymous mapping doesn't need resizing.
155  if (fd == -1) {
156  return -1;
157  }
158 
159  // Make sure the file backing this mapping is large enough.
160  if (osmium::util::file_size(fd) < m_size + m_offset) {
161  osmium::util::resize_file(fd, m_size + m_offset);
162  }
163  return fd;
164  }
165 
166  public:
167 
184  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
185 
191  OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
192  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
193  }
194 
196  MemoryMapping(const MemoryMapping&) = delete;
197 
199  MemoryMapping& operator=(const MemoryMapping&) = delete;
200 
205  MemoryMapping(MemoryMapping&& other);
206 
211 
216  ~MemoryMapping() noexcept {
217  try {
218  unmap();
219  } catch (const std::system_error&) {
220  // Ignore any exceptions because destructor must not throw.
221  }
222  }
223 
230  void unmap();
231 
243  void resize(size_t new_size);
244 
249  explicit operator bool() const noexcept {
250  return is_valid();
251  }
252 
258  size_t size() const noexcept {
259  return m_size;
260  }
261 
267  int fd() const noexcept {
268  return m_fd;
269  }
270 
274  bool writable() const noexcept {
275  return m_mapping_mode != mapping_mode::readonly;
276  }
277 
283  template <typename T = void>
284  T* get_addr() const {
285  if (is_valid()) {
286  return reinterpret_cast<T*>(m_addr);
287  }
288  throw std::runtime_error("invalid memory mapping");
289  }
290 
291  }; // class MemoryMapping
292 
304 
305  public:
306 
307  explicit AnonymousMemoryMapping(size_t size) :
309  }
310 
311 #ifndef __linux__
312 
316  void resize(size_t) = delete;
317 #endif
318 
319  }; // class AnonymousMemoryMapping
320 
330  template <typename T>
332 
334 
335  public:
336 
343  explicit TypedMemoryMapping(size_t size) :
344  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
345  }
346 
357  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
358  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
359  }
360 
366  OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
367  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
368  }
369 
371  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
372 
375 
380  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
381 
385  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
386 
391  ~TypedMemoryMapping() noexcept = default;
392 
399  void unmap() {
400  m_mapping.unmap();
401  }
402 
413  void resize(size_t new_size) {
414  m_mapping.resize(sizeof(T) * new_size);
415  }
416 
421  explicit operator bool() const noexcept {
422  return !!m_mapping;
423  }
424 
430  size_t size() const noexcept {
431  assert(m_mapping.size() % sizeof(T) == 0);
432  return m_mapping.size() / sizeof(T);
433  }
434 
440  int fd() const noexcept {
441  return m_mapping.fd();
442  }
443 
447  bool writable() const noexcept {
448  return m_mapping.writable();
449  }
450 
456  T* begin() {
457  return m_mapping.get_addr<T>();
458  }
459 
465  T* end() {
466  return m_mapping.get_addr<T>() + size();
467  }
468 
469  const T* cbegin() const {
470  return m_mapping.get_addr<T>();
471  }
472 
473  const T* cend() const {
474  return m_mapping.get_addr<T>() + size();
475  }
476 
477  const T* begin() const {
478  return m_mapping.get_addr<T>();
479  }
480 
481  const T* end() const {
482  return m_mapping.get_addr<T>() + size();
483  }
484 
485  }; // class TypedMemoryMapping
486 
487  template <typename T>
489 
490  public:
491 
493  TypedMemoryMapping<T>(size) {
494  }
495 
496 #ifndef __linux__
497 
501  void resize(size_t) = delete;
502 #endif
503 
504  }; // class AnonymousTypedMemoryMapping
505 
506  } // namespace util
507 
508 } // namespace osmium
509 
510 #ifndef _WIN32
511 
512 // =========== Unix implementation =============
513 
514 // MAP_FAILED is often a macro containing an old style cast
515 #pragma GCC diagnostic push
516 #pragma GCC diagnostic ignored "-Wold-style-cast"
517 
518 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
519  return m_addr != MAP_FAILED;
520 }
521 
523  m_addr = MAP_FAILED;
524 }
525 
526 #pragma GCC diagnostic pop
527 
528 // for BSD systems
529 #ifndef MAP_ANONYMOUS
530 # define MAP_ANONYMOUS MAP_ANON
531 #endif
532 
533 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
535  return PROT_READ;
536  }
537  return PROT_READ | PROT_WRITE;
538 }
539 
540 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
541  if (m_fd == -1) {
542  return MAP_PRIVATE | MAP_ANONYMOUS;
543  }
545  return MAP_SHARED;
546  }
547  return MAP_PRIVATE;
548 }
549 
550 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
551  m_size(check_size(size)),
552  m_offset(offset),
553  m_fd(resize_fd(fd)),
554  m_mapping_mode(mode),
555  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
556  assert(!(fd == -1 && mode == mapping_mode::readonly));
557  if (!is_valid()) {
558  throw std::system_error(errno, std::system_category(), "mmap failed");
559  }
560 }
561 
563  m_size(other.m_size),
564  m_offset(other.m_offset),
565  m_fd(other.m_fd),
567  m_addr(other.m_addr) {
568  other.make_invalid();
569 }
570 
572  unmap();
573  m_size = other.m_size;
574  m_offset = other.m_offset;
575  m_fd = other.m_fd;
576  m_mapping_mode = other.m_mapping_mode;
577  m_addr = other.m_addr;
578  other.make_invalid();
579  return *this;
580 }
581 
583  if (is_valid()) {
584  if (::munmap(m_addr, m_size) != 0) {
585  throw std::system_error(errno, std::system_category(), "munmap failed");
586  }
587  make_invalid();
588  }
589 }
590 
591 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
592  assert(new_size > 0 && "can not resize to zero size");
593  if (m_fd == -1) { // anonymous mapping
594 #ifdef __linux__
595  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
596  if (!is_valid()) {
597  throw std::system_error(errno, std::system_category(), "mremap failed");
598  }
599  m_size = new_size;
600 #else
601  assert(false && "can't resize anonymous mappings on non-linux systems");
602 #endif
603  } else { // file-based mapping
604  unmap();
605  m_size = new_size;
606  resize_fd(m_fd);
607  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
608  if (!is_valid()) {
609  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
610  }
611  }
612 }
613 
614 #else
615 
616 // =========== Windows implementation =============
617 
618 /* References:
619  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
620  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
621  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
622  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
623  */
624 
625 namespace osmium {
626 
627  namespace util {
628 
629  inline DWORD dword_hi(uint64_t x) {
630  return static_cast<DWORD>(x >> 32);
631  }
632 
633  inline DWORD dword_lo(uint64_t x) {
634  return static_cast<DWORD>(x & 0xffffffff);
635  }
636 
637  } // namespace util
638 
639 } // namespace osmium
640 
641 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
642  switch (m_mapping_mode) {
644  return PAGE_READONLY;
646  return PAGE_WRITECOPY;
648  return PAGE_READWRITE;
649  }
650 }
651 
652 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
653  switch (m_mapping_mode) {
655  return FILE_MAP_READ;
657  return FILE_MAP_COPY;
659  return FILE_MAP_WRITE;
660  }
661 }
662 
663 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
664  if (m_fd == -1) {
665  return INVALID_HANDLE_VALUE;
666  }
667  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
668 }
669 
670 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
671  if (m_fd != -1) {
672  _setmode(m_fd, _O_BINARY);
673  }
674  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
675 }
676 
677 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
678  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
679 }
680 
681 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
682  return m_addr != nullptr;
683 }
684 
685 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
686  m_addr = nullptr;
687 }
688 
689 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
690  m_size(check_size(size)),
691  m_offset(offset),
692  m_fd(resize_fd(fd)),
693  m_mapping_mode(mode),
694  m_handle(create_file_mapping()),
695  m_addr(nullptr) {
696 
697  if (!m_handle) {
698  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
699  }
700 
701  m_addr = map_view_of_file();
702  if (!is_valid()) {
703  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
704  }
705 }
706 
708  m_size(other.m_size),
709  m_offset(other.m_offset),
710  m_fd(other.m_fd),
711  m_mapping_mode(other.m_mapping_mode),
712  m_handle(std::move(other.m_handle)),
713  m_addr(other.m_addr) {
714  other.make_invalid();
715  other.m_handle = nullptr;
716 }
717 
719  unmap();
720  m_size = other.m_size;
721  m_offset = other.m_offset;
722  m_fd = other.m_fd;
723  m_mapping_mode = other.m_mapping_mode;
724  m_handle = std::move(other.m_handle);
725  m_addr = other.m_addr;
726  other.make_invalid();
727  other.m_handle = nullptr;
728  return *this;
729 }
730 
732  if (is_valid()) {
733  if (! UnmapViewOfFile(m_addr)) {
734  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
735  }
736  make_invalid();
737  }
738 
739  if (m_handle) {
740  if (! CloseHandle(m_handle)) {
741  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
742  }
743  m_handle = nullptr;
744  }
745 }
746 
747 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
748  unmap();
749 
750  m_size = new_size;
751  resize_fd(m_fd);
752 
753  m_handle = create_file_mapping();
754  if (!m_handle) {
755  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
756  }
757 
758  m_addr = map_view_of_file();
759  if (!is_valid()) {
760  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
761  }
762 }
763 
764 #endif
765 
766 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:216
bool is_valid() const noexcept
Definition: memory_mapping.hpp:518
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:533
MemoryMapping m_mapping
Definition: memory_mapping.hpp:333
int flag_type
Definition: memory_mapping.hpp:133
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:550
size_t file_size(int fd)
Definition: file.hpp:67
const T * begin() const
Definition: memory_mapping.hpp:477
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:540
int fd() const noexcept
Definition: memory_mapping.hpp:267
void unmap()
Definition: memory_mapping.hpp:582
Definition: memory_mapping.hpp:95
mapping_mode
Definition: memory_mapping.hpp:99
void resize(size_t new_size)
Definition: memory_mapping.hpp:413
static size_t check_size(size_t size)
Definition: memory_mapping.hpp:140
int resize_fd(int fd)
Definition: memory_mapping.hpp:153
T * end()
Definition: memory_mapping.hpp:465
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:124
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:111
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:530
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:114
size_t size() const noexcept
Definition: memory_mapping.hpp:258
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:117
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:307
Definition: memory_mapping.hpp:488
OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:366
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:357
size_t size() const noexcept
Definition: memory_mapping.hpp:430
T * begin()
Definition: memory_mapping.hpp:456
Definition: memory_mapping.hpp:303
void make_invalid() noexcept
Definition: memory_mapping.hpp:522
const T * cbegin() const
Definition: memory_mapping.hpp:469
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:492
int fd() const noexcept
Definition: memory_mapping.hpp:440
void resize(size_t new_size)
Definition: memory_mapping.hpp:591
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:331
const T * end() const
Definition: memory_mapping.hpp:481
void unmap()
Definition: memory_mapping.hpp:399
bool writable() const noexcept
Definition: memory_mapping.hpp:274
void resize_file(int fd, size_t new_size)
Definition: file.hpp:132
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:108
T * get_addr() const
Definition: memory_mapping.hpp:284
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:343
OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:191
const T * cend() const
Definition: memory_mapping.hpp:473
bool writable() const noexcept
Definition: memory_mapping.hpp:447