Skip to content

Commit

Permalink
Pad the file for windows.
Browse files Browse the repository at this point in the history
  • Loading branch information
trivialfis committed Jun 13, 2023
1 parent 341c8fb commit 2660c66
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 16 deletions.
14 changes: 11 additions & 3 deletions rabit/include/rabit/internal/io.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
*/
#ifndef RABIT_INTERNAL_IO_H_
#define RABIT_INTERNAL_IO_H_

#if !defined(NOMINMAX) && defined(_WIN32)
#define NOMINMAX
#endif // !defined(NOMINMAX)

#include <cstdio>
#include <vector>
#include <cstring>
Expand All @@ -26,6 +31,9 @@ struct MemoryFixSizeBuffer : public SeekStream {
// similar to SEEK_END in libc
static size_t constexpr kSeekEnd = std::numeric_limits<size_t>::max();

protected:
MemoryFixSizeBuffer() = default;

public:
MemoryFixSizeBuffer(void *p_buffer, size_t buffer_size)
: p_buffer_(reinterpret_cast<char*>(p_buffer)),
Expand Down Expand Up @@ -62,11 +70,11 @@ struct MemoryFixSizeBuffer : public SeekStream {

protected:
/*! \brief in memory buffer */
char *p_buffer_;
char* p_buffer_{nullptr};
/*! \brief current pointer */
size_t buffer_size_;
std::size_t buffer_size_{ 0 };
/*! \brief current pointer */
size_t curr_ptr_;
std::size_t curr_ptr_{ 0 };
}; // class MemoryFixSizeBuffer

/*! \brief a in memory buffer that can be read and write as stream interface */
Expand Down
37 changes: 26 additions & 11 deletions src/common/io.cc
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <sys/stat.h>
#include <unistd.h> // for close, getpagesize
#elif defined(_MSC_VER)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif // defined(__unix__)

Expand All @@ -26,6 +27,7 @@

#include "io.h"
#include "xgboost/logging.h"
#include "xgboost/collective/socket.h"

namespace xgboost {
namespace common {
Expand Down Expand Up @@ -170,7 +172,8 @@ std::size_t GetPageSize() {
#if defined(_MSC_VER)
SYSTEM_INFO sys_info;
GetSystemInfo(&sys_info);
return sys_info.dwPageSize;
// During testing, `sys_info.dwPageSize` is of size 4096 while `dwAllocationGranularity` is of size 65536.
return sys_info.dwAllocationGranularity;
#else
return getpagesize();
#endif
Expand All @@ -190,18 +193,21 @@ std::size_t PadPageForMmap(std::size_t file_bytes, dmlc::Stream* fo) {

struct PrivateMmapStream::MMAPFile {
#if defined(_MSC_VER)
HANDLE fd;
HANDLE fd{ INVALID_HANDLE_VALUE };
#else
std::int32_t fd;
std::int32_t fd {0};
#endif
std::string path;
};

PrivateMmapStream::PrivateMmapStream(std::string path, bool read_only, std::size_t offset,
std::size_t length)
: MemoryFixSizeBuffer{Open(std::move(path), read_only, offset, length), length} {}
: MemoryFixSizeBuffer{} {
this->p_buffer_ = Open(std::move(path), read_only, offset, length);
this->buffer_size_ = length;
}

void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offset,
char* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offset,
std::size_t length) {
#if defined(_MSC_VER)
HANDLE fd = CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
Expand All @@ -211,7 +217,8 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs
auto fd = open(path.c_str(), O_RDONLY);
CHECK_GE(fd, 0) << "Failed to open:" << path << ". " << strerror(errno);
#endif
handle_.reset(new MMAPFile{fd, std::move(path)});
handle_ = nullptr;
handle_.reset(new MMAPFile{fd, path});

void* ptr{nullptr};
#if defined(__linux__) || defined(__GLIBC__)
Expand All @@ -228,8 +235,12 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs
access = read_only ? FILE_MAP_READ : FILE_MAP_ALL_ACCESS;
std::uint32_t loff = static_cast<std::uint32_t>(offset);
std::uint32_t hoff = offset >> 32;
CHECK(map_file) << "Failed to map: " << handle_->path << ". " << GetLastError();;
ptr = MapViewOfFile(map_file, access, hoff, loff, length);
CHECK_NE(ptr, nullptr) << "Failed to map: " << handle_->path << ". " << GetLastError();
if (ptr == nullptr) {
system::ThrowAtError("MapViewOfFile");
}
CHECK_NE(ptr, nullptr) << "Failed to map: " << handle_->path << ". " << GetLastError() ;
#else
CHECK_LE(offset, std::numeric_limits<off_t>::max())
<< "File size has exceeded the limit on the current system.";
Expand All @@ -240,18 +251,22 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs
ptr = reinterpret_cast<char*>(mmap(nullptr, length, prot, MAP_PRIVATE, fd_, offset));
CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << handle_->path << ". " << strerror(errno);
#endif // defined(__linux__)
return ptr;
return reinterpret_cast<char*>(ptr);
}

PrivateMmapStream::~PrivateMmapStream() {
CHECK(handle_);
#if defined(_MSC_VER)
CHECK(UnmapViewOfFile(p_buffer_)) "Faled to munmap." << handle_->path << ". " << GetLastError();
CloseHandle(handle_->fd);
if (p_buffer_) {
CHECK(UnmapViewOfFile(p_buffer_)) "Faled to munmap." << GetLastError();
}
if (handle_->fd != INVALID_HANDLE_VALUE) {
CHECK(CloseHandle(handle_->fd));
}
#else
CHECK_NE(munmap(p_buffer_, buffer_size_), -1)
<< "Faled to munmap." << handle_->path << ". " << strerror(errno);
CHECK_NE(close(fd_), -1) << "Faled to close: " << handle_->path << ". " << strerror(errno);
CHECK_NE(close(handle_->fd), -1) << "Faled to close: " << handle_->path << ". " << strerror(errno);
#endif
}
} // namespace common
Expand Down
4 changes: 2 additions & 2 deletions src/common/io.h
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,9 @@ std::size_t PadPageForMmap(std::size_t file_bytes, dmlc::Stream* fo);
class PrivateMmapStream : public MemoryFixSizeBuffer {
struct MMAPFile;

std::unique_ptr<MMAPFile> handle_;
std::unique_ptr<MMAPFile> handle_{nullptr};

void* Open(std::string path, bool read_only, std::size_t offset, std::size_t length);
char* Open(std::string path, bool read_only, std::size_t offset, std::size_t length);

public:
/**
Expand Down

0 comments on commit 2660c66

Please sign in to comment.