Merge pull request #5583 from 0xdky/dhruva/build-with-nommap Build with NO_MMAP
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 8e35726..fa83491 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -96,6 +96,17 @@ jobs:
ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
TSAN_OPTIONS: suppressions=/home/libgit2/source/script/thread-sanitizer.supp second_deadlock_stack=1
os: ubuntu-latest
+ - # Focal, Clang 10, mmap emulation (NO_MMAP)
+ container:
+ name: focal
+ env:
+ CC: clang-10
+ CFLAGS: -DNO_MMAP
+ CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local
+ CMAKE_GENERATOR: Ninja
+ SKIP_SSH_TESTS: true
+ SKIP_NEGOTIATE_TESTS: true
+ os: ubuntu-latest
- # macOS
os: macos-10.15
env:
@@ -114,6 +125,15 @@ jobs:
CMAKE_OPTIONS: -A x64 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON
SKIP_SSH_TESTS: true
SKIP_NEGOTIATE_TESTS: true
+ - # Windows amd64 Visual Studio (NO_MMAP)
+ os: windows-2019
+ env:
+ ARCH: amd64
+ CMAKE_GENERATOR: Visual Studio 16 2019
+ CFLAGS: -DNO_MMAP
+ CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON
+ SKIP_SSH_TESTS: true
+ SKIP_NEGOTIATE_TESTS: true
- # Windows x86 Visual Studio
os: windows-2019
env:
diff --git a/src/indexer.c b/src/indexer.c
index 453bb3d..ba82d71 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -603,6 +603,23 @@ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t size)
{
+#ifdef NO_MMAP
+ size_t remaining_size = size;
+ const char *ptr = (const char *)data;
+
+ /* Handle data size larger that ssize_t */
+ while (remaining_size > 0) {
+ ssize_t nb;
+ HANDLE_EINTR(nb, p_pwrite(idx->pack->mwf.fd, (void *)ptr,
+ remaining_size, offset));
+ if (nb <= 0)
+ return -1;
+
+ ptr += nb;
+ offset += nb;
+ remaining_size -= nb;
+ }
+#else
git_file fd = idx->pack->mwf.fd;
size_t mmap_alignment;
size_t page_offset;
@@ -627,6 +644,7 @@ static int write_at(git_indexer *idx, const void *data, off64_t offset, size_t s
map_data = (unsigned char *)map.data;
memcpy(map_data + page_offset, data, size);
p_munmap(&map);
+#endif
return 0;
}
diff --git a/src/posix.c b/src/posix.c
index 9cd96d2..bf764ae 100644
--- a/src/posix.c
+++ b/src/posix.c
@@ -238,24 +238,43 @@ int git__mmap_alignment(size_t *alignment)
int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, off64_t offset)
{
+ const char *ptr;
+ size_t remaining_len;
+
GIT_MMAP_VALIDATE(out, len, prot, flags);
- out->data = NULL;
- out->len = 0;
+ /* writes cannot be emulated without handling pagefaults since write happens by
+ * writing to mapped memory */
+ if (prot & GIT_PROT_WRITE) {
+ git_error_set(GIT_ERROR_OS, "trying to map %s-writeable",
+ ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED) ? "shared": "private");
+ return -1;
+ }
- if ((prot & GIT_PROT_WRITE) && ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)) {
- git_error_set(GIT_ERROR_OS, "trying to map shared-writeable");
+ if (!git__is_ssizet(len)) {
+ errno = EINVAL;
return -1;
}
+ out->len = 0;
out->data = git__malloc(len);
GIT_ERROR_CHECK_ALLOC(out->data);
- if (!git__is_ssizet(len) ||
- (p_lseek(fd, offset, SEEK_SET) < 0) ||
- (p_read(fd, out->data, len) != (ssize_t)len)) {
- git_error_set(GIT_ERROR_OS, "mmap emulation failed");
- return -1;
+ remaining_len = len;
+ ptr = (const char *)out->data;
+ while (remaining_len > 0) {
+ ssize_t nb;
+ HANDLE_EINTR(nb, p_pread(fd, (void *)ptr, remaining_len, offset));
+ if (nb <= 0) {
+ git_error_set(GIT_ERROR_OS, "mmap emulation failed");
+ git__free(out->data);
+ out->data = NULL;
+ return -1;
+ }
+
+ ptr += nb;
+ offset += nb;
+ remaining_len -= nb;
}
out->len = len;
@@ -267,6 +286,10 @@ int p_munmap(git_map *map)
GIT_ASSERT_ARG(map);
git__free(map->data);
+ /* Initializing will help debug use-after-free */
+ map->len = 0;
+ map->data = NULL;
+
return 0;
}
diff --git a/src/posix.h b/src/posix.h
index eef6677..d98bc82 100644
--- a/src/posix.h
+++ b/src/posix.h
@@ -89,6 +89,12 @@
#define EAFNOSUPPORT (INT_MAX-1)
#endif
+/* Compiler independent macro to handle signal interrpted system calls */
+#define HANDLE_EINTR(result, x) do { \
+ result = (x); \
+ } while (result == -1 && errno == EINTR);
+
+
/* Provide a 64-bit size for offsets. */
#if defined(_MSC_VER)
@@ -119,6 +125,9 @@ typedef int git_file;
extern ssize_t p_read(git_file fd, void *buf, size_t cnt);
extern int p_write(git_file fd, const void *buf, size_t cnt);
+extern ssize_t p_pread(int fd, void *data, size_t size, off64_t offset);
+extern ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset);
+
#define p_close(fd) close(fd)
#define p_umask(m) umask(m)
diff --git a/src/unix/posix.h b/src/unix/posix.h
index b5527a4..7b3325e 100644
--- a/src/unix/posix.h
+++ b/src/unix/posix.h
@@ -101,4 +101,7 @@ GIT_INLINE(int) p_futimes(int f, const struct p_timeval t[2])
# define p_futimes futimes
#endif
+#define p_pread(f, d, s, o) pread(f, d, s, o)
+#define p_pwrite(f, d, s, o) pwrite(f, d, s, o)
+
#endif
diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c
index 5a5e921..0a8f2be 100644
--- a/src/win32/posix_w32.c
+++ b/src/win32/posix_w32.c
@@ -981,3 +981,73 @@ int p_inet_pton(int af, const char *src, void *dst)
errno = EINVAL;
return -1;
}
+
+ssize_t p_pread(int fd, void *data, size_t size, off64_t offset)
+{
+ HANDLE fh;
+ DWORD rsize = 0;
+ OVERLAPPED ov = {0};
+ LARGE_INTEGER pos = {0};
+ off64_t final_offset = 0;
+
+ /* Fail if the final offset would have overflowed to match POSIX semantics. */
+ if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ /*
+ * Truncate large writes to the maximum allowable size: the caller
+ * needs to always call this in a loop anyways.
+ */
+ if (size > INT32_MAX) {
+ size = INT32_MAX;
+ }
+
+ pos.QuadPart = offset;
+ ov.Offset = pos.LowPart;
+ ov.OffsetHigh = pos.HighPart;
+ fh = (HANDLE)_get_osfhandle(fd);
+
+ if (ReadFile(fh, data, (DWORD)size, &rsize, &ov)) {
+ return (ssize_t)rsize;
+ }
+
+ set_errno();
+ return -1;
+}
+
+ssize_t p_pwrite(int fd, const void *data, size_t size, off64_t offset)
+{
+ HANDLE fh;
+ DWORD wsize = 0;
+ OVERLAPPED ov = {0};
+ LARGE_INTEGER pos = {0};
+ off64_t final_offset = 0;
+
+ /* Fail if the final offset would have overflowed to match POSIX semantics. */
+ if (!git__is_ssizet(size) || git__add_int64_overflow(&final_offset, offset, (int64_t)size)) {
+ errno = EINVAL;
+ return -1;
+ }
+
+ /*
+ * Truncate large writes to the maximum allowable size: the caller
+ * needs to always call this in a loop anyways.
+ */
+ if (size > INT32_MAX) {
+ size = INT32_MAX;
+ }
+
+ pos.QuadPart = offset;
+ ov.Offset = pos.LowPart;
+ ov.OffsetHigh = pos.HighPart;
+ fh = (HANDLE)_get_osfhandle(fd);
+
+ if (WriteFile(fh, data, (DWORD)size, &wsize, &ov)) {
+ return (ssize_t)wsize;
+ }
+
+ set_errno();
+ return -1;
+}