mman: Check mmap() MAP_SHARED and MAP_PRIVATE flags

Signed-off-by: Pekka Enberg <penberg@cloudius-systems.com>

libc/mman.cc

@@ -13,6 +13,7 @@
 #include "libc/libc.hh"
 
 TRACEPOINT(trace_memory_mmap, "addr=%p, length=%d, prot=%d, flags=%d, fd=%d, offset=%d", void *, size_t, int, int, int, off_t);
+TRACEPOINT(trace_memory_mmap_err, "%d", int);
 TRACEPOINT(trace_memory_mmap_ret, "%p", void *);
 TRACEPOINT(trace_memory_munmap, "addr=%p, length=%d", void *, size_t);
 
@@ -57,6 +58,12 @@ void *mmap(void *addr, size_t length, int prot, int flags,
 {
     trace_memory_mmap(addr, length, prot, flags, fd, offset);
 
+    if (!(flags & (MAP_SHARED|MAP_PRIVATE))) {
+        errno = EINVAL;
+        trace_memory_mmap_err(errno);
+        return MAP_FAILED;
+    }
+
     // TODO: should fail with EINVAL in some cases of addr, length, offset.
 
     // make use the payload isn't remapping physical memory

tests/tst-mmap.cc

@@ -107,6 +107,10 @@ int main(int argc, char **argv)
     }
     munmap(buf, hugepagesize*9+4096);
 
+    // Test for missing MAP_PRIVATE or MAP_SHARED flag
+    buf = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_ANONYMOUS, -1, 0);
+    assert(buf == MAP_FAILED);
+
     // test mprotect making a read-only page.
     buf = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
     assert(buf != MAP_FAILED);