Monday, August 22, 2016

Mongodb: mmap private failed with out of memory. (64 bit build)

Leave a Comment

Mongo seems to have run out of memory!? What can I do?

I am not even sure it's actually a memory problem.

2016-08-11T14:23:26.875+0200 E CONTROL  [conn2575] mmap private failed with out of memory. (64 bit build) 2016-08-11T14:23:26.875+0200 I -        [conn2575] Assertion: 13636:file /data/db/project.6 open/create failed in createPrivateMap (look in log for more information) 2016-08-11T14:23:26.892+0200 I CONTROL  [conn2575]  0xf9b632 0xf36699 0xf1a22e 0xf1a2dc 0xd56925 0xd56b5f 0xd3cf06 0xd6ff6d 0xd70380 0xd7061d 0xd74ec2 0xd80b34 0xd756ab 0xd75859 0x90a811 0x90b3ec 0xa3e5fc 0xa3f334 0xbe8012 0xbe832b 0xbe8a0d 0x9b1769 0x9b2fc5 0x9b35f5 0x9b62dc 0x9d9071 0x9da07c 0x9dad4b 0xbb544b 0xac3e4a 0x7fd9c5 0xf4af79 0x7f9122c60aa1 0x7f91217b293d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B9B632","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B36699","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B1A22E","s":"_ZN5mongo11msgassertedEiPKc"},{"b":"400000","o":"B1A2DC"},{"b":"400000","o":"956925","s":"_ZN5mongo17DurableMappedFile13finishOpeningEv"},{"b":"400000","o":"956B5F","s":"_ZN5mongo17DurableMappedFile6createERKSsRyb"},{"b":"400000","o":"93CF06","s":"_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib"},{"b":"400000","o":"96FF6D","s":"_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib"},{"b":"400000","o":"970380","s":"_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib"},{"b":"400000","o":"97061D","s":"_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib"},{"b":"400000","o":"974EC2","s":"_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib"},{"b":"400000","o":"980B34","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"9756AB","s":"_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"975859","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"50A811","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"50B3EC","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"63E5FC","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"63F334","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"7E8012","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"7E832B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"7E8A0D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5B1769","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5B2FC5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5B35F5","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE"},{"b":"400000","o":"5B62DC","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5D9071","s":"_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DA07C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DAD4B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7B544B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"400000","o":"6C3E4A","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"3FD9C5","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B4AF79","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7F9122C59000","o":"7AA1"},{"b":"7F91216CA000","o":"E893D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.9", "gitVersion" : "20d60d3491908f1ae252fe452300de3978a040c7", "uname" : { "sysname" : "Linux", "release" : "2.6.32-042stab113.11", "version" : "#1 SMP Fri Dec 18 17:32:04 MSK 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "47DFD0896283C6BF009395E08B673D3246CC7AE5" }, { "b" : "7FFD9BE1E000", "elfType" : 3, "buildId" : "F466F3FAB4F73E9911206E2486A3C5F8FCCA8EA5" }, { "b" : "7F9122C59000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "C56DD1B811FC0D9263248EBB308C73FCBCD80FC1" }, { "b" : "7F91229ED000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "24A4E7055CABC29C2DDAC730D635A20C08687AC6" }, { "b" : "7F912260A000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F6F5BB27A852F29A134680794DD2EB5C3CC5D88B" }, { "b" : "7F9122402000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "95159178F1A4A3DBDC7819FBEA2C80E5FCDD6BAC" }, { "b" : "7F91221FE000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "29B61382141595ECBA6576232E44F2310C3AAB72" }, { "b" : "7F9121EF8000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "b" : "7F9121C74000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "b" : "7F9121A5E000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "b" : "7F91216CA000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "8E6FA4C4B0594C355C1B90C1D49990368C81A040" }, { "b" : "7F9122E76000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "959C5E10A47EE8A633E7681B64B4B9F74E242ED5" }, { "b" : "7F9121486000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "b" : "7F912119F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "b" : "7F9120F9B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "b" : "7F9120D6F000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "b" : "7F9120B59000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F912094E000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "b" : "7F912074B000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9120531000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C39D7FFB49DFB1B55AD09D1D711AD802123F6623" }, { "b" : "7F9120312000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}  mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xf9b632]  mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf36699]  mongod(_ZN5mongo11msgassertedEiPKc+0xAE) [0xf1a22e]  mongod(+0xB1A2DC) [0xf1a2dc]  mongod(_ZN5mongo17DurableMappedFile13finishOpeningEv+0x2F5) [0xd56925]  mongod(_ZN5mongo17DurableMappedFile6createERKSsRyb+0x6F) [0xd56b5f]  mongod(_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib+0x106) [0xd3cf06]  mongod(_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib+0x15D) [0xd6ff6d]  mongod(_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib+0xE0) [0xd70380]  mongod(_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib+0x1FD) [0xd7061d]  mongod(_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib+0x42) [0xd74ec2]  mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x1F4) [0xd80b34]  mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd756ab]  mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd75859]  mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x90a811]  mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x90b3ec]  mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa3e5fc]  mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa3f334]  mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xbe8012]  mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xbe832b]  mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbe8a0d]  mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9b1769]  mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x245) [0x9b2fc5]  mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9b35f5]  mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9b62dc]  mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9d9071]  mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9da07c]  mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9dad4b]  mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbb544b]  mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xac3e4a]  mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x7fd9c5]  mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf4af79]  libpthread.so.0(+0x7AA1) [0x7f9122c60aa1]  libc.so.6(clone+0x6D) [0x7f91217b293d] -----  END BACKTRACE  ----- 

ulimit -a

core file size          (blocks, -c) 0 data seg size           (kbytes, -d) unlimited scheduling priority             (-e) 0 file size               (blocks, -f) unlimited pending signals                 (-i) 127948 max locked memory       (kbytes, -l) 64 max memory size         (kbytes, -m) unlimited open files                      (-n) 1024 pipe size            (512 bytes, -p) 8 POSIX message queues     (bytes, -q) 819200 real-time priority              (-r) 0 stack size              (kbytes, -s) 10240 cpu time               (seconds, -t) unlimited max user processes              (-u) 127948 virtual memory          (kbytes, -v) unlimited file locks                      (-x) unlimited 

ulimit -v

unlimited 

swapon -s

Filename                                Type            Size    Used    Priority /dev/null                               partition       524288  175344  -1 

free -m

             total       used       free     shared    buffers     cached Mem:          4096       3331        764         15          0       2985 -/+ buffers/cache:        345       3750 Swap:          512        171        340 

df -h

Filesystem         Size  Used Avail Use% Mounted on /dev/ploop10759p1   40G   26G   12G  69% / none                16G  4.0K   16G   1% /dev none                16G     0   16G   0% /dev/shm  # cat /proc/sys/vm/overcommit_memory 0 

ps aux | grep mongo

webdev    4006  0.2 26.0 9123780 1091292 ?     Sl   14:54   0:06 mongod --logpath /var/log/mongodb/mongod.log webdev    6031  0.0  0.0 103312   872 pts/1    S+   15:35   0:00 grep mongo 

mongod --version

db version v3.0.12 git version: 33934938e0e95d534cebbaff656cde916b9c3573 

cat /etc/centos-release

CentOS release 6.8 (Final) 

db.serverStatus()

{     "host" : "host",     "version" : "3.0.12",     "process" : "mongod",     "pid" : NumberLong(4006),     "uptime" : 3590,     "uptimeMillis" : NumberLong(3590668),     "uptimeEstimate" : 3555,     "localTime" : ISODate("2016-08-11T13:54:51.869Z"),     "asserts" : {         "regular" : 0,         "warning" : 1,         "msg" : 0,         "user" : 0,         "rollovers" : 0     },     "backgroundFlushing" : {         "flushes" : 59,         "total_ms" : 80,         "average_ms" : 1.3559322033898304,         "last_ms" : 1,         "last_finished" : ISODate("2016-08-11T13:54:01.250Z")     },     "connections" : {         "current" : 4,         "available" : 815,         "totalCreated" : NumberLong(6)     },     "cursors" : {         "note" : "deprecated, use server status metrics",         "clientCursors_size" : 0,         "totalOpen" : 0,         "pinned" : 0,         "totalNoTimeout" : 0,         "timedOut" : 0     },     "dur" : {         "commits" : 24,         "journaledMB" : 0,         "writeToDataFilesMB" : 0,         "compression" : 0,         "commitsInWriteLock" : 0,         "earlyCommits" : 0,         "timeMs" : {             "dt" : 0,             "prepLogBuffer" : 0,             "writeToJournal" : 0,             "writeToDataFiles" : 0,             "remapPrivateView" : 0,             "commits" : 0,             "commitsInWriteLock" : 0         }     },     "extra_info" : {         "note" : "fields vary by platform",         "heap_usage_bytes" : 63466248,         "page_faults" : 28     },     "globalLock" : {         "totalTime" : NumberLong("3590669000"),         "currentQueue" : {             "total" : 0,             "readers" : 0,             "writers" : 0         },         "activeClients" : {             "total" : 12,             "readers" : 0,             "writers" : 0         }     },     "locks" : {         "Global" : {             "acquireCount" : {                 "r" : NumberLong(47395),                 "w" : NumberLong(1270),                 "W" : NumberLong(5)             },             "acquireWaitCount" : {                 "r" : NumberLong(1)             },             "timeAcquiringMicros" : {                 "r" : NumberLong(641)             }         },         "MMAPV1Journal" : {             "acquireCount" : {                 "r" : NumberLong(23058),                 "w" : NumberLong(1300),                 "R" : NumberLong(35093)             },             "acquireWaitCount" : {                 "w" : NumberLong(3),                 "R" : NumberLong(3)             },             "timeAcquiringMicros" : {                 "w" : NumberLong(24),                 "R" : NumberLong(63215)             }         },         "Database" : {             "acquireCount" : {                 "r" : NumberLong(23056),                 "w" : NumberLong(1247),                 "R" : NumberLong(4),                 "W" : NumberLong(23)             }         },         "Collection" : {             "acquireCount" : {                 "R" : NumberLong(24059),                 "W" : NumberLong(1247)             }         },         "Metadata" : {             "acquireCount" : {                 "W" : NumberLong(1)             }         }     },     "network" : {         "bytesIn" : NumberLong(28574),         "bytesOut" : NumberLong(14793714),         "numRequests" : NumberLong(229)     },     "opcounters" : {         "insert" : 0,         "query" : 216,         "update" : 1,         "delete" : 0,         "getmore" : 3,         "command" : 11     },     "opcountersRepl" : {         "insert" : 0,         "query" : 0,         "update" : 0,         "delete" : 0,         "getmore" : 0,         "command" : 0     },     "storageEngine" : {         "name" : "mmapv1"     },     "writeBacksQueued" : false,     "mem" : {         "bits" : 64,         "resident" : 1068,         "virtual" : 8909,         "supported" : true,         "mapped" : 4287,         "mappedWithJournal" : 8574     },     "metrics" : {         "commands" : {             "getLog" : {                 "failed" : NumberLong(0),                 "total" : NumberLong(1)             },             "isMaster" : {                 "failed" : NumberLong(0),                 "total" : NumberLong(5)             },             "replSetGetStatus" : {                 "failed" : NumberLong(1),                 "total" : NumberLong(1)             },             "serverStatus" : {                 "failed" : NumberLong(0),                 "total" : NumberLong(2)             },             "update" : {                 "failed" : NumberLong(0),                 "total" : NumberLong(1)             },             "whatsmyuri" : {                 "failed" : NumberLong(0),                 "total" : NumberLong(2)             }         },         "cursor" : {             "timedOut" : NumberLong(0),             "open" : {                 "noTimeout" : NumberLong(0),                 "pinned" : NumberLong(0),                 "total" : NumberLong(0)             }         },         "document" : {             "deleted" : NumberLong(0),             "inserted" : NumberLong(0),             "returned" : NumberLong(26915),             "updated" : NumberLong(0)         },         "getLastError" : {             "wtime" : {                 "num" : 0,                 "totalMillis" : 0             },             "wtimeouts" : NumberLong(0)         },         "operation" : {             "fastmod" : NumberLong(0),             "idhack" : NumberLong(0),             "scanAndOrder" : NumberLong(140),             "writeConflicts" : NumberLong(0)         },         "queryExecutor" : {             "scanned" : NumberLong(599),             "scannedObjects" : NumberLong(439045)         },         "record" : {             "moves" : NumberLong(0)         },         "repl" : {             "apply" : {                 "batches" : {                     "num" : 0,                     "totalMillis" : 0                 },                 "ops" : NumberLong(0)             },             "buffer" : {                 "count" : NumberLong(0),                 "maxSizeBytes" : 268435456,                 "sizeBytes" : NumberLong(0)             },             "network" : {                 "bytes" : NumberLong(0),                 "getmores" : {                     "num" : 0,                     "totalMillis" : 0                 },                 "ops" : NumberLong(0),                 "readersCreated" : NumberLong(0)             },             "preload" : {                 "docs" : {                     "num" : 0,                     "totalMillis" : 0                 },                 "indexes" : {                     "num" : 0,                     "totalMillis" : 0                 }             }         },         "storage" : {             "freelist" : {                 "search" : {                     "bucketExhausted" : NumberLong(0),                     "requests" : NumberLong(1),                     "scanned" : NumberLong(0)                 }             }         },         "ttl" : {             "deletedDocuments" : NumberLong(0),             "passes" : NumberLong(59)         }     },     "ok" : 1 } 

1 Answers

Answers 1

Looks like you're currently capped at 1024 open files. Although there's usually a boost error to accompany that kind of failure, it's worth trying something like ulimit -n 4096

see the docs: https://docs.mongodb.com/manual/reference/ulimit/

If You Enjoyed This, Take 5 Seconds To Share It

0 comments:

Post a Comment