Mongo seems to have run out of memory!? What can I do?
I am not even sure it's actually a memory problem.
2016-08-11T14:23:26.875+0200 E CONTROL [conn2575] mmap private failed with out of memory. (64 bit build) 2016-08-11T14:23:26.875+0200 I - [conn2575] Assertion: 13636:file /data/db/project.6 open/create failed in createPrivateMap (look in log for more information) 2016-08-11T14:23:26.892+0200 I CONTROL [conn2575] 0xf9b632 0xf36699 0xf1a22e 0xf1a2dc 0xd56925 0xd56b5f 0xd3cf06 0xd6ff6d 0xd70380 0xd7061d 0xd74ec2 0xd80b34 0xd756ab 0xd75859 0x90a811 0x90b3ec 0xa3e5fc 0xa3f334 0xbe8012 0xbe832b 0xbe8a0d 0x9b1769 0x9b2fc5 0x9b35f5 0x9b62dc 0x9d9071 0x9da07c 0x9dad4b 0xbb544b 0xac3e4a 0x7fd9c5 0xf4af79 0x7f9122c60aa1 0x7f91217b293d ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"400000","o":"B9B632","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B36699","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B1A22E","s":"_ZN5mongo11msgassertedEiPKc"},{"b":"400000","o":"B1A2DC"},{"b":"400000","o":"956925","s":"_ZN5mongo17DurableMappedFile13finishOpeningEv"},{"b":"400000","o":"956B5F","s":"_ZN5mongo17DurableMappedFile6createERKSsRyb"},{"b":"400000","o":"93CF06","s":"_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib"},{"b":"400000","o":"96FF6D","s":"_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib"},{"b":"400000","o":"970380","s":"_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib"},{"b":"400000","o":"97061D","s":"_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib"},{"b":"400000","o":"974EC2","s":"_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib"},{"b":"400000","o":"980B34","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"9756AB","s":"_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"975859","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"50A811","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"50B3EC","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"63E5FC","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"63F334","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"7E8012","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"7E832B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"7E8A0D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5B1769","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5B2FC5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5B35F5","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE"},{"b":"400000","o":"5B62DC","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5D9071","s":"_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DA07C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DAD4B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7B544B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"400000","o":"6C3E4A","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"3FD9C5","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B4AF79","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7F9122C59000","o":"7AA1"},{"b":"7F91216CA000","o":"E893D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.9", "gitVersion" : "20d60d3491908f1ae252fe452300de3978a040c7", "uname" : { "sysname" : "Linux", "release" : "2.6.32-042stab113.11", "version" : "#1 SMP Fri Dec 18 17:32:04 MSK 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "47DFD0896283C6BF009395E08B673D3246CC7AE5" }, { "b" : "7FFD9BE1E000", "elfType" : 3, "buildId" : "F466F3FAB4F73E9911206E2486A3C5F8FCCA8EA5" }, { "b" : "7F9122C59000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "C56DD1B811FC0D9263248EBB308C73FCBCD80FC1" }, { "b" : "7F91229ED000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "24A4E7055CABC29C2DDAC730D635A20C08687AC6" }, { "b" : "7F912260A000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F6F5BB27A852F29A134680794DD2EB5C3CC5D88B" }, { "b" : "7F9122402000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "95159178F1A4A3DBDC7819FBEA2C80E5FCDD6BAC" }, { "b" : "7F91221FE000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "29B61382141595ECBA6576232E44F2310C3AAB72" }, { "b" : "7F9121EF8000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "b" : "7F9121C74000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "b" : "7F9121A5E000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "b" : "7F91216CA000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "8E6FA4C4B0594C355C1B90C1D49990368C81A040" }, { "b" : "7F9122E76000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "959C5E10A47EE8A633E7681B64B4B9F74E242ED5" }, { "b" : "7F9121486000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "b" : "7F912119F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "b" : "7F9120F9B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "b" : "7F9120D6F000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "b" : "7F9120B59000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F912094E000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "b" : "7F912074B000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9120531000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C39D7FFB49DFB1B55AD09D1D711AD802123F6623" }, { "b" : "7F9120312000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xf9b632] mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf36699] mongod(_ZN5mongo11msgassertedEiPKc+0xAE) [0xf1a22e] mongod(+0xB1A2DC) [0xf1a2dc] mongod(_ZN5mongo17DurableMappedFile13finishOpeningEv+0x2F5) [0xd56925] mongod(_ZN5mongo17DurableMappedFile6createERKSsRyb+0x6F) [0xd56b5f] mongod(_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib+0x106) [0xd3cf06] mongod(_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib+0x15D) [0xd6ff6d] mongod(_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib+0xE0) [0xd70380] mongod(_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib+0x1FD) [0xd7061d] mongod(_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib+0x42) [0xd74ec2] mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x1F4) [0xd80b34] mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd756ab] mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd75859] mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x90a811] mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x90b3ec] mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa3e5fc] mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa3f334] mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xbe8012] mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xbe832b] mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbe8a0d] mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9b1769] mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x245) [0x9b2fc5] mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9b35f5] mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9b62dc] mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9d9071] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9da07c] mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9dad4b] mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbb544b] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xac3e4a] mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x7fd9c5] mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf4af79] libpthread.so.0(+0x7AA1) [0x7f9122c60aa1] libc.so.6(clone+0x6D) [0x7f91217b293d] ----- END BACKTRACE ----- ulimit -a
core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 127948 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 1024 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 10240 cpu time (seconds, -t) unlimited max user processes (-u) 127948 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited ulimit -v
unlimited swapon -s
Filename Type Size Used Priority /dev/null partition 524288 175344 -1 free -m
total used free shared buffers cached Mem: 4096 3331 764 15 0 2985 -/+ buffers/cache: 345 3750 Swap: 512 171 340 df -h
Filesystem Size Used Avail Use% Mounted on /dev/ploop10759p1 40G 26G 12G 69% / none 16G 4.0K 16G 1% /dev none 16G 0 16G 0% /dev/shm # cat /proc/sys/vm/overcommit_memory 0 ps aux | grep mongo
webdev 4006 0.2 26.0 9123780 1091292 ? Sl 14:54 0:06 mongod --logpath /var/log/mongodb/mongod.log webdev 6031 0.0 0.0 103312 872 pts/1 S+ 15:35 0:00 grep mongo mongod --version
db version v3.0.12 git version: 33934938e0e95d534cebbaff656cde916b9c3573 cat /etc/centos-release
CentOS release 6.8 (Final) db.serverStatus()
{ "host" : "host", "version" : "3.0.12", "process" : "mongod", "pid" : NumberLong(4006), "uptime" : 3590, "uptimeMillis" : NumberLong(3590668), "uptimeEstimate" : 3555, "localTime" : ISODate("2016-08-11T13:54:51.869Z"), "asserts" : { "regular" : 0, "warning" : 1, "msg" : 0, "user" : 0, "rollovers" : 0 }, "backgroundFlushing" : { "flushes" : 59, "total_ms" : 80, "average_ms" : 1.3559322033898304, "last_ms" : 1, "last_finished" : ISODate("2016-08-11T13:54:01.250Z") }, "connections" : { "current" : 4, "available" : 815, "totalCreated" : NumberLong(6) }, "cursors" : { "note" : "deprecated, use server status metrics", "clientCursors_size" : 0, "totalOpen" : 0, "pinned" : 0, "totalNoTimeout" : 0, "timedOut" : 0 }, "dur" : { "commits" : 24, "journaledMB" : 0, "writeToDataFilesMB" : 0, "compression" : 0, "commitsInWriteLock" : 0, "earlyCommits" : 0, "timeMs" : { "dt" : 0, "prepLogBuffer" : 0, "writeToJournal" : 0, "writeToDataFiles" : 0, "remapPrivateView" : 0, "commits" : 0, "commitsInWriteLock" : 0 } }, "extra_info" : { "note" : "fields vary by platform", "heap_usage_bytes" : 63466248, "page_faults" : 28 }, "globalLock" : { "totalTime" : NumberLong("3590669000"), "currentQueue" : { "total" : 0, "readers" : 0, "writers" : 0 }, "activeClients" : { "total" : 12, "readers" : 0, "writers" : 0 } }, "locks" : { "Global" : { "acquireCount" : { "r" : NumberLong(47395), "w" : NumberLong(1270), "W" : NumberLong(5) }, "acquireWaitCount" : { "r" : NumberLong(1) }, "timeAcquiringMicros" : { "r" : NumberLong(641) } }, "MMAPV1Journal" : { "acquireCount" : { "r" : NumberLong(23058), "w" : NumberLong(1300), "R" : NumberLong(35093) }, "acquireWaitCount" : { "w" : NumberLong(3), "R" : NumberLong(3) }, "timeAcquiringMicros" : { "w" : NumberLong(24), "R" : NumberLong(63215) } }, "Database" : { "acquireCount" : { "r" : NumberLong(23056), "w" : NumberLong(1247), "R" : NumberLong(4), "W" : NumberLong(23) } }, "Collection" : { "acquireCount" : { "R" : NumberLong(24059), "W" : NumberLong(1247) } }, "Metadata" : { "acquireCount" : { "W" : NumberLong(1) } } }, "network" : { "bytesIn" : NumberLong(28574), "bytesOut" : NumberLong(14793714), "numRequests" : NumberLong(229) }, "opcounters" : { "insert" : 0, "query" : 216, "update" : 1, "delete" : 0, "getmore" : 3, "command" : 11 }, "opcountersRepl" : { "insert" : 0, "query" : 0, "update" : 0, "delete" : 0, "getmore" : 0, "command" : 0 }, "storageEngine" : { "name" : "mmapv1" }, "writeBacksQueued" : false, "mem" : { "bits" : 64, "resident" : 1068, "virtual" : 8909, "supported" : true, "mapped" : 4287, "mappedWithJournal" : 8574 }, "metrics" : { "commands" : { "getLog" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "isMaster" : { "failed" : NumberLong(0), "total" : NumberLong(5) }, "replSetGetStatus" : { "failed" : NumberLong(1), "total" : NumberLong(1) }, "serverStatus" : { "failed" : NumberLong(0), "total" : NumberLong(2) }, "update" : { "failed" : NumberLong(0), "total" : NumberLong(1) }, "whatsmyuri" : { "failed" : NumberLong(0), "total" : NumberLong(2) } }, "cursor" : { "timedOut" : NumberLong(0), "open" : { "noTimeout" : NumberLong(0), "pinned" : NumberLong(0), "total" : NumberLong(0) } }, "document" : { "deleted" : NumberLong(0), "inserted" : NumberLong(0), "returned" : NumberLong(26915), "updated" : NumberLong(0) }, "getLastError" : { "wtime" : { "num" : 0, "totalMillis" : 0 }, "wtimeouts" : NumberLong(0) }, "operation" : { "fastmod" : NumberLong(0), "idhack" : NumberLong(0), "scanAndOrder" : NumberLong(140), "writeConflicts" : NumberLong(0) }, "queryExecutor" : { "scanned" : NumberLong(599), "scannedObjects" : NumberLong(439045) }, "record" : { "moves" : NumberLong(0) }, "repl" : { "apply" : { "batches" : { "num" : 0, "totalMillis" : 0 }, "ops" : NumberLong(0) }, "buffer" : { "count" : NumberLong(0), "maxSizeBytes" : 268435456, "sizeBytes" : NumberLong(0) }, "network" : { "bytes" : NumberLong(0), "getmores" : { "num" : 0, "totalMillis" : 0 }, "ops" : NumberLong(0), "readersCreated" : NumberLong(0) }, "preload" : { "docs" : { "num" : 0, "totalMillis" : 0 }, "indexes" : { "num" : 0, "totalMillis" : 0 } } }, "storage" : { "freelist" : { "search" : { "bucketExhausted" : NumberLong(0), "requests" : NumberLong(1), "scanned" : NumberLong(0) } } }, "ttl" : { "deletedDocuments" : NumberLong(0), "passes" : NumberLong(59) } }, "ok" : 1 } 1 Answers
Answers 1
Looks like you're currently capped at 1024 open files. Although there's usually a boost error to accompany that kind of failure, it's worth trying something like ulimit -n 4096
see the docs: https://docs.mongodb.com/manual/reference/ulimit/
0 comments:
Post a Comment