Invariant failure in repair_database.js on 3.6 branch

[IanWhalen]

https://jira.mongodb.org/browse/SERVER-33720

2018-02-23T18:13:47.231+0000 F - [conn52] Invariant failure slice.size() == sizeof(int64_t) src/mongo/db/modules/rocksdb/src/rocks_record_store.cpp 1008
--
2018-02-23T18:13:47.231+0000 F - [conn52]
***aborting after invariant() failure
 
 
2018-02-23T18:13:47.260+0000 F - [conn52] Got signal: 6 (Aborted).
 
0x24628e8 0x2461d2d 0x2462134 0x7f96a7f07330 0x7f96a7b64c37 0x7f96a7b68028 0xbf5dae 0xf1ef78 0xf24c35 0xf1ab2a 0xefe4bf 0xef7dcb 0x11878b7 0xf90728 0x1e10f06 0x1e12217 0x1e0f1bf 0xf50cbe 0xf52ebf 0xf53cf0 0xf5fcef 0xf5bb87 0xf5ec5a 0x1dd0042 0xf5aae5 0xf5cef2 0xf5d801 0xf5bbf5 0xf5ec5a 0x1dd05c1 0x23381b4 0x7f96a7eff184 0x7f96a7c2c03d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"20628E8","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"2061D2D"},{"b":"400000","o":"2062134"},{"b":"7F96A7EF7000","o":"10330"},{"b":"7F96A7B2E000","o":"36C37","s":"gsignal"},{"b":"7F96A7B2E000","o":"3A028","s":"abort"},{"b":"400000","o":"7F5DAE","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"B1EF78"},{"b":"400000","o":"B24C35","s":"_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE"},{"b":"400000","o":"B1AB2A","s":"_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE"},{"b":"400000","o":"AFE4BF","s":"_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb"},{"b":"400000","o":"AF7DCB","s":"_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE"},{"b":"400000","o":"D878B7","s":"_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb"},{"b":"400000","o":"B90728","s":"_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE"},{"b":"400000","o":"1A10F06","s":"_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A12217","s":"_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"1A0F1BF","s":"_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE"},{"b":"400000","o":"B50CBE"},{"b":"400000","o":"B52EBF"},{"b":"400000","o":"B53CF0","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"400000","o":"B5FCEF","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BB87","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D0042","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE"},{"b":"400000","o":"B5AAE5","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE"},{"b":"400000","o":"B5CEF2","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"400000","o":"B5D801","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"400000","o":"B5BBF5","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"400000","o":"B5EC5A"},{"b":"400000","o":"19D05C1"},{"b":"400000","o":"1F381B4"},{"b":"7F96A7EF7000","o":"8184"},{"b":"7F96A7B2E000","o":"FE03D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.3-2-ge1cc0fb", "gitVersion" : "e1cc0fb45f6b5bbe596158258122d188667a43fe", "compiledModules" : [ "rocksdb" ], "uname" : { "sysname" : "Linux", "release" : "3.13.0-24-generic", "version" : "#46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E2B68AC9F64C81DEB87F0DBA0BB1FD9E47624ED3" }, { "b" : "7FFFDB8FE000", "elfType" : 3, "buildId" : "6755FAD2CADACDF1667E5B57FF1EDFC28DD1C976" }, { "b" : "7F96A8E68000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "7EE9BC42787A0F9A793BDBFCB0671FD12C52BC2A" }, { "b" : "7F96A8C58000", "path" : "/lib/x86_64-linux-gnu/libbz2.so.1.0", "elfType" : 3, "buildId" : "E1031DDBFFE20367E874B7093EEC0C8D9F3B43F6" }, { "b" : "7F96A8A3D000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "AD304AFCE6847F7A4D66D22853E87CCBF5A66966" }, { "b" : "7F96A8835000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "4F930712D3609C93E380E5BE5DF73E7AD273531C" }, { "b" : "7F96A8631000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "034D6A4EE9DCAB4A34ABD644345CBBB42DC63088" }, { "b" : "7F96A832B000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "300C7884CDEB5667BEA2357D2B8E7A76397562D6" }, { "b" : "7F96A8115000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F96A7EF7000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "F64B8AD471FBA1B7A3A64EFB01551E694975E1F7" }, { "b" : "7F96A7B2E000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "D9A10B8EF90300628DD0A3A535106967714D7328" }, { "b" : "7F96A9081000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2CA513EDC89C7BC06EC183D1A3A03CC0F606319C" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x28) [0x24628e8]
mongod(+0x2061D2D) [0x2461d2d]
mongod(+0x2062134) [0x2462134]
libpthread.so.0(+0x10330) [0x7f96a7f07330]
libc.so.6(gsignal+0x37) [0x7f96a7b64c37]
libc.so.6(abort+0x148) [0x7f96a7b68028]
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0xbf5dae]
mongod(+0xB1EF78) [0xf1ef78]
mongod(_ZN5mongo16RocksRecordStoreC1ENS_10StringDataES1_PN7rocksdb2DBEPNS_19RocksCounterManagerEPNS_22RocksDurabilityManagerEPNS_24RocksCompactionSchedulerENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbllPNS_14CappedCallbackE+0xD05) [0xf24c35]
mongod(_ZN5mongo11RocksEngine14getRecordStoreEPNS_16OperationContextENS_10StringDataES3_RKNS_17CollectionOptionsE+0x82A) [0xf1ab2a]
mongod(_ZN5mongo26KVDatabaseCatalogEntryBase14initCollectionEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb+0x2AF) [0xefe4bf]
mongod(_ZN5mongo15KVStorageEngine17repairRecordStoreEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x1DB) [0xef7dcb]
mongod(_ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb+0x5F7) [0x11878b7]
mongod(_ZN5mongo17CmdRepairDatabase9errmsgRunEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERS8_RNS_14BSONObjBuilderE+0x298) [0xf90728]
mongod(_ZN5mongo23ErrmsgCommandDeprecated3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x36) [0x1e10f06]
mongod(_ZN5mongo12BasicCommand11enhancedRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x67) [0x1e12217]
mongod(_ZN5mongo7Command9publicRunEPNS_16OperationContextERKNS_12OpMsgRequestERNS_14BSONObjBuilderE+0x1F) [0x1e0f1bf]
mongod(+0xB50CBE) [0xf50cbe]
mongod(+0xB52EBF) [0xf52ebf]
mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x290) [0xf53cf0]
mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0x9F) [0xf5fcef]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x77) [0xf5bb87]
mongod(+0xB5EC5A) [0xf5ec5a]
mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsE+0x182) [0x1dd0042]
mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS0_9OwnershipE+0x125) [0xf5aae5]
mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xAF2) [0xf5cef2]
mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x201) [0xf5d801]
mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0xE5) [0xf5bbf5]
mongod(+0xB5EC5A) [0xf5ec5a]
mongod(+0x19D05C1) [0x1dd05c1]
mongod(+0x1F381B4) [0x23381b4]
libpthread.so.0(+0x8184) [0x7f96a7eff184]
libc.so.6(clone+0x6D) [0x7f96a7c2c03d]
-----  END BACKTRACE  -----