Created
July 26, 2017 20:15
-
-
Save efischer19/ed8b4eb106ecdad19666b06e0f656c9a to your computer and use it in GitHub Desktop.
Memory dump debugging
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| dict, 144002424 to 64600208, diff: -79402216, 55.1394995962% | |
| unicode, 31903600 to 6837592, diff: -25066008, 78.5679609825% | |
| TransformerDataMap, 6756400 to 0, diff: -6756400, 100.0% | |
| str, 31529181 to 25779986, diff: -5749195, 18.2345205859% | |
| list, 14326176 to 8664072, diff: -5662104, 39.5227868204% | |
| TransformerData, 2657408 to 0, diff: -2657408, 100.0% | |
| set, 3438584 to 1098936, diff: -2339648, 68.0410308429% | |
| tuple, 6837360 to 4552008, diff: -2285352, 33.4244796237% | |
| ReturnDict, 1525776 to 0, diff: -1525776, 100.0% | |
| function, 7609440 to 6513720, diff: -1095720, 14.3994827477% | |
| type, 7900056 to 6928256, diff: -971800, 12.3011786246% | |
| _sre.SRE_Pattern, 1265520 to 301512, diff: -964008, 76.1748530248% | |
| float, 822024 to 68184, diff: -753840, 91.7053516686% | |
| code, 6588416 to 5857536, diff: -730880, 11.0934100093% | |
| MultiValueDict, 616072 to 0, diff: -616072, 100.0% | |
| BlockUsageLocator, 531760 to 0, diff: -531760, 100.0% | |
| BlockData, 501760 to 0, diff: -501760, 100.0% | |
| datetime.datetime, 787920 to 361152, diff: -426768, 54.163874505% | |
| _MergedGroupAccess, 404800 to 0, diff: -404800, 100.0% | |
| ObjectId, 340704 to 168, diff: -340536, 99.9506903353% | |
| DatastoreNode, 330720 to 0, diff: -330720, 100.0% | |
| FunctionNode, 312968 to 0, diff: -312968, 100.0% | |
| cell, 869120 to 602112, diff: -267008, 30.7216494845% | |
| DatabaseNode, 258200 to 0, diff: -258200, 100.0% | |
| weakref, 1207624 to 962280, diff: -245344, 20.3162573781% | |
| BlockKey, 222624 to 0, diff: -222624, 100.0% | |
| TimeStats, 182720 to 0, diff: -182720, 100.0% | |
| int, 5367912 to 5239560, diff: -128352, 2.39109732052% | |
| OrderedDict, 354856 to 236184, diff: -118672, 33.4422977208% | |
| tzutc, 109440 to 0, diff: -109440, 100.0% | |
| EditInfo, 96960 to 0, diff: -96960, 100.0% | |
| instancemethod, 302000 to 206400, diff: -95600, 31.6556291391% | |
| RegexURLPattern, 95360 to 384, diff: -94976, 99.5973154362% | |
| _BlockRelations, 91712 to 0, diff: -91712, 100.0% | |
| DescendantList, 91712 to 0, diff: -91712, 100.0% | |
| BlockSerializer, 91712 to 0, diff: -91712, 100.0% | |
| CourseLocator, 63120 to 0, diff: -63120, 100.0% | |
| SerializerMetaclass, 87688 to 30736, diff: -56952, 64.9484536082% | |
| ABCMeta, 311880 to 261256, diff: -50624, 16.231884058% | |
| module, 238481 to 190993, diff: -47488, 19.9126974476% | |
| getset_descriptor, 423072 to 378648, diff: -44424, 10.5003403676% | |
| Col, 41408 to 0, diff: -41408, 100.0% | |
| ImmutableList, 98528 to 58784, diff: -39744, 40.3377720039% | |
| ChildrenModelMetaclass, 140120 to 109384, diff: -30736, 21.935483871% | |
| FixedOffset, 26496 to 64, diff: -26432, 99.7584541063% | |
| KeyedRef, 27456 to 3648, diff: -23808, 86.7132867133% | |
| FunctionWrapper, 42760 to 20472, diff: -22288, 52.1234798877% | |
| collections.deque, 24848 to 3120, diff: -21728, 87.4436574372% | |
| DeclarativeFieldsMetaclass, 44296 to 23504, diff: -20792, 46.9387755102% | |
| RegexURLResolver, 18304 to 0, diff: -18304, 100.0% | |
| datetime.timedelta, 31760 to 15080, diff: -16680, 52.5188916877% | |
| QueryDict, 16576 to 0, diff: -16576, 100.0% | |
| property, 272272 to 257312, diff: -14960, 5.49450549451% | |
| long, 19400 to 4884, diff: -14516, 74.824742268% | |
| classobj, 82056 to 69680, diff: -12376, 15.082382763% | |
| DefinitionLocator, 11040 to 0, diff: -11040, 100.0% | |
| FormMixinBase, 18080 to 7232, diff: -10848, 60.0% | |
| _HashedSeq, 10952 to 136, diff: -10816, 98.7582176771% | |
| WeakSet, 72128 to 61376, diff: -10752, 14.9068322981% | |
| Random, 20256 to 10128, diff: -10128, 50.0% | |
| TagInfo, 9600 to 0, diff: -9600, 100.0% | |
| collections.defaultdict, 27744 to 18912, diff: -8832, 31.8339100346% | |
| classmethod, 80360 to 72240, diff: -8120, 10.1045296167% | |
| instance, 1694696 to 1687040, diff: -7656, 0.451762439989% | |
| CharField, 47744 to 40192, diff: -7552, 15.81769437% | |
| frame, 15208 to 8528, diff: -6680, 43.9242503945% | |
| ModelState, 6528 to 0, diff: -6528, 100.0% | |
| ScopeIds, 6512 to 0, diff: -6512, 100.0% | |
| ProcessAwareLogger, 27392 to 20992, diff: -6400, 23.3644859813% | |
| frozenset, 138112 to 132392, diff: -5720, 4.14156626506% | |
| Template, 5760 to 64, diff: -5696, 98.8888888889% | |
| ModuleInfo, 5632 to 0, diff: -5632, 100.0% | |
| __proxy__, 98240 to 92672, diff: -5568, 5.667752443% | |
| TaskType, 11752 to 6328, diff: -5424, 46.1538461538% | |
| builtin_function_or_method, 164016 to 159120, diff: -4896, 2.98507462687% | |
| InheritingFieldData, 4736 to 0, diff: -4736, 100.0% | |
| SplitMongoKVS, 4736 to 0, diff: -4736, 100.0% | |
| CountStats, 4640 to 0, diff: -4640, 100.0% | |
| SQLStatement, 4224 to 0, diff: -4224, 100.0% | |
| SQLDatabase, 4224 to 0, diff: -4224, 100.0% | |
| DefinitionLazyLoader, 4096 to 0, diff: -4096, 100.0% | |
| staticmethod, 41944 to 38192, diff: -3752, 8.94526034713% | |
| MediaDefiningClass, 197976 to 194360, diff: -3616, 1.82648401826% | |
| WhereNode, 6784 to 3200, diff: -3584, 52.8301886792% | |
| PlaceHolder, 10752 to 7232, diff: -3520, 32.7380952381% | |
| method_descriptor, 99288 to 95976, diff: -3312, 3.33575054387% | |
| cStringIO.StringO, 3248 to 0, diff: -3248, 100.0% | |
| _socketobject, 4256 to 1008, diff: -3248, 76.3157894737% | |
| operator.itemgetter, 41088 to 37888, diff: -3200, 7.78816199377% | |
| DateTimePattern, 3072 to 0, diff: -3072, 100.0% | |
| SerializerMethodField, 3456 to 448, diff: -3008, 87.037037037% | |
| Group, 4048 to 1216, diff: -2832, 69.9604743083% | |
| Connection, 2832 to 0, diff: -2832, 100.0% | |
| weakproxy, 2832 to 64, diff: -2768, 97.7401129944% | |
| EnrollmentTrackUserPartition, 2760 to 0, diff: -2760, 100.0% | |
| ModelBase, 329056 to 326344, diff: -2712, 0.824175824176% | |
| FilterSetMetaclass, 4520 to 1808, diff: -2712, 60.0% | |
| ModelFormMetaclass, 28928 to 26216, diff: -2712, 9.375% | |
| RenameFieldMethods, 52432 to 49720, diff: -2712, 5.1724137931% | |
| NumberPattern, 2368 to 0, diff: -2368, 100.0% | |
| PSLiteral, 2304 to 0, diff: -2304, 100.0% | |
| SampledDataSet, 2240 to 0, diff: -2240, 100.0% | |
| ExternalNode, 2160 to 0, diff: -2160, 100.0% | |
| Morsel, 2128 to 0, diff: -2128, 100.0% | |
| thread.lock, 7776 to 5696, diff: -2080, 26.7489711934% | |
| SplitResult, 2080 to 0, diff: -2080, 100.0% | |
| PSKeyword, 1984 to 0, diff: -1984, 100.0% | |
| CustomMetrics, 1984 to 0, diff: -1984, 100.0% | |
| MovedModule, 18368 to 16384, diff: -1984, 10.8013937282% | |
| UserPartition, 1920 to 0, diff: -1920, 100.0% | |
| SocketUnreader, 1856 to 0, diff: -1856, 100.0% | |
| Sentinel, 2688 to 832, diff: -1856, 69.0476190476% | |
| SiteTheme, 1856 to 0, diff: -1856, 100.0% | |
| ResolverMatch, 1856 to 0, diff: -1856, 100.0% | |
| WSGIRequest, 1856 to 0, diff: -1856, 100.0% | |
| _WSGIInputWrapper, 1856 to 0, diff: -1856, 100.0% | |
| SessionStorage, 1856 to 0, diff: -1856, 100.0% | |
| WebTransaction, 1856 to 0, diff: -1856, 100.0% | |
| SessionStore, 1856 to 0, diff: -1856, 100.0% | |
| LengthReader, 1856 to 0, diff: -1856, 100.0% | |
| Body, 1856 to 0, diff: -1856, 100.0% | |
| Query, 3392 to 1600, diff: -1792, 52.8301886792% | |
| LimitedStream, 1792 to 0, diff: -1792, 100.0% | |
| QuerySet, 3328 to 1536, diff: -1792, 53.8461538462% | |
| ApdexStats, 1760 to 0, diff: -1760, 100.0% | |
| PromiseProxy, 3608 to 1848, diff: -1760, 48.7804878049% | |
| User, 1792 to 64, diff: -1728, 96.4285714286% | |
| ParseResult, 1680 to 0, diff: -1680, 100.0% | |
| member_descriptor, 52056 to 50400, diff: -1656, 3.18118948824% | |
| LocaleDataDict, 1600 to 0, diff: -1600, 100.0% | |
| UserProfile, 1600 to 0, diff: -1600, 100.0% | |
| BlockStructureBlockData, 1600 to 0, diff: -1600, 100.0% | |
| ParseResults, 2176 to 576, diff: -1600, 73.5294117647% | |
| RoleCache, 1600 to 0, diff: -1600, 100.0% | |
| SupportedFieldType, 1536 to 0, diff: -1536, 100.0% | |
| BooleanField, 17472 to 15936, diff: -1536, 8.79120879121% | |
| ParseException, 1600 to 80, diff: -1520, 95.0% | |
| Extension, 4096 to 2624, diff: -1472, 35.9375% | |
| ConfigurationModelAdmin, 3136 to 1664, diff: -1472, 46.9387755102% | |
| _Condition, 2624 to 1152, diff: -1472, 56.0975609756% | |
| TextInput, 5440 to 3968, diff: -1472, 27.0588235294% | |
| Literal, 21696 to 20288, diff: -1408, 6.48967551622% | |
| IntegerField, 7040 to 5696, diff: -1344, 19.0909090909% | |
| MaxLengthValidator, 42240 to 40896, diff: -1344, 3.18181818182% | |
| HtmlDescriptorWithMixins, 1280 to 0, diff: -1280, 100.0% | |
| ModelAdmin, 3328 to 2048, diff: -1280, 38.4615384615% | |
| _cffi_backend.CTypeDescr, 5784 to 4511, diff: -1273, 22.0089903181% | |
| DateTimeField, 18176 to 17024, diff: -1152, 6.33802816901% | |
| functools.partial, 11264 to 10120, diff: -1144, 10.15625% | |
| VerticalBlockWithMixins, 1088 to 0, diff: -1088, 100.0% | |
| DiscussionXBlockWithMixins, 1088 to 0, diff: -1088, 100.0% | |
| Queues, 1064 to 0, diff: -1064, 100.0% | |
| Alias, 1024 to 0, diff: -1024, 100.0% |
TransformerDataMap seems like a good first line of investigation. Maybe we could add some logging to that class? https://github.com/edx/edx-platform/blob/6e61f063fab05aa28bf81736292fcb18d5e317f1/openedx/core/djangoapps/content/block_structure/block_structure.py#L332
Here's a different report - in the form <type>:<cumulative size>:<# of instances>
dict:144002424:256389
unicode:31903600:155066
str:31529181:416768
list:14326176:130747
type:7900056:8739
function:7609440:63412
tuple:6837360:87784
TransformerDataMap:6756400:6350
code:6588416:51472
int:5367912:223663
set:3438584:11819
TransformerData:2657408:41522
instance:1694696:23535
ReturnDict:1525776:1434
_sre.SRE_Pattern:1265520:3441
weakref:1207624:13723
cell:869120:15520
float:822024:34251
datetime.datetime:787920:16415
MultiValueDict:616072:317
BlockUsageLocator:531760:6647
BlockData:501760:7840
getset_descriptor:423072:5876
_MergedGroupAccess:404800:6325
OrderedDict:354856:833
ObjectId:340704:6084
DatastoreNode:330720:2067
ModelBase:329056:364
FunctionNode:312968:2059
ABCMeta:311880:345
instancemethod:302000:3775
property:272272:3094
DatabaseNode:258200:1291
module:238481:4258
BlockKey:222624:3092
MediaDefiningClass:197976:219
TimeStats:182720:1142
wrapper_descriptor:166480:2081
builtin_function_or_method:164016:2278
ChildrenModelMetaclass:140120:155
frozenset:138112:272
tzutc:109440:1710
method_descriptor:99288:1379
ImmutableList:98528:822
__proxy__:98240:1535
EditInfo:96960:1515
RegexURLPattern:95360:1490
_BlockRelations:91712:1433
DescendantList:91712:1433
BlockSerializer:91712:1433
SerializerMetaclass:87688:97
classobj:82056:789
classmethod:80360:1435
RegexLexerMeta:72320:80
WeakSet:72128:1127
CourseLocator:63120:526
NodeType:58760:65
RenameFieldMethods:52432:58
member_descriptor:52056:723
SettingMeta:50624:56
StgDict:49352:53
CharField:47744:746
DeclarativeFieldsMetaclass:44296:49
FunctionWrapper:42760:382
MaxLengthValidator:42240:660
staticmethod:41944:749
numpy.ufunc:41904:291
Col:41408:647
operator.itemgetter:41088:642
_FuncPtr:37232:179
LazySet:33248:2
datetime.timedelta:31760:794
ManagerDescriptor:29120:455
ModelFormMetaclass:28928:32
collections.defaultdict:27744:23
KeyedRef:27456:286
ProcessAwareLogger:27392:428
FixedOffset:26496:414
_ctypes.PyCSimpleType:26216:29
vectorize:25408:397
OpaqueKeyMetaclass:25312:28
collections.deque:24848:39
EntryPoint:23936:374
ReverseSingleRelatedObjectDescriptor:23808:372
Options:23360:365
Manager:21952:343
ManyToOneRel:21888:342
ForeignKey:21824:341
Literal:21696:339
String:21392:299
AutoField:20480:320
Random:20256:4
_typedict:20208:6
long:19400:632
MovedAttribute:18880:295
include:18617:237
MovedModule:18368:287
RegexURLResolver:18304:286
DateTimeField:18176:284
FormMixinBase:18080:20
PointerType:17600:275
BooleanField:17472:273
MiniProduction:17408:272
DistInfoDistribution:17024:266
QueryDict:16576:56
ForeignRelatedObjectsDescriptor:15936:249
fortran:15720:393
frame:15208:32
SystemRandom:15192:3
_ctypes.PyCFuncPtrType:14464:16
And:13952:218
Path:12824:48
TextField:12160:190
TaskType:11752:13
YAMLObjectMetaclass:11752:13
functools.partial:11264:128
DefinitionLocator:11040:138
_HashedSeq:10952:96
PlaceHolder:10752:168
Requirement:10688:167
SpecifierSet:10688:167
Production:10368:162
EnumMeta:10240:10
7840 BlockData objects, with 6350 TransformerDataMap objects mapping 41522 TransformerData objects. relevant code
6325 _MergedGroupAccess objects and 6647 BlockUsageLocator objects seem interesting as well.
The mergedgroupaccess object is only used in one place; it would seem we're running this loop some 6k times: https://github.com/edx/edx-platform/blob/9c4869c1d59899a1e4a0ed2bf9cf92c77acc4447/lms/djangoapps/course_blocks/transformers/user_partitions.py#L62-L70
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Generated using https://github.com/efischer19/meliae_scripts as before
Cutoff for types is
[(large size) - (small size)] < 1000, just like https://gist.github.com/efischer19/2c4c64992e8e2f8212e87e046528ad56Here, the larger dump is an old lms process, the smaller is a brand new one with no traffic.