Patch to add memory address profiling features to Python 2.7.15 interpreter

patch304.c

diff - Naru
a / Include / objimpl.h
b / Include / objimpl.h
--- a / Include / objimpl.h
2018 - 04 - 30
06:47:33.000000000 + 0
800
+++ b / Include / objimpl.h
2018 - 10 - 31
11:59:39.000000000 + 0
800


@ @


-98, 10 + 98, 8 @ @
PyAPI_FUNC(void *)
PyObject_Realloc(void *, size_t);
PyAPI_FUNC(void)
PyObject_Free(void *);

-
/ *Macros * /
-  # ifdef WITH_PYMALLOC
-  # ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+  # if defined(WITH_PYMALLOC) && defined(PYMALLOC_DEBUG)
PyAPI_FUNC(void *)
_PyObject_DebugMalloc(size_t
nbytes);
PyAPI_FUNC(void *)
_PyObject_DebugRealloc(void * p, size_t
nbytes);
PyAPI_FUNC(void)
_PyObject_DebugFree(void * p);


@ @


-115, 28 + 113, 17 @ @
PyAPI_FUNC(void *)
_PyMem_DebugMalloc(size_t
nbytes);
PyAPI_FUNC(void *)
_PyMem_DebugRealloc(void * p, size_t
nbytes);
PyAPI_FUNC(void)
_PyMem_DebugFree(void * p);
-  # define PyObject_MALLOC         _PyObject_DebugMalloc
-  # define PyObject_Malloc         _PyObject_DebugMalloc
-  # define PyObject_REALLOC        _PyObject_DebugRealloc
-  # define PyObject_Realloc        _PyObject_DebugRealloc
-  # define PyObject_FREE           _PyObject_DebugFree
-  # define PyObject_Free           _PyObject_DebugFree
+  # endif

-  # else   /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
# define PyObject_MALLOC         PyObject_Malloc
# define PyObject_REALLOC        PyObject_Realloc
# define PyObject_FREE           PyObject_Free
-  # endif
-
-  # else   /* ! WITH_PYMALLOC */
-  # define PyObject_MALLOC         PyMem_MALLOC
-  # define PyObject_REALLOC        PyMem_REALLOC
-  # define PyObject_FREE           PyMem_FREE
-
-  # endif  /* WITH_PYMALLOC */
-
# define PyObject_Del            PyObject_Free
-  # define PyObject_DEL            PyObject_FREE
+  # define PyObject_DEL            PyObject_Free
+
+  # ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+PyAPI_FUNC(void)
_PyObject_DebugMallocStats(void);
+  # endif

/ * for source
compatibility
with 2.2 * /
# define _PyObject_Del           PyObject_Free
diff - Naru
a / Include / pymem.h
b / Include / pymem.h
--- a / Include / pymem.h
2018 - 04 - 30
06:47:33.000000000 + 0
800
+++ b / Include / pymem.h
2018 - 10 - 31
12:00:33.000000000 + 0
800


@ @


-11, 6 + 11, 11 @ @
extern
"C"
{
    # endif

    +PyAPI_FUNC(void *)
PyMem_RawMalloc(size_t
size);
+PyAPI_FUNC(void *)
PyMem_RawRealloc(void * ptr, size_t
new_size);
+PyAPI_FUNC(void)
PyMem_RawFree(void * ptr);
+
+
/ * BEWARE:

Each
interface
exports
both
functions and macros.Extension
modules
should
@ @ -49, 21 + 54, 17 @ @
performed
on
failure(no
exception is set, no
warning is printed, etc).
* /

-PyAPI_FUNC(void *)
PyMem_Malloc(size_t);
-PyAPI_FUNC(void *)
PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void)
PyMem_Free(void *);
+PyAPI_FUNC(void *)
PyMem_Malloc(size_t
size);
+PyAPI_FUNC(void *)
PyMem_Realloc(void * ptr, size_t
new_size);
+PyAPI_FUNC(void)
PyMem_Free(void * ptr);
+
+PyAPI_FUNC(char *)
_PyMem_RawStrdup(const
char * str);
+PyAPI_FUNC(char *)
_PyMem_Strdup(const
char * str);

/ *Starting
from Python

1.6, the
wrappers
Py_
{Malloc, Realloc, Free}
are
no
longer
supported.They
used
to
call
PyErr_NoMemory()
on
failure. * /

/ * Macros. * /
-  # ifdef PYMALLOC_DEBUG
- / * Redirect
all
memory
operations
to
Python
's debugging allocator. */
-  # define PyMem_MALLOC      _PyMem_DebugMalloc
-  # define PyMem_REALLOC     _PyMem_DebugRealloc
-  # define PyMem_FREE    _PyMem_DebugFree
-
-  # else  /* ! PYMALLOC_DEBUG */

/ * PyMem_MALLOC(0)
means
malloc(1).Some
systems
would
return NULL
for malloc(0), which would be treated as an error.Some platforms


@ @


-71, 13 + 72, 9 @ @
pymalloc.To
solve
these
problems, allocate
an
extra
byte. * /
/ *Returns
NULL
to
indicate
error if a
negative
size or size
larger
than
Py_ssize_t
can
represent is supplied.Helps
prevents
security
holes. * /
-  # define PyMem_MALLOC(n)       ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
-        : malloc(((n) != 0) ? (n) : 1  ))
-  # define PyMem_REALLOC(p, n)    ((size_t)(n) > (size_t)PY_SSIZE_T_MAX  ? NULL \
-           : realloc((p), ((n) != 0) ?(n) : 1  ))
-  # define PyMem_FREE    free
-
-# endif   /* PYMALLOC_DEBUG   */
+# define PyMem_MALLOC(n)         PyMem_Malloc(  n)
+# define PyMem_REALLOC(p, n)     PyMem_Realloc(p,   n)
+# define PyMem_FREE(p)           PyMem_Free(p )

/*
* Ty pe-orien
ed mem
ry interface
                       @@ -1 15, 6 +1 12,6 7 @@
   # define PyMem_Del     PyMem_Free
   # define PyMem_DEL     PyMem_FREE

                       +type
ef e
um {
    +    /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree( ) */
    +    PYMEM_DOMAIN_RAW,
    +
    +    /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free( ) */
    +    PYMEM_DOMAIN_MEM,
    +
    +    /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free( ) */
    +    PYMEM_DOMAIN_OBJ
+} PyMemAllocatorDomain;
+
+typedef
struct {
    +    / * u
er cont
xt passed as
the fi
st argum
nt
to
the 3 function s */
                                                               +    voi d *ctx;
+
+    /* alloc
te a mem
ry bloc k */
                          +    v oid* (*mallc) (voi d *ctx, siz
_t size);
+
+    /* allocate or res
ze a mem
ry bloc k */
                                    +    v oid* (*reallc) (voi d *ctx, voi d *ptr, siz
_t new_size);
+
+    /* rele
se a mem
ry bloc k */
                         +    vid (*fre) (voi d *ctx, voi d *ptr);
+} PyMemAllocator; + +/* Get the memory block allocator of the specified domain. */
+PyAPI_FUNC(void)
PyMem_GetAllocator(PyMemAllocatorDomain
domain,
+                                    PyMemAllocator * allocator);
+
+ / * Set
the
memory
block
allocator
of
the
specified
domain.
+
+   The
new
allocator
must
return a
distinct
non - NULL
pointer
when
requesting
+   zero
bytes.
+
+   For
the
PYMEM_DOMAIN_RAW
domain, the
allocator
must
be
thread - safe: the
GIL
+ is not held
when
the
allocator is called.
+
+   If
the
new
allocator is not a
hook(don
't call the previous allocator), the
+ PyMem_SetupDebugHooks()
function
must
be
called
to
reinstall
the
debug
hooks
+ on
top
on
the
new
allocator. * /
+PyAPI_FUNC(void)
PyMem_SetAllocator(PyMemAllocatorDomain
domain,
+                                    PyMemAllocator * allocator);
+
+ / * Setup
hooks
to
detect
bugs in the
following
Python
memory
allocator
+ functions:
+
+   - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+ - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+ - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+   Newly
allocated
memory is filled
with the byte 0xCB, freed memory is filled
+   with the byte 0xDB.Additionnal checks:
    +
    +   - detect
API
violations, ex: PyObject_Free()
called
on
a
buffer
allocated
+ by
PyMem_Malloc()
+ - detect
write
before
the
start
of
the
buffer(buffer
underflow)
+   - detect
write
after
the
end
of
the
buffer(buffer
overflow)
+
+   The
function
does
nothing if Python is not compiled is debug
mode. * /
+PyAPI_FUNC(void)
PyMem_SetupDebugHooks(void);
+
# ifdef __cplusplus
}
# endif
diff - Naru
a / Objects / object.c
b / Objects / object.c
--- a / Objects / object.c
2018 - 04 - 30
06:47:33.000000000 + 0
800
+++ b / Objects / object.c
2018 - 10 - 31
12:00:49.000000000 + 0
800


@ @


-2340, 27 + 2340, 6 @ @
Py_ssize_t(*_Py_abstract_hack)(PyObject *) = PyObject_Size;

- / * Python
's malloc wrappers (see pymem.h) */
-
-void *
-PyMem_Malloc(size_t
nbytes)
-{
    -
return PyMem_MALLOC(nbytes);
-}
-
-void *
-PyMem_Realloc(void * p, size_t
nbytes)
-{
-
return PyMem_REALLOC(p, nbytes);
-}
-
-void
- PyMem_Free(void * p)
- {
-    PyMem_FREE(p);
-}
-
-
/ * These
methods
are
used
to
control
infinite
recursion in repr, str, print,
etc.Container
objects
that
may
recursively
contain
themselves,
e.g.builtin
dictionaries and lists, should
use
Py_ReprEnter() and
diff - Naru
a / Objects / obmalloc.c
b / Objects / obmalloc.c
- -- a / Objects / obmalloc.c
2018 - 04 - 30
06:47:33.000000000 + 0
800
+ ++ b / Objects / obmalloc.c
2018 - 10 - 31
12:01:05.000000000 + 0
800
@ @ -18, 6 + 18, 281 @ @
# endif
# endif

+ / * Python
's malloc wrappers (see pymem.h) */
+
+  # ifdef PYMALLOC_DEBUG   /* WITH_PYMALLOC && PYMALLOC_DEBUG */
+ / * Forward
declaration * /
+static
void * _PyMem_DebugMallocCtx(void * ctx, size_t
size);
+static
void
_PyMem_DebugFreeCtx(void * ctx, void * p);
+static
void * _PyMem_DebugReallocCtx(void * ctx, void * ptr, size_t
size);
+
+static
void
_PyMem_DebugCheckAddress(char
api_id, const
void * p);
+  # endif
+
+  # ifdef WITH_PYMALLOC
+
+  # ifdef MS_WINDOWS
+  # include <windows.h>
+  # elif defined(HAVE_MMAP)
+  # include <sys/mman.h>
+  # ifdef MAP_ANONYMOUS
+  # define ARENAS_USE_MMAP
+  # endif
+  # endif
+
+ / * Forward
declaration * /
+static
void * _PyObject_Malloc(void * ctx, size_t
size);
+static
void
_PyObject_Free(void * ctx, void * p);
+static
void * _PyObject_Realloc(void * ctx, void * ptr, size_t
size);
+  # endif
+
+
+static
void *
+_PyMem_RawMalloc(void * ctx, size_t
size)
+{
+ / * PyMem_Malloc(0)
means
malloc(1).Some
systems
would
return NULL
+
for malloc(0), which would be treated as an error.Some platforms would
+
return a
pointer
with no memory behind it, which would break pymalloc.
+       To
solve
these
problems, allocate
an
extra
byte. * /
+ if (size == 0)
+        size = 1;
+
return malloc(size);
+}
+
+static
void *
+_PyMem_RawRealloc(void * ctx, void * ptr, size_t
size)
+{
+ if (size == 0)
+        size = 1;
+
return realloc(ptr, size);
+}
+
+static
void
+ _PyMem_RawFree(void * ctx, void * ptr)
+ {
+    free(ptr);
+}
+
+  # define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree
+  # ifdef WITH_PYMALLOC
+  # define PYOBJECT_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free
+  # else
+  # define PYOBJECT_FUNCS PYRAW_FUNCS
+  # endif
+
+  # ifdef PYMALLOC_DEBUG
+typedef
struct
{
+ / * We
tag
each
block
with an API ID in order to tag API violations * /
+    char
api_id;
+    PyMemAllocator
alloc;
+} debug_alloc_api_t;
+static
struct
{
+    debug_alloc_api_t
raw;
+    debug_alloc_api_t
mem;
+    debug_alloc_api_t
obj;
+} _PyMem_Debug = {
+    {'r', {NULL, PYRAW_FUNCS}},
+    {'m', {NULL, PYRAW_FUNCS}},
+    {'o', {NULL, PYOBJECT_FUNCS}}
+};
+
+  # define PYDEBUG_FUNCS _PyMem_DebugMallocCtx, _PyMem_DebugReallocCtx, _PyMem_DebugFreeCtx
+  # endif
+
+static
PyMemAllocator
_PyMem_Raw = {
+  # ifdef PYMALLOC_DEBUG
+ & _PyMem_Debug.raw, PYDEBUG_FUNCS
+  # else
+    NULL, PYRAW_FUNCS
+  # endif
+};
+
+static
PyMemAllocator
_PyMem = {
+  # ifdef PYMALLOC_DEBUG
+ & _PyMem_Debug.mem, PYDEBUG_FUNCS
+  # else
+    NULL, PYRAW_FUNCS
+  # endif
+};
+
+static
PyMemAllocator
_PyObject = {
+  # ifdef PYMALLOC_DEBUG
+ & _PyMem_Debug.obj, PYDEBUG_FUNCS
+  # else
+    NULL, PYOBJECT_FUNCS
+  # endif
+};
+
+  # undef PYRAW_FUNCS
+  # undef PYOBJECT_FUNCS
+  # undef PYDEBUG_FUNCS
+
+void
+ PyMem_SetupDebugHooks(void)
+ {
+  # ifdef PYMALLOC_DEBUG
+    PyMemAllocator
alloc;
+
+    alloc.malloc = _PyMem_DebugMallocCtx;
+    alloc.realloc = _PyMem_DebugReallocCtx;
+    alloc.free = _PyMem_DebugFreeCtx;
+
+ if (_PyMem_Raw.malloc != _PyMem_DebugMallocCtx)
{
    +        alloc.ctx = & _PyMem_Debug.raw;
+        PyMem_GetAllocator(PYMEM_DOMAIN_RAW, & _PyMem_Debug.raw.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_RAW, & alloc);
+}
+
+ if (_PyMem.malloc != _PyMem_DebugMallocCtx)
{
    +        alloc.ctx = & _PyMem_Debug.mem;
+        PyMem_GetAllocator(PYMEM_DOMAIN_MEM, & _PyMem_Debug.mem.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_MEM, & alloc);
+}
+
+ if (_PyObject.malloc != _PyMem_DebugMallocCtx)
{
    +        alloc.ctx = & _PyMem_Debug.obj;
+        PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, & _PyMem_Debug.obj.alloc);
+        PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, & alloc);
+}
+  # endif
+}
+
+void
+ PyMem_GetAllocator(PyMemAllocatorDomain
domain, PyMemAllocator * allocator)
+{
+    switch(domain)
+    {
    +    case
PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw;
break;
+    case
PYMEM_DOMAIN_MEM: *allocator = _PyMem;
break;
+    case
PYMEM_DOMAIN_OBJ: *allocator = _PyObject;
break;
+    default:
+ / * unknown
domain * /
+        allocator->ctx = NULL;
+        allocator->malloc = NULL;
+        allocator->realloc = NULL;
+        allocator->free = NULL;
+}
+}
+
+void
+ PyMem_SetAllocator(PyMemAllocatorDomain
domain, PyMemAllocator * allocator)
+{
    +    switch(domain)
    + {
+    case
PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator;
break;
+    case
PYMEM_DOMAIN_MEM: _PyMem = *allocator;
break;
+    case
PYMEM_DOMAIN_OBJ: _PyObject = *allocator;
break;
+ / * ignore
unknown
domain * /
+}
+
+}
+
+void *
+PyMem_RawMalloc(size_t
size)
+{
    + / *
    + * Limit
ourselves
to
PY_SSIZE_T_MAX
bytes
to
prevent
security
holes.
+
*Most
python
internals
blindly
use
a
signed
Py_ssize_t
to
track
+
*things
without
checking
for overflows or negatives.
    +     *As size_t is unsigned, checking for size < 0 is not required.
+ * /
+ if (size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
+
return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
+}
+
+void *
+PyMem_RawRealloc(void * ptr, size_t
new_size)
+{
+ / * see
PyMem_RawMalloc() * /
+ if (new_size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
+}
+
+void
PyMem_RawFree(void * ptr)
+ {
+    _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
+}
+
+void *
+PyMem_Malloc(size_t
size)
+{
+ / * see
PyMem_RawMalloc() * /
+ if (size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
return _PyMem.malloc(_PyMem.ctx, size);
+}
+
+void *
+PyMem_Realloc(void * ptr, size_t
new_size)
+{
+ / * see
PyMem_RawMalloc() * /
+ if (new_size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
return _PyMem.realloc(_PyMem.ctx, ptr, new_size);
+}
+
+void
+ PyMem_Free(void * ptr)
+ {
+    _PyMem.free(_PyMem.ctx, ptr);
+}
+
+char *
+_PyMem_RawStrdup(const
char * str)
+{
+    size_t
size;
+    char * copy;
+
+    size = strlen(str) + 1;
+    copy = PyMem_RawMalloc(size);
+ if (copy == NULL)
+
return NULL;
+    memcpy(copy, str, size);
+
return copy;
+}
+
+char *
+_PyMem_Strdup(const
char * str)
+{
+    size_t
size;
+    char * copy;
+
+    size = strlen(str) + 1;
+    copy = PyMem_Malloc(size);
+ if (copy == NULL)
+
return NULL;
+    memcpy(copy, str, size);
+
return copy;
+}
+
+void *
+PyObject_Malloc(size_t
size)
+{
+ / * see
PyMem_RawMalloc() * /
+ if (size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
return _PyObject.malloc(_PyObject.ctx, size);
+}
+
+void *
+PyObject_Realloc(void * ptr, size_t
new_size)
+{
+ / * see
PyMem_RawMalloc() * /
+ if (new_size > (size_t)PY_SSIZE_T_MAX)
+
return NULL;
+
return _PyObject.realloc(_PyObject.ctx, ptr, new_size);
+}
+
+void
+ PyObject_Free(void * ptr)
+ {
+    _PyObject.free(_PyObject.ctx, ptr);
+}
+
+
# ifdef WITH_PYMALLOC

# ifdef HAVE_MMAP
@ @ -214, 7 + 489, 7 @ @
* Arenas
are
allocated
with mmap() on systems supporting anonymous memory
* mappings to reduce heap fragmentation.
* /
-  # define ARENA_SIZE              (256 << 10)     /* 256KiB */
+  # define ARENA_SIZE              (256 << 10)     /* 256KB */

# ifdef WITH_MEMORY_LIMITS
# define MAX_ARENAS              (SMALL_MEMORY_LIMIT / ARENA_SIZE)
@ @ -581, 7 +856, 7 @ @
return NULL; / *overflow * /
# endif
nbytes = numarenas * sizeof(*arenas);
-        arenaobj = (struct arena_object *)
realloc(arenas, nbytes);
+        arenaobj = (struct arena_object *)
PyMem_Realloc(arenas, nbytes);
if (arenaobj == NULL)
    return NULL;
arenas = arenaobj;


@ @


-785, 9 + 1060, 8 @ @
*Unless
the
optimizer
reorders
everything, being
too
smart...
* /

-  # undef PyObject_Malloc
-void *
-PyObject_Malloc(size_t
nbytes)
+static
void *
+_PyObject_Malloc(void * ctx, size_t
nbytes)
{
    block * bp;
poolp
pool;
@

@


-802, 15 + 1076, 6 @ @
# endif

/ *
- * Limit
ourselves
to
PY_SSIZE_T_MAX
bytes
to
prevent
security
holes.
-
*Most
python
internals
blindly
use
a
signed
Py_ssize_t
to
track
-
*things
without
checking
for overflows or negatives.
    -     *As size_t is unsigned, checking for nbytes < 0 is not required.
- * /
- if (nbytes > PY_SSIZE_T_MAX)
-
return NULL;
-
- / *
*This
implicitly
redirects
malloc(0).
* /
if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
@ @ -981, 17 +1246, 13 @ @
* last chance to serve the request) or when the max memory limit
*has
been
reached.
* /
- if (nbytes == 0)
-        nbytes = 1;
-
return (void *)
malloc(nbytes);
+
return PyMem_Malloc(nbytes);
}

/ *free * /

   -  # undef PyObject_Free
   -ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
   - void
   - PyObject_Free(void * p)
   + static
void
+ _PyObject_Free(void * ctx, void * p)
{
poolp
pool;
block * lastfree;


@ @


-1211, 7 + 1472, 7 @ @
redirect:
# endif
/ *We
didn
't allocate this address. */
-    free(p);
+    PyMem_Free(p);
}

/ *realloc.If
p is NULL, this
acts
like
malloc(nbytes).Else if nbytes == 0,


@ @


-1219, 10 + 1480, 8 @ @
*
return a
non - NULL
result.
* /

-  # undef PyObject_Realloc
-ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
-void *
-PyObject_Realloc(void * p, size_t
nbytes)
+static
void *
+_PyObject_Realloc(void * ctx, void * p, size_t
nbytes)
{
    void * bp;
poolp
pool;
@

@


-1232, 16 + 1491, 7 @ @
# endif

if (p == NULL)
   -
return PyObject_Malloc(nbytes);
-
- / *
- * Limit
ourselves
to
PY_SSIZE_T_MAX
bytes
to
prevent
security
holes.
- * Most
python
internals
blindly
use
a
signed
Py_ssize_t
to
track
- * things
without
checking
for overflows or negatives.
    - * As
    size_t is unsigned, checking
    for nbytes < 0 is not required.
        - * /
- if (nbytes > PY_SSIZE_T_MAX)
-
return NULL;
+
return _PyObject_Malloc(ctx, nbytes);

# ifdef WITH_VALGRIND
/ *Treat
running_on_valgrind == -1
the
same as 0 * /


@ @


-1269, 10 + 1519, 10 @ @
}
size = nbytes;
}
-        bp = PyObject_Malloc(nbytes);
+        bp = _PyObject_Malloc(ctx, nbytes);
if (bp != NULL)
{
    memcpy(bp, p, size);
-            PyObject_Free(p);
+            _PyObject_Free(ctx, p);
}
return bp;
}

@ @


-1290, 40 + 1540, 17 @ @
* at
p.Instead
we
punt:  let
C
continue
to
manage
this
block.
* /
if (nbytes)
    -
return realloc(p, nbytes);
+
return PyMem_Realloc(p, nbytes);
/ *C
doesn
't define the result of realloc(p, 0) (it may or may not
* return NULL
then), but
Python
's docs promise that nbytes==0 never
* returns
NULL.We
don
't pass 0 to realloc(), to avoid that endcase
* to
begin
with.Even then, we can't be sure that realloc() won't
*
return NULL.
* /
-    bp = realloc(p, 1);
+    bp = PyMem_Realloc(p, 1);
return bp ? bp: p;
}

-  # else   /* ! WITH_PYMALLOC */
-
- / *= == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == == = * /
- / * pymalloc
not enabled:  Redirect
the
entry
points
to
malloc.These
will
-
*only
be
used
by
extensions
that
are
compiled
with pymalloc enabled.* /
-
-void *
-PyObject_Malloc(size_t n)
-{
-
return PyMem_MALLOC(n);
-}
-
-void *
-PyObject_Realloc(void * p, size_t
n)
-{
-
return PyMem_REALLOC(p, n);
-}
-
-void
- PyObject_Free(void * p)
- {
-    PyMem_FREE(p);
-}

# endif /* WITH_PYMALLOC */

# ifdef PYMALLOC_DEBUG
@ @


-1343, 10 + 1570, 6 @ @
# define DEADBYTE       0xDB    /* dead (newly freed) memory */
# define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */

- / * We
tag
each
block
with an API ID in order to tag API violations * /
-  # define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
-  # define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
-
static size_t serialno = 0; / * incremented on each debug {m, re}alloc * /

/ * serialno is always incremented via calling this routine.The point is
@ @ -1429, 58 +1652, 18 @ @
p[2 * S+n: 2 * S + n + S]
Copies
of
FORBIDDENBYTE.Used
to
catch
over - writes and reads.
                      p[2 * S + n + S: 2 * S + n + 2 * S]
- A
serial
number, incremented
by
1
on
each
call
to
_PyObject_DebugMalloc
- and _PyObject_DebugRealloc.
+ A
serial
number, incremented
by
1
on
each
call
to
_PyMem_DebugMalloc
+ and _PyMem_DebugRealloc.
    This is a
big - endian
size_t.
    If
"bad memory" is detected
later, the
serial
number
gives
an
excellent
way
to
set
a
breakpoint
on
the
next
run, to
capture
the
instant
at
which
this
block
was
passed
out.
* /

- / * debug
replacements
for the PyMem_ * memory API * /
-void *
-_PyMem_DebugMalloc(size_t nbytes)
-{
-
return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
-}
-void *
-_PyMem_DebugRealloc(void * p, size_t
nbytes)
-{
-
return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
-}
-void
- _PyMem_DebugFree(void * p)
- {
-    _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
-}
-
- / * debug
replacements
for the PyObject_ * memory API * /
-void *
-_PyObject_DebugMalloc(size_t nbytes)
-{
-
return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
-}
-void *
-_PyObject_DebugRealloc(void * p, size_t
nbytes)
-{
-
return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
-}
-void
- _PyObject_DebugFree(void * p)
- {
-    _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
-}
-void
- _PyObject_DebugCheckAddress(const
void * p)
-{
-    _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
-}
-
-
- / * generic
debug
memory
api,
with an "id" to identify the API in use * /
-void *
-_PyObject_DebugMallocApi(char id, size_t nbytes)
+static void *
+_PyMem_DebugMallocCtx(void * ctx, size_t nbytes)
{
+    debug_alloc_api_t * api = (debug_alloc_api_t *)
ctx;
uchar * p; / *base
address
of
malloc
'ed block */
uchar * tail; / *p + 2 * SST + nbytes == pointer
to
tail
pad
bytes * /
size_t
total; / *nbytes + 4 * SST * /


@ @


-1491, 14 + 1674, 14 @ @
/ *overflow:  can
't represent total as a size_t */
return NULL;

-    p = (uchar *)
PyObject_Malloc(total);
+    p = (uchar *)
api->alloc.malloc(api->alloc.ctx, total);
if (p == NULL)
    return NULL;

/ *at
p, write
size(SST
bytes), id(1
byte), pad(SST - 1
bytes) * /
write_size_t(p, nbytes);
-    p[SST] = (uchar)
id;
-    memset(p + SST + 1, FORBIDDENBYTE, SST - 1);
+    p[SST] = (uchar)
api->api_id;
+    memset(p + SST + 1, FORBIDDENBYTE, SST - 1);

if (nbytes > 0)
    memset(p + 2 * SST, CLEANBYTE, nbytes);


@ @


-1516, 35 + 1699, 37 @ @
Then
fills
the
original
bytes
with DEADBYTE.
    Then
    calls
    the
    underlying
    free.
* /
-void
-_PyObject_DebugFreeApi(char
api, void * p)
+static
void
+_PyMem_DebugFreeCtx(void * ctx, void * p)
{
    +    debug_alloc_api_t * api = (debug_alloc_api_t *)
ctx;
uchar * q = (uchar *)
p - 2 * SST; / *address
returned
from malloc * /
size_t
nbytes;

if (p == NULL)
return;
-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q);
nbytes += 4 * SST;
if (nbytes > 0)
    memset(q, DEADBYTE, nbytes);
-    PyObject_Free(q);
+    api->alloc.free(api->alloc.ctx, q);
}

-void *
-_PyObject_DebugReallocApi(char
api, void * p, size_t
nbytes)
+static
void *
+_PyMem_DebugReallocCtx(void * ctx, void * p, size_t
nbytes)
{
-    uchar * q = (uchar *)
p;
+    debug_alloc_api_t * api = (debug_alloc_api_t *)
ctx;
+    uchar * q = (uchar *)
p, *oldq;
uchar * tail;
size_t
total; / *nbytes + 4 * SST * /
size_t
original_nbytes;
int
i;

if (p == NULL)
    -
return _PyObject_DebugMallocApi(api, nbytes);
+
return _PyMem_DebugMallocCtx(ctx, nbytes);

-    _PyObject_DebugCheckAddressApi(api, p);
+    _PyMem_DebugCheckAddress(api->api_id, p);
bumpserialno();
original_nbytes = read_size_t(q - 2 * SST);
total = nbytes + 4 * SST;


@ @


-1552, 16 + 1737, 12 @ @
/ *overflow:  can
't represent total as a size_t */
return NULL;

- if (nbytes <= original_nbytes)
{
    - / * shrinking: mark old extra memory dead * /
                                           -        memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2 * SST);
-}
-
/ *Resize and add
decorations.We
may
get
a
new
pointer
here, in which
*case
we
didn
't get the chance to mark the old memory with DEADBYTE,
*but
we
live
with that.
    * /
    -    q = (uchar *)
    PyObject_Realloc(q - 2 * SST, total);
    +    oldq = q;
    +    q = (uchar *)
    api->alloc.realloc(api->alloc.ctx, q - 2 * SST, total);
    if (q == NULL) {
    if (nbytes <= original_nbytes) {
    / * bpo-31626: the
    memset()
    above
    expects
    that
    realloc
    never
    fails
    @ @ -1571, 11 + 1752, 17 @ @
    return NULL;
    }

    + if (q == oldq & & nbytes <= original_nbytes)
    {
    + / * shrinking:  mark
    old
    extra
    memory
    dead * /
    +        memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
    +}
    + \
        write_size_t(q, nbytes);
    -
    assert (q[SST] == (uchar)api);
    +
    assert (q[SST] == (uchar)api->api_id);
    for (i = 1; i < SST; ++i)
    assert (q[SST + i] == FORBIDDENBYTE);
    q += 2 * SST;
    + \
            tail = q + nbytes;
    memset(tail, FORBIDDENBYTE, SST);
    write_size_t(tail + SST, serialno);
    @ @ -1594, 8 +1781, 8 @ @
    * and call Py_FatalError to kill the program.
    * The API id, is also checked.
    * /
    - void
    -_PyObject_DebugCheckAddressApi(char api, const void * p)
    +static void
    +_PyMem_DebugCheckAddress(char api, const void * p)
    {
    const
    uchar * q = (const uchar *)
    p;
    char
    msgbuf[64];


    @ @


    -1941, 3 + 2128, 44 @ @
    arenas[arenaindex_temp].address != 0;
    }
    # endif
    +
    +
    +  # if defined(WITH_PYMALLOC) && defined(PYMALLOC_DEBUG)
    + / * Dummy
    functions
    only
    present
    to
    keep
    the
    same
    ABI
    with the vanilla Python
    +   compiled in debug mode: they
    are
    not used in practice.See
    issue:
    +   https: // github.com / vstinner / pytracemalloc / issues / 1 * /
                  +
                  +void * _PyMem_DebugMalloc(size_t
    nbytes)
    +{
    return PyMem_RawMalloc(nbytes);}
    +
    +void * _PyMem_DebugRealloc(void * p, size_t
    nbytes)
    +{
    return PyMem_RawRealloc(p, nbytes);}
    +
    +void
    _PyObject_DebugFree(void * p)
    + {
    return PyObject_Free(p);}
    +
    +void * _PyObject_DebugMalloc(size_t
    nbytes)
    +{
    return PyObject_Malloc(nbytes);}
    +
    +void * _PyObject_DebugRealloc(void * p, size_t
    nbytes)
    +{
    return PyObject_Realloc(p, nbytes);}
    +
    +void
    _PyMem_DebugFree(void * p)
    + {PyMem_RawFree(p);}
    +
    +void
    _PyObject_DebugCheckAddress(const
    void * p)
    +{}
    +
    +void * _PyObject_DebugMallocApi(char
    api, size_t
    nbytes)
    +{
    return PyObject_Malloc(nbytes);}
    +
    +void * _PyObject_DebugReallocApi(char
    api, void * p, size_t
    nbytes)
    +{
    return PyObject_Realloc(p, nbytes);}
    +
    +void
    _PyObject_DebugFreeApi(char
    api, void * p)
    +{
    return PyObject_Free(p);}
    +
    +void
    _PyObject_DebugCheckAddressApi(char
    api, const
    void * p)
    +{}
    +  # endif
    +
    diff - Naru
    a / Python / pythonrun.c
    b / Python / pythonrun.c
    - -- a / Python / pythonrun.c
    2018 - 04 - 30
    06:47:33.000000000 + 0
    800
    + ++ b / Python / pythonrun.c
    2018 - 10 - 31
    12:01:31.000000000 + 0
    800
    @ @ -158, 6 + 158, 42 @ @
    return 0;
    }

    +static
    void
    + inittracemalloc(void)
    + {
    +    PyObject * mod = NULL, *res = NULL;
    +    char * p, *endptr;
    +    long
    nframe;
    +
    +    p = Py_GETENV("PYTHONTRACEMALLOC");
    + if (p == NULL | | * p == '\0')
    +
    return;
    +
    +    endptr = p;
    +    nframe = strtol(p, & endptr, 10);
    + if (*endptr != '\0' | | nframe < 1 | | nframe > 100000)
    +        Py_FatalError("PYTHONTRACEMALLOC: invalid number of frames");
    +
    +    mod = PyImport_ImportModule("_tracemalloc");
    + if (mod == NULL)
    +        goto
    error;
    +
    +    res = PyObject_CallMethod(mod, "start", "i", (int)
    nframe);
    + if (res == NULL)
    +        goto
    error;
    +
    +    goto
    done;
    +
    +error:
    +    fprintf(stderr, "failed to start tracemalloc:\n");
    +    PyErr_Print();
    +
    +done:
    +    Py_XDECREF(mod);
    +    Py_XDECREF(res);
    +}
    +
    +
    void
    Py_InitializeEx(int
    install_sigs)
    {


    @ @


    -290, 6 + 326, 8 @ @
    if (!Py_NoSiteFlag)
        initsite(); / *Module
        site * /

    +    inittracemalloc();
    +
    if ((p = Py_GETENV("PYTHONIOENCODING")) & & * p != '\0') {
    p = icodeset = codeset = strdup(p);
    free_codeset = 1;