clones/go
1
0
Fork 0
mirror of https://github.com/golang/go.git synced 2024-09-29 22:37:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

split heapmap, which is specific to 64-bit pointer addresses,

Browse Source 
out of malloc proper.

TBR=r
OCL=26689
CL=26689
This commit is contained in:
Russ Cox 2009-03-24 15:11:56 -07:00
parent 209865be7c
commit 80f4ab47ee
4 changed files with 214 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
src/runtime/malloc.h
View File

@ -253,100 +253,7 @@ void MCentral_Init(MCentral *c, int32 sizeclass);
int32 MCentral_AllocList(MCentral *c, int32 n, MLink **first);
void MCentral_FreeList(MCentral *c, int32 n, MLink *first);
// Free(v) must be able to determine the MSpan containing v.
// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans.
//
// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers),
// we can swap in a 2-level radix tree.
//
// NOTE(rsc): We use a 3-level tree because tcmalloc does, but
// having only three levels requires approximately 1 MB per node
// in the tree, making the minimum map footprint 3 MB.
// Using a 4-level tree would cut the minimum footprint to 256 kB.
// On the other hand, it's just virtual address space: most of
// the memory is never going to be touched, thus never paged in.
typedef struct MHeapMapNode2 MHeapMapNode2;
typedef struct MHeapMapNode3 MHeapMapNode3;
enum
{
// 64 bit address - 12 bit page size = 52 bits to map
MHeapMap_Level1Bits = 18,
MHeapMap_Level2Bits = 18,
MHeapMap_Level3Bits = 16,
MHeapMap_TotalBits =
MHeapMap_Level1Bits +
MHeapMap_Level2Bits +
MHeapMap_Level3Bits,
MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1,
};
struct MHeapMap
{
void *(*allocator)(uintptr);
MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
};
struct MHeapMapNode2
{
MHeapMapNode3 *p[1<<MHeapMap_Level2Bits];
};
struct MHeapMapNode3
{
MSpan *s[1<<MHeapMap_Level3Bits];
};
void MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
bool MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
MSpan* MHeapMap_Get(MHeapMap *m, PageID k);
MSpan* MHeapMap_GetMaybe(MHeapMap *m, PageID k);
void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
// Much of the time, free(v) needs to know only the size class for v,
// not which span it came from. The MHeapMap finds the size class
// by looking up the span.
//
// An MHeapMapCache is a simple direct-mapped cache translating
// page numbers to size classes. It avoids the expensive MHeapMap
// lookup for hot pages.
//
// The cache entries are 64 bits, with the page number in the low part
// and the value at the top.
//
// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
// we can use a 16-bit cache entry by not storing the redundant 12 bits
// of the key that are used as the entry index. Here in 64-bit land,
// that trick won't work unless the hash table has 2^28 entries.
enum
{
MHeapMapCache_HashBits = 12
};
struct MHeapMapCache
{
uintptr array[1<<MHeapMapCache_HashBits];
};
// All macros for speed (sorry).
#define HMASK ((1<<MHeapMapCache_HashBits)-1)
#define KBITS MHeapMap_TotalBits
#define KMASK ((1LL<<KBITS)-1)
#define MHeapMapCache_SET(cache, key, value) \
((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
#define MHeapMapCache_GET(cache, key, tmp) \
(tmp = (cache)->array[(key) & HMASK], \
(tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)
#include "mheapmap64.h"
// Main malloc heap.
// The heap itself is the "free[]" and "large" arrays,

107
src/runtime/mheap.c
View File

@ -281,113 +281,6 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
// TODO(rsc): IncrementalScavenge() to return memory to OS.
}
// 3-level radix tree mapping page ids to Span*.
void
MHeapMap_Init(MHeapMap *m, void *(*allocator)(size_t))
{
m->allocator = allocator;
}
MSpan*
MHeapMap_Get(MHeapMap *m, PageID k)
{
int32 i1, i2, i3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Get");
return m->p[i1]->p[i2]->s[i3];
}
MSpan*
MHeapMap_GetMaybe(MHeapMap *m, PageID k)
{
int32 i1, i2, i3;
MHeapMapNode2 *p2;
MHeapMapNode3 *p3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Get");
p2 = m->p[i1];
if(p2 == nil)
return nil;
p3 = p2->p[i2];
if(p3 == nil)
return nil;
return p3->s[i3];
}
void
MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
{
int32 i1, i2, i3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Set");
m->p[i1]->p[i2]->s[i3] = s;
}
// Allocate the storage required for entries [k, k+1, ..., k+len-1]
// so that Get and Set calls need not check for nil pointers.
bool
MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
{
uintptr end;
int32 i1, i2;
MHeapMapNode2 *p2;
MHeapMapNode3 *p3;
end = k+len;
while(k < end) {
if((k >> MHeapMap_TotalBits) != 0)
return false;
i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask;
i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask;
// first-level pointer
if((p2 = m->p[i1]) == nil) {
p2 = m->allocator(sizeof *p2);
if(p2 == nil)
return false;
sys_memclr((byte*)p2, sizeof *p2);
m->p[i1] = p2;
}
// second-level pointer
if(p2->p[i2] == nil) {
p3 = m->allocator(sizeof *p3);
if(p3 == nil)
return false;
sys_memclr((byte*)p3, sizeof *p3);
p2->p[i2] = p3;
}
// advance key past this leaf node
k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits;
}
return true;
}
// Initialize a new span with the given start and npages.
void
MSpan_Init(MSpan *span, PageID start, uintptr npages)

117
src/runtime/mheapmap64.c Normal file
View File

@ -0,0 +1,117 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Heap map, 64-bit version
// See malloc.h and mheap.c for overview.
#include "runtime.h"
#include "malloc.h"
// 3-level radix tree mapping page ids to Span*.
void
MHeapMap_Init(MHeapMap *m, void *(*allocator)(size_t))
{
m->allocator = allocator;
}
MSpan*
MHeapMap_Get(MHeapMap *m, PageID k)
{
int32 i1, i2, i3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Get");
return m->p[i1]->p[i2]->s[i3];
}
MSpan*
MHeapMap_GetMaybe(MHeapMap *m, PageID k)
{
int32 i1, i2, i3;
MHeapMapNode2 *p2;
MHeapMapNode3 *p3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Get");
p2 = m->p[i1];
if(p2 == nil)
return nil;
p3 = p2->p[i2];
if(p3 == nil)
return nil;
return p3->s[i3];
}
void
MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
{
int32 i1, i2, i3;
i3 = k & MHeapMap_Level3Mask;
k >>= MHeapMap_Level3Bits;
i2 = k & MHeapMap_Level2Mask;
k >>= MHeapMap_Level2Bits;
i1 = k & MHeapMap_Level1Mask;
k >>= MHeapMap_Level1Bits;
if(k != 0)
throw("MHeapMap_Set");
m->p[i1]->p[i2]->s[i3] = s;
}
// Allocate the storage required for entries [k, k+1, ..., k+len-1]
// so that Get and Set calls need not check for nil pointers.
bool
MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
{
uintptr end;
int32 i1, i2;
MHeapMapNode2 *p2;
MHeapMapNode3 *p3;
end = k+len;
while(k < end) {
if((k >> MHeapMap_TotalBits) != 0)
return false;
i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask;
i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask;
// first-level pointer
if((p2 = m->p[i1]) == nil) {
p2 = m->allocator(sizeof *p2);
if(p2 == nil)
return false;
sys_memclr((byte*)p2, sizeof *p2);
m->p[i1] = p2;
}
// second-level pointer
if(p2->p[i2] == nil) {
p3 = m->allocator(sizeof *p3);
if(p3 == nil)
return false;
sys_memclr((byte*)p3, sizeof *p3);
p2->p[i2] = p3;
}
// advance key past this leaf node
k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits;
}
return true;
}

96
src/runtime/mheapmap64.h Normal file
View File

@ -0,0 +1,96 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Free(v) must be able to determine the MSpan containing v.
// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans.
//
// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers),
// we can swap in a 2-level radix tree.
//
// NOTE(rsc): We use a 3-level tree because tcmalloc does, but
// having only three levels requires approximately 1 MB per node
// in the tree, making the minimum map footprint 3 MB.
// Using a 4-level tree would cut the minimum footprint to 256 kB.
// On the other hand, it's just virtual address space: most of
// the memory is never going to be touched, thus never paged in.
typedef struct MHeapMapNode2 MHeapMapNode2;
typedef struct MHeapMapNode3 MHeapMapNode3;
enum
{
// 64 bit address - 12 bit page size = 52 bits to map
MHeapMap_Level1Bits = 18,
MHeapMap_Level2Bits = 18,
MHeapMap_Level3Bits = 16,
MHeapMap_TotalBits =
MHeapMap_Level1Bits +
MHeapMap_Level2Bits +
MHeapMap_Level3Bits,
MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1,
};
struct MHeapMap
{
void *(*allocator)(uintptr);
MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
};
struct MHeapMapNode2
{
MHeapMapNode3 *p[1<<MHeapMap_Level2Bits];
};
struct MHeapMapNode3
{
MSpan *s[1<<MHeapMap_Level3Bits];
};
void MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
bool MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
MSpan* MHeapMap_Get(MHeapMap *m, PageID k);
MSpan* MHeapMap_GetMaybe(MHeapMap *m, PageID k);
void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
// Much of the time, free(v) needs to know only the size class for v,
// not which span it came from. The MHeapMap finds the size class
// by looking up the span.
//
// An MHeapMapCache is a simple direct-mapped cache translating
// page numbers to size classes. It avoids the expensive MHeapMap
// lookup for hot pages.
//
// The cache entries are 64 bits, with the page number in the low part
// and the value at the top.
//
// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
// we can use a 16-bit cache entry by not storing the redundant 12 bits
// of the key that are used as the entry index. Here in 64-bit land,
// that trick won't work unless the hash table has 2^28 entries.
enum
{
MHeapMapCache_HashBits = 12
};
struct MHeapMapCache
{
uintptr array[1<<MHeapMapCache_HashBits];
};
// All macros for speed (sorry).
#define HMASK ((1<<MHeapMapCache_HashBits)-1)
#define KBITS MHeapMap_TotalBits
#define KMASK ((1LL<<KBITS)-1)
#define MHeapMapCache_SET(cache, key, value) \
((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
#define MHeapMapCache_GET(cache, key, tmp) \
(tmp = (cache)->array[(key) & HMASK], \
(tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)