summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/mmzone.h2
-rw-r--r--include/linux/vmstat.h4
-rw-r--r--mm/hugetlb_vmemmap.c17
-rw-r--r--mm/mm_init.c3
-rw-r--r--mm/page_alloc.c1
-rw-r--r--mm/page_ext.c32
-rw-r--r--mm/sparse-vmemmap.c8
-rw-r--r--mm/sparse.c7
-rw-r--r--mm/vmstat.c26
9 files changed, 85 insertions, 15 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 586a8f0104d7..cb7f265c2b96 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -220,6 +220,8 @@ enum node_stat_item {
PGDEMOTE_KSWAPD,
PGDEMOTE_DIRECT,
PGDEMOTE_KHUGEPAGED,
+ NR_MEMMAP, /* page metadata allocated through buddy allocator */
+ NR_MEMMAP_BOOT, /* page metadata allocated through boot allocator */
NR_VM_NODE_STAT_ITEMS
};
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 735eae6e272c..16b0cfa80502 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -624,4 +624,8 @@ static inline void lruvec_stat_sub_folio(struct folio *folio,
{
lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
}
+
+void __meminit mod_node_early_perpage_metadata(int nid, long delta);
+void __meminit store_early_perpage_metadata(void);
+
#endif /* _LINUX_VMSTAT_H */
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index b9a55322e52c..fa00d61b6c5a 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -184,10 +184,13 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
*/
static inline void free_vmemmap_page(struct page *page)
{
- if (PageReserved(page))
+ if (PageReserved(page)) {
free_bootmem_page(page);
- else
+ mod_node_page_state(page_pgdat(page), NR_MEMMAP_BOOT, -1);
+ } else {
__free_page(page);
+ mod_node_page_state(page_pgdat(page), NR_MEMMAP, -1);
+ }
}
/* Free a list of the vmemmap pages */
@@ -338,6 +341,7 @@ static int vmemmap_remap_free(unsigned long start, unsigned long end,
copy_page(page_to_virt(walk.reuse_page),
(void *)walk.reuse_addr);
list_add(&walk.reuse_page->lru, vmemmap_pages);
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, 1);
}
/*
@@ -384,14 +388,19 @@ static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
unsigned long nr_pages = (end - start) >> PAGE_SHIFT;
int nid = page_to_nid((struct page *)start);
struct page *page, *next;
+ int i;
- while (nr_pages--) {
+ for (i = 0; i < nr_pages; i++) {
page = alloc_pages_node(nid, gfp_mask, 0);
- if (!page)
+ if (!page) {
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, i);
goto out;
+ }
list_add(&page->lru, list);
}
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP, nr_pages);
+
return 0;
out:
list_for_each_entry_safe(page, next, list, lru)
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 1bab27398302..b882ae7977ae 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -29,6 +29,7 @@
#include <linux/cma.h>
#include <linux/crash_dump.h>
#include <linux/execmem.h>
+#include <linux/vmstat.h>
#include "internal.h"
#include "slab.h"
#include "shuffle.h"
@@ -1618,6 +1619,8 @@ static void __init alloc_node_mem_map(struct pglist_data *pgdat)
panic("Failed to allocate %ld bytes for node %d memory map\n",
size, pgdat->node_id);
pgdat->node_mem_map = map + offset;
+ mod_node_early_perpage_metadata(pgdat->node_id,
+ DIV_ROUND_UP(size, PAGE_SIZE));
pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
__func__, pgdat->node_id, (unsigned long)pgdat,
(unsigned long)pgdat->node_mem_map);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 804c171a3c3a..0576ac081a1f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5738,6 +5738,7 @@ void __init setup_per_cpu_pageset(void)
for_each_online_pgdat(pgdat)
pgdat->per_cpu_nodestats =
alloc_percpu(struct per_cpu_nodestat);
+ store_early_perpage_metadata();
}
__meminit void zone_pcp_init(struct zone *zone)
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 95dd8ffeaf81..c191e490c401 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -214,6 +214,8 @@ static int __init alloc_node_page_ext(int nid)
return -ENOMEM;
NODE_DATA(nid)->node_page_ext = base;
total_usage += table_size;
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP_BOOT,
+ DIV_ROUND_UP(table_size, PAGE_SIZE));
return 0;
}
@@ -268,12 +270,15 @@ static void *__meminit alloc_page_ext(size_t size, int nid)
void *addr = NULL;
addr = alloc_pages_exact_nid(nid, size, flags);
- if (addr) {
+ if (addr)
kmemleak_alloc(addr, size, 1, flags);
- return addr;
- }
+ else
+ addr = vzalloc_node(size, nid);
- addr = vzalloc_node(size, nid);
+ if (addr) {
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP,
+ DIV_ROUND_UP(size, PAGE_SIZE));
+ }
return addr;
}
@@ -316,18 +321,27 @@ static int __meminit init_section_page_ext(unsigned long pfn, int nid)
static void free_page_ext(void *addr)
{
+ size_t table_size;
+ struct page *page;
+ struct pglist_data *pgdat;
+
+ table_size = page_ext_size * PAGES_PER_SECTION;
+
if (is_vmalloc_addr(addr)) {
+ page = vmalloc_to_page(addr);
+ pgdat = page_pgdat(page);
vfree(addr);
} else {
- struct page *page = virt_to_page(addr);
- size_t table_size;
-
- table_size = page_ext_size * PAGES_PER_SECTION;
-
+ page = virt_to_page(addr);
+ pgdat = page_pgdat(page);
BUG_ON(PageReserved(page));
kmemleak_free(addr);
free_pages_exact(addr, table_size);
}
+
+ mod_node_page_state(pgdat, NR_MEMMAP,
+ -1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));
+
}
static void __free_page_ext(unsigned long pfn)
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index a2cbe44c48e1..1dda6c53370b 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -469,5 +469,13 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
if (r < 0)
return NULL;
+ if (system_state == SYSTEM_BOOTING) {
+ mod_node_early_perpage_metadata(nid, DIV_ROUND_UP(end - start,
+ PAGE_SIZE));
+ } else {
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP,
+ DIV_ROUND_UP(end - start, PAGE_SIZE));
+ }
+
return pfn_to_page(pfn);
}
diff --git a/mm/sparse.c b/mm/sparse.c
index 0c64db3f49a6..8724a06d8128 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -14,7 +14,7 @@
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/bootmem_info.h>
-
+#include <linux/vmstat.h>
#include "internal.h"
#include <asm/dma.h>
@@ -465,6 +465,9 @@ static void __init sparse_buffer_init(unsigned long size, int nid)
*/
sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
sparsemap_buf_end = sparsemap_buf + size;
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
+ mod_node_early_perpage_metadata(nid, DIV_ROUND_UP(size, PAGE_SIZE));
+#endif
}
static void __init sparse_buffer_fini(void)
@@ -643,6 +646,8 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page);
+ mod_node_page_state(page_pgdat(pfn_to_page(pfn)), NR_MEMMAP,
+ -1L * (DIV_ROUND_UP(end - start, PAGE_SIZE)));
vmemmap_free(start, end, altmap);
}
static void free_map_bootmem(struct page *memmap)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 8507c497218b..73d791d1caad 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1255,7 +1255,8 @@ const char * const vmstat_text[] = {
"pgdemote_kswapd",
"pgdemote_direct",
"pgdemote_khugepaged",
-
+ "nr_memmap",
+ "nr_memmap_boot",
/* enum writeback_stat_item counters */
"nr_dirty_threshold",
"nr_dirty_background_threshold",
@@ -2282,4 +2283,27 @@ static int __init extfrag_debug_init(void)
}
module_init(extfrag_debug_init);
+
#endif
+
+/*
+ * Page metadata size (struct page and page_ext) in pages
+ */
+static unsigned long early_perpage_metadata[MAX_NUMNODES] __meminitdata;
+
+void __meminit mod_node_early_perpage_metadata(int nid, long delta)
+{
+ early_perpage_metadata[nid] += delta;
+}
+
+void __meminit store_early_perpage_metadata(void)
+{
+ int nid;
+ struct pglist_data *pgdat;
+
+ for_each_online_pgdat(pgdat) {
+ nid = pgdat->node_id;
+ mod_node_page_state(NODE_DATA(nid), NR_MEMMAP_BOOT,
+ early_perpage_metadata[nid]);
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-23 05:13:11 +0000