Newer
Older
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
VM_BUG_ON(!list_empty(&cc.freepages));
VM_BUG_ON(!list_empty(&cc.migratepages));
}
/*
* Regardless of success, we are done until woken up next. But remember
* the requested order/classzone_idx in case it was higher/tighter than
* our current ones
*/
if (pgdat->kcompactd_max_order <= cc.order)
pgdat->kcompactd_max_order = 0;
if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
}
void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
{
if (!order)
return;
if (pgdat->kcompactd_max_order < order)
pgdat->kcompactd_max_order = order;
if (pgdat->kcompactd_classzone_idx > classzone_idx)
pgdat->kcompactd_classzone_idx = classzone_idx;
/*
* Pairs with implicit barrier in wait_event_freezable()
* such that wakeups are not missed.
*/
if (!wq_has_sleeper(&pgdat->kcompactd_wait))
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
return;
if (!kcompactd_node_suitable(pgdat))
return;
trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
classzone_idx);
wake_up_interruptible(&pgdat->kcompactd_wait);
}
/*
* The background compaction daemon, started as a kernel thread
* from the init process.
*/
static int kcompactd(void *p)
{
pg_data_t *pgdat = (pg_data_t*)p;
struct task_struct *tsk = current;
const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
if (!cpumask_empty(cpumask))
set_cpus_allowed_ptr(tsk, cpumask);
set_freezable();
pgdat->kcompactd_max_order = 0;
pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
while (!kthread_should_stop()) {
trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
wait_event_freezable(pgdat->kcompactd_wait,
kcompactd_work_requested(pgdat));
kcompactd_do_work(pgdat);
}
return 0;
}
/*
* This kcompactd start function will be called by init and node-hot-add.
* On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
*/
int kcompactd_run(int nid)
{
pg_data_t *pgdat = NODE_DATA(nid);
int ret = 0;
if (pgdat->kcompactd)
return 0;
pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
if (IS_ERR(pgdat->kcompactd)) {
pr_err("Failed to start kcompactd on node %d\n", nid);
ret = PTR_ERR(pgdat->kcompactd);
pgdat->kcompactd = NULL;
}
return ret;
}
/*
* Called by memory hotplug when all memory in a node is offlined. Caller must
* hold mem_hotplug_begin/end().
*/
void kcompactd_stop(int nid)
{
struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
if (kcompactd) {
kthread_stop(kcompactd);
NODE_DATA(nid)->kcompactd = NULL;
}
}
/*
* It's optimal to keep kcompactd on the same CPUs as their memory, but
* not required for correctness. So if the last cpu in a node goes
* away, we get changed to run anywhere: as the first one comes back,
* restore their cpu bindings.
*/
static int kcompactd_cpu_online(unsigned int cpu)
for_each_node_state(nid, N_MEMORY) {
pg_data_t *pgdat = NODE_DATA(nid);
const struct cpumask *mask;
mask = cpumask_of_node(pgdat->node_id);
if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
/* One of our CPUs online: restore mask */
set_cpus_allowed_ptr(pgdat->kcompactd, mask);
}
static int __init kcompactd_init(void)
{
int nid;
int ret;
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"mm/compaction:online",
kcompactd_cpu_online, NULL);
if (ret < 0) {
pr_err("kcompactd: failed to register hotplug callbacks.\n");
return ret;
}
for_each_node_state(nid, N_MEMORY)
kcompactd_run(nid);
return 0;
}
subsys_initcall(kcompactd_init)
#endif /* CONFIG_COMPACTION */