Newer
Older
unsigned long private, bool offlining,
enum migrate_mode mode)
int pass, rc;
for (pass = 0; pass < 10; pass++) {
rc = unmap_and_move_huge_page(get_new_page,
private, hpage, pass > 2, offlining,
mode);
switch (rc) {
case -ENOMEM:
goto out;
case -EAGAIN:
/* try again */
break;
case 0:
goto out;
default:
rc = -EIO;
goto out;
Christoph Lameter
committed
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
#ifdef CONFIG_NUMA
/*
* Move a list of individual pages
*/
struct page_to_node {
unsigned long addr;
struct page *page;
int node;
int status;
};
static struct page *new_page_node(struct page *p, unsigned long private,
int **result)
{
struct page_to_node *pm = (struct page_to_node *)private;
while (pm->node != MAX_NUMNODES && pm->page != p)
pm++;
if (pm->node == MAX_NUMNODES)
return NULL;
*result = &pm->status;
Mel Gorman
committed
return alloc_pages_exact_node(pm->node,
Mel Gorman
committed
GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
Christoph Lameter
committed
}
/*
* Move a set of pages as indicated in the pm array. The addr
* field must be set to the virtual address of the page to be moved
* and the node number must contain a valid target node.
* The pm array ends with node = MAX_NUMNODES.
Christoph Lameter
committed
*/
static int do_move_page_to_node_array(struct mm_struct *mm,
struct page_to_node *pm,
int migrate_all)
Christoph Lameter
committed
{
int err;
struct page_to_node *pp;
LIST_HEAD(pagelist);
down_read(&mm->mmap_sem);
/*
* Build a list of pages to migrate
*/
for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
struct vm_area_struct *vma;
struct page *page;
err = -EFAULT;
vma = find_vma(mm, pp->addr);
if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
Christoph Lameter
committed
goto set_status;
page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
err = PTR_ERR(page);
if (IS_ERR(page))
goto set_status;
Christoph Lameter
committed
err = -ENOENT;
if (!page)
goto set_status;
/* Use PageReserved to check for zero page */
if (PageReserved(page) || PageKsm(page))
Christoph Lameter
committed
goto put_and_set;
pp->page = page;
err = page_to_nid(page);
if (err == pp->node)
/*
* Node already in the right place
*/
goto put_and_set;
err = -EACCES;
if (page_mapcount(page) > 1 &&
!migrate_all)
goto put_and_set;
err = isolate_lru_page(page);
if (!err) {
list_add_tail(&page->lru, &pagelist);
inc_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
}
Christoph Lameter
committed
put_and_set:
/*
* Either remove the duplicate refcount from
* isolate_lru_page() or drop the page ref if it was
* not isolated.
*/
put_page(page);
set_status:
pp->status = err;
}
err = 0;
if (!list_empty(&pagelist)) {
Christoph Lameter
committed
err = migrate_pages(&pagelist, new_page_node,
(unsigned long)pm, 0, MIGRATE_SYNC);
if (err)
putback_lru_pages(&pagelist);
}
Christoph Lameter
committed
up_read(&mm->mmap_sem);
return err;
}
/*
* Migrate an array of page address onto an array of nodes and fill
* the corresponding array of status.
*/
static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
unsigned long nr_pages,
const void __user * __user *pages,
const int __user *nodes,
int __user *status, int flags)
{
struct page_to_node *pm;
unsigned long chunk_nr_pages;
unsigned long chunk_start;
int err;
err = -ENOMEM;
pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
if (!pm)
migrate_prep();
* Store a chunk of page_to_node array in a page,
* but keep the last one as a marker
chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
for (chunk_start = 0;
chunk_start < nr_pages;
chunk_start += chunk_nr_pages) {
int j;
if (chunk_start + chunk_nr_pages > nr_pages)
chunk_nr_pages = nr_pages - chunk_start;
/* fill the chunk pm with addrs and nodes from user-space */
for (j = 0; j < chunk_nr_pages; j++) {
const void __user *p;
err = -EFAULT;
if (get_user(p, pages + j + chunk_start))
goto out_pm;
pm[j].addr = (unsigned long) p;
if (get_user(node, nodes + j + chunk_start))
goto out_pm;
err = -ENODEV;
if (node < 0 || node >= MAX_NUMNODES)
goto out_pm;
if (!node_state(node, N_HIGH_MEMORY))
goto out_pm;
err = -EACCES;
if (!node_isset(node, task_nodes))
goto out_pm;
pm[j].node = node;
}
/* End marker for this chunk */
pm[chunk_nr_pages].node = MAX_NUMNODES;
/* Migrate this chunk */
err = do_move_page_to_node_array(mm, pm,
flags & MPOL_MF_MOVE_ALL);
if (err < 0)
goto out_pm;
/* Return status information */
for (j = 0; j < chunk_nr_pages; j++)
if (put_user(pm[j].status, status + j + chunk_start)) {
goto out_pm;
}
}
err = 0;
free_page((unsigned long)pm);
out:
return err;
}
Christoph Lameter
committed
/*
* Determine the nodes of an array of pages and store it in an array of status.
Christoph Lameter
committed
*/
static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
const void __user **pages, int *status)
Christoph Lameter
committed
{
unsigned long i;
Christoph Lameter
committed
down_read(&mm->mmap_sem);
for (i = 0; i < nr_pages; i++) {
unsigned long addr = (unsigned long)(*pages);
Christoph Lameter
committed
struct vm_area_struct *vma;
struct page *page;
int err = -EFAULT;
vma = find_vma(mm, addr);
if (!vma || addr < vma->vm_start)
Christoph Lameter
committed
goto set_status;
page = follow_page(vma, addr, 0);
err = PTR_ERR(page);
if (IS_ERR(page))
goto set_status;
Christoph Lameter
committed
err = -ENOENT;
/* Use PageReserved to check for zero page */
if (!page || PageReserved(page) || PageKsm(page))
Christoph Lameter
committed
goto set_status;
err = page_to_nid(page);
set_status:
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
*status = err;
pages++;
status++;
}
up_read(&mm->mmap_sem);
}
/*
* Determine the nodes of a user array of pages and store it in
* a user array of status.
*/
static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
const void __user * __user *pages,
int __user *status)
{
#define DO_PAGES_STAT_CHUNK_NR 16
const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
int chunk_status[DO_PAGES_STAT_CHUNK_NR];
while (nr_pages) {
unsigned long chunk_nr;
chunk_nr = nr_pages;
if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
chunk_nr = DO_PAGES_STAT_CHUNK_NR;
if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
break;
do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
break;
Christoph Lameter
committed
pages += chunk_nr;
status += chunk_nr;
nr_pages -= chunk_nr;
}
return nr_pages ? -EFAULT : 0;
Christoph Lameter
committed
}
/*
* Move a list of pages in the address space of the currently executing
* process.
*/
SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
const void __user * __user *, pages,
const int __user *, nodes,
int __user *, status, int, flags)
Christoph Lameter
committed
{
David Howells
committed
const struct cred *cred = current_cred(), *tcred;
Christoph Lameter
committed
struct task_struct *task;
struct mm_struct *mm;
nodemask_t task_nodes;
Christoph Lameter
committed
/* Check flags */
if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
return -EINVAL;
if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
return -EPERM;
/* Find the mm_struct */
task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter
committed
if (!task) {
Christoph Lameter
committed
return -ESRCH;
}
get_task_struct(task);
Christoph Lameter
committed
/*
* Check if this process has the right to modify the specified
* process. The right exists if the process has administrative
* capabilities, superuser privileges or the same
* userid as the target process.
*/
David Howells
committed
tcred = __task_cred(task);
Eric W. Biederman
committed
if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
!uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
Christoph Lameter
committed
!capable(CAP_SYS_NICE)) {
David Howells
committed
rcu_read_unlock();
Christoph Lameter
committed
err = -EPERM;
Christoph Lameter
committed
}
David Howells
committed
rcu_read_unlock();
Christoph Lameter
committed
err = security_task_movememory(task);
if (err)
task_nodes = cpuset_mems_allowed(task);
mm = get_task_mm(task);
put_task_struct(task);
if (!mm)
return -EINVAL;
if (nodes)
err = do_pages_move(mm, task_nodes, nr_pages, pages,
nodes, status, flags);
else
err = do_pages_stat(mm, nr_pages, pages, status);
Christoph Lameter
committed
mmput(mm);
return err;
out:
put_task_struct(task);
return err;
Christoph Lameter
committed
}
/*
* Call migration functions in the vma_ops that may prepare
* memory in a vm for migration. migration functions may perform
* the migration for vmas that do not have an underlying page struct.
*/
int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
const nodemask_t *from, unsigned long flags)
{
struct vm_area_struct *vma;
int err = 0;
for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
if (vma->vm_ops && vma->vm_ops->migrate) {
err = vma->vm_ops->migrate(vma, to, from, flags);
if (err)
break;
}
}
return err;
}