sys_ia32.c

linux-2.6.15.6

			return 1;
		}
		return 0;
	}

	vma = find_vma(current->mm, pstart);
	if (!vma || vma->vm_start > pstart) {
		return -ENOMEM;
	}

	/* new a partial_page */
	pp = kmem_cache_alloc(partial_page_cachep, GFP_KERNEL);
	if (!pp)
		return -ENOMEM;
	pp->base = pstart;
	pp->bitmap = 0;
	for (i = 0; i < start_bit; i++)
		set_bit(i, &(pp->bitmap));
	for (i = end_bit; i < PAGE_SIZE / IA32_PAGE_SIZE; i++)
		set_bit(i, &(pp->bitmap));
	pp->next = NULL;
	__ia32_insert_pp(current->thread.ppl, pp, prev, rb_link, rb_parent);
	return 0;
}

/*
 * Delete pp between PAGE_ALIGN(start) and PAGE_START(end) by calling
 * __ia32_delete_pp_range(). Unset possible partial pages by calling
 * __ia32_unset_pp().
 * The returned value see __ia32_unset_pp().
 */
static int
ia32_unset_pp(unsigned int *startp, unsigned int *endp)
{
	unsigned int start = *startp, end = *endp;
	int ret = 0;

	down_write(&current->mm->mmap_sem);

	__ia32_delete_pp_range(PAGE_ALIGN(start), PAGE_START(end));

	if (end < PAGE_ALIGN(start)) {
		ret = __ia32_unset_pp(start, end);
		if (ret == 1) {
			*startp = PAGE_START(start);
			*endp = PAGE_ALIGN(end);
		}
		if (ret == 0) {
			/* to shortcut sys_munmap() in sys32_munmap() */
			*startp = PAGE_START(start);
			*endp = PAGE_START(end);
		}
	} else {
		if (offset_in_page(start)) {
			ret = __ia32_unset_pp(start, PAGE_ALIGN(start));
			if (ret == 1)
				*startp = PAGE_START(start);
			if (ret == 0)
				*startp = PAGE_ALIGN(start);
			if (ret < 0)
				goto out;
		}
		if (offset_in_page(end)) {
			ret = __ia32_unset_pp(PAGE_START(end), end);
			if (ret == 1)
				*endp = PAGE_ALIGN(end);
			if (ret == 0)
				*endp = PAGE_START(end);
		}
	}

 out:
	up_write(&current->mm->mmap_sem);
	return ret;
}

/*
 * Compare the range between @start and @end with bitmap in partial page.
 * @start and @end should be IA32 page aligned and in the same IA64 page.
 */
static int
__ia32_compare_pp(unsigned int start, unsigned int end)
{
	struct partial_page *pp, *prev;
	struct rb_node ** rb_link, *rb_parent;
	unsigned int pstart, start_bit, end_bit, size;
	unsigned int first_bit, next_zero_bit;	/* the first range in bitmap */

	pstart = PAGE_START(start);

	pp = __ia32_find_pp(current->thread.ppl, pstart, &prev,
					&rb_link, &rb_parent);
	if (!pp)
		return 1;

	start_bit = (start % PAGE_SIZE) / IA32_PAGE_SIZE;
	end_bit = (end % PAGE_SIZE) / IA32_PAGE_SIZE;
	size = sizeof(pp->bitmap) * 8;
	first_bit = find_first_bit(&pp->bitmap, size);
	next_zero_bit = find_next_zero_bit(&pp->bitmap, size, first_bit);
	if ((start_bit < first_bit) || (end_bit > next_zero_bit)) {
		/* exceeds the first range in bitmap */
		return -ENOMEM;
	} else if ((start_bit == first_bit) && (end_bit == next_zero_bit)) {
		first_bit = find_next_bit(&pp->bitmap, size, next_zero_bit);
		if ((next_zero_bit < first_bit) && (first_bit < size))
			return 1;	/* has next range */
		else
			return 0; 	/* no next range */
	} else
		return 1;
}

/*
 * @start and @end should be IA32 page aligned, but don't need to be in the
 * same IA64 page. Split @start and @end to make sure they're in the same IA64
 * page, then call __ia32_compare_pp().
 *
 * Take this as example: the range is the 1st and 2nd 4K page.
 * Return 0 if they fit bitmap exactly, i.e. bitmap = 00000011;
 * Return 1 if the range doesn't cover whole bitmap, e.g. bitmap = 00001111;
 * Return -ENOMEM if the range exceeds the bitmap, e.g. bitmap = 00000001 or
 * 	bitmap = 00000101.
 */
static int
ia32_compare_pp(unsigned int *startp, unsigned int *endp)
{
	unsigned int start = *startp, end = *endp;
	int retval = 0;

	down_write(&current->mm->mmap_sem);

	if (end < PAGE_ALIGN(start)) {
		retval = __ia32_compare_pp(start, end);
		if (retval == 0) {
			*startp = PAGE_START(start);
			*endp = PAGE_ALIGN(end);
		}
	} else {
		if (offset_in_page(start)) {
			retval = __ia32_compare_pp(start,
						   PAGE_ALIGN(start));
			if (retval == 0)
				*startp = PAGE_START(start);
			if (retval < 0)
				goto out;
		}
		if (offset_in_page(end)) {
			retval = __ia32_compare_pp(PAGE_START(end), end);
			if (retval == 0)
				*endp = PAGE_ALIGN(end);
		}
	}

 out:
	up_write(&current->mm->mmap_sem);
	return retval;
}

static void
__ia32_drop_pp_list(struct partial_page_list *ppl)
{
	struct partial_page *pp = ppl->pp_head;

	while (pp) {
		struct partial_page *next = pp->next;
		kmem_cache_free(partial_page_cachep, pp);
		pp = next;
	}

	kfree(ppl);
}

void
ia32_drop_partial_page_list(struct task_struct *task)
{
	struct partial_page_list* ppl = task->thread.ppl;

	if (ppl && atomic_dec_and_test(&ppl->pp_count))
		__ia32_drop_pp_list(ppl);
}

/*
 * Copy current->thread.ppl to ppl (already initialized).
 */
static int
__ia32_copy_pp_list(struct partial_page_list *ppl)
{
	struct partial_page *pp, *tmp, *prev;
	struct rb_node **rb_link, *rb_parent;

	ppl->pp_head = NULL;
	ppl->pp_hint = NULL;
	ppl->ppl_rb = RB_ROOT;
	rb_link = &ppl->ppl_rb.rb_node;
	rb_parent = NULL;
	prev = NULL;

	for (pp = current->thread.ppl->pp_head; pp; pp = pp->next) {
		tmp = kmem_cache_alloc(partial_page_cachep, GFP_KERNEL);
		if (!tmp)
			return -ENOMEM;
		*tmp = *pp;
		__ia32_insert_pp(ppl, tmp, prev, rb_link, rb_parent);
		prev = tmp;
		rb_link = &tmp->pp_rb.rb_right;
		rb_parent = &tmp->pp_rb;
	}
	return 0;
}

int
ia32_copy_partial_page_list(struct task_struct *p, unsigned long clone_flags)
{
	int retval = 0;

	if (clone_flags & CLONE_VM) {
		atomic_inc(&current->thread.ppl->pp_count);
		p->thread.ppl = current->thread.ppl;
	} else {
		p->thread.ppl = ia32_init_pp_list();
		if (!p->thread.ppl)
			return -ENOMEM;
		down_write(&current->mm->mmap_sem);
		{
			retval = __ia32_copy_pp_list(p->thread.ppl);
		}
		up_write(&current->mm->mmap_sem);
	}

	return retval;
}

static unsigned long
emulate_mmap (struct file *file, unsigned long start, unsigned long len, int prot, int flags,
	      loff_t off)
{
	unsigned long tmp, end, pend, pstart, ret, is_congruent, fudge = 0;
	struct inode *inode;
	loff_t poff;

	end = start + len;
	pstart = PAGE_START(start);
	pend = PAGE_ALIGN(end);

	if (flags & MAP_FIXED) {
		ia32_set_pp((unsigned int)start, (unsigned int)end, flags);
		if (start > pstart) {
			if (flags & MAP_SHARED)
				printk(KERN_INFO
				       "%s(%d): emulate_mmap() can't share head (addr=0x%lx)\n",
				       current->comm, current->pid, start);
			ret = mmap_subpage(file, start, min(PAGE_ALIGN(start), end), prot, flags,
					   off);
			if (IS_ERR((void *) ret))
				return ret;
			pstart += PAGE_SIZE;
			if (pstart >= pend)
				goto out;	/* done */
		}
		if (end < pend) {
			if (flags & MAP_SHARED)
				printk(KERN_INFO
				       "%s(%d): emulate_mmap() can't share tail (end=0x%lx)\n",
				       current->comm, current->pid, end);
			ret = mmap_subpage(file, max(start, PAGE_START(end)), end, prot, flags,
					   (off + len) - offset_in_page(end));
			if (IS_ERR((void *) ret))
				return ret;
			pend -= PAGE_SIZE;
			if (pstart >= pend)
				goto out;	/* done */
		}
	} else {
		/*
		 * If a start address was specified, use it if the entire rounded out area
		 * is available.
		 */
		if (start && !pstart)
			fudge = 1;	/* handle case of mapping to range (0,PAGE_SIZE) */
		tmp = arch_get_unmapped_area(file, pstart - fudge, pend - pstart, 0, flags);
		if (tmp != pstart) {
			pstart = tmp;
			start = pstart + offset_in_page(off);	/* make start congruent with off */
			end = start + len;
			pend = PAGE_ALIGN(end);
		}
	}

	poff = off + (pstart - start);	/* note: (pstart - start) may be negative */
	is_congruent = (flags & MAP_ANONYMOUS) || (offset_in_page(poff) == 0);

	if ((flags & MAP_SHARED) && !is_congruent)
		printk(KERN_INFO "%s(%d): emulate_mmap() can't share contents of incongruent mmap "
		       "(addr=0x%lx,off=0x%llx)\n", current->comm, current->pid, start, off);

	DBG("mmap_body: mapping [0x%lx-0x%lx) %s with poff 0x%llx\n", pstart, pend,
	    is_congruent ? "congruent" : "not congruent", poff);

	down_write(&current->mm->mmap_sem);
	{
		if (!(flags & MAP_ANONYMOUS) && is_congruent)
			ret = do_mmap(file, pstart, pend - pstart, prot, flags | MAP_FIXED, poff);
		else
			ret = do_mmap(NULL, pstart, pend - pstart,
				      prot | ((flags & MAP_ANONYMOUS) ? 0 : PROT_WRITE),
				      flags | MAP_FIXED | MAP_ANONYMOUS, 0);
	}
	up_write(&current->mm->mmap_sem);

	if (IS_ERR((void *) ret))
		return ret;

	if (!is_congruent) {
		/* read the file contents */
		inode = file->f_dentry->d_inode;
		if (!inode->i_fop || !file->f_op->read
		    || ((*file->f_op->read)(file, (char __user *) pstart, pend - pstart, &poff)
			< 0))
		{
			sys_munmap(pstart, pend - pstart);
			return -EINVAL;
		}
		if (!(prot & PROT_WRITE) && sys_mprotect(pstart, pend - pstart, prot) < 0)
			return -EINVAL;
	}

	if (!(flags & MAP_FIXED))
		ia32_set_pp((unsigned int)start, (unsigned int)end, flags);
out:
	return start;
}

#endif /* PAGE_SHIFT > IA32_PAGE_SHIFT */

static inline unsigned int
get_prot32 (unsigned int prot)
{
	if (prot & PROT_WRITE)
		/* on x86, PROT_WRITE implies PROT_READ which implies PROT_EEC */
		prot |= PROT_READ | PROT_WRITE | PROT_EXEC;
	else if (prot & (PROT_READ | PROT_EXEC))
		/* on x86, there is no distinction between PROT_READ and PROT_EXEC */
		prot |= (PROT_READ | PROT_EXEC);

	return prot;
}

unsigned long
ia32_do_mmap (struct file *file, unsigned long addr, unsigned long len, int prot, int flags,
	      loff_t offset)
{
	DBG("ia32_do_mmap(file=%p,addr=0x%lx,len=0x%lx,prot=%x,flags=%x,offset=0x%llx)\n",
	    file, addr, len, prot, flags, offset);

	if (file && (!file->f_op || !file->f_op->mmap))
		return -ENODEV;

	len = IA32_PAGE_ALIGN(len);
	if (len == 0)
		return addr;

	if (len > IA32_PAGE_OFFSET || addr > IA32_PAGE_OFFSET - len)
	{
		if (flags & MAP_FIXED)
			return -ENOMEM;
		else
		return -EINVAL;
	}

	if (OFFSET4K(offset))
		return -EINVAL;

	prot = get_prot32(prot);

#if PAGE_SHIFT > IA32_PAGE_SHIFT
	down(&ia32_mmap_sem);
	{
		addr = emulate_mmap(file, addr, len, prot, flags, offset);
	}
	up(&ia32_mmap_sem);
#else
	down_write(&current->mm->mmap_sem);
	{
		addr = do_mmap(file, addr, len, prot, flags, offset);
	}
	up_write(&current->mm->mmap_sem);
#endif
	DBG("ia32_do_mmap: returning 0x%lx\n", addr);
	return addr;
}

/*
 * Linux/i386 didn't use to be able to handle more than 4 system call parameters, so these
 * system calls used a memory block for parameter passing..
 */

struct mmap_arg_struct {
	unsigned int addr;
	unsigned int len;
	unsigned int prot;
	unsigned int flags;
	unsigned int fd;
	unsigned int offset;
};

asmlinkage long
sys32_mmap (struct mmap_arg_struct __user *arg)
{
	struct mmap_arg_struct a;
	struct file *file = NULL;
	unsigned long addr;
	int flags;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;

	if (OFFSET4K(a.offset))
		return -EINVAL;

	flags = a.flags;

	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
	if (!(flags & MAP_ANONYMOUS)) {
		file = fget(a.fd);
		if (!file)
			return -EBADF;
	}

	addr = ia32_do_mmap(file, a.addr, a.len, a.prot, flags, a.offset);

	if (file)
		fput(file);
	return addr;
}

asmlinkage long
sys32_mmap2 (unsigned int addr, unsigned int len, unsigned int prot, unsigned int flags,
	     unsigned int fd, unsigned int pgoff)
{
	struct file *file = NULL;
	unsigned long retval;

	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
	if (!(flags & MAP_ANONYMOUS)) {
		file = fget(fd);
		if (!file)
			return -EBADF;
	}

	retval = ia32_do_mmap(file, addr, len, prot, flags,
			      (unsigned long) pgoff << IA32_PAGE_SHIFT);

	if (file)
		fput(file);
	return retval;
}

asmlinkage long
sys32_munmap (unsigned int start, unsigned int len)
{
	unsigned int end = start + len;
	long ret;

#if PAGE_SHIFT <= IA32_PAGE_SHIFT
	ret = sys_munmap(start, end - start);
#else
	if (OFFSET4K(start))
		return -EINVAL;

	end = IA32_PAGE_ALIGN(end);
	if (start >= end)
		return -EINVAL;

	ret = ia32_unset_pp(&start, &end);
	if (ret < 0)
		return ret;

	if (start >= end)
		return 0;

	down(&ia32_mmap_sem);
	{
		ret = sys_munmap(start, end - start);
	}
	up(&ia32_mmap_sem);
#endif
	return ret;
}

#if PAGE_SHIFT > IA32_PAGE_SHIFT

/*
 * When mprotect()ing a partial page, we set the permission to the union of the old
 * settings and the new settings.  In other words, it's only possible to make access to a
 * partial page less restrictive.
 */
static long
mprotect_subpage (unsigned long address, int new_prot)
{
	int old_prot;
	struct vm_area_struct *vma;

	if (new_prot == PROT_NONE)
		return 0;		/* optimize case where nothing changes... */
	vma = find_vma(current->mm, address);
	old_prot = get_page_prot(vma, address);
	return sys_mprotect(address, PAGE_SIZE, new_prot | old_prot);
}

#endif /* PAGE_SHIFT > IA32_PAGE_SHIFT */

asmlinkage long
sys32_mprotect (unsigned int start, unsigned int len, int prot)
{
	unsigned int end = start + len;
#if PAGE_SHIFT > IA32_PAGE_SHIFT
	long retval = 0;
#endif