vm_fault.c

open bsd vm device design

			vm_object_collapse(object);
			object->paging_in_progress++;
		}
		else {
		    	prot &= ~VM_PROT_WRITE;
			m->flags |= PG_COPYONWRITE;
		}
	}

	if (m->flags & (PG_ACTIVE|PG_INACTIVE))
		panic("vm_fault: active or inactive before copy object handling");

	/*
	 *	If the page is being written, but hasn't been
	 *	copied to the copy-object, we have to copy it there.
	 */
    RetryCopy:
	if (first_object->copy != NULL) {
		vm_object_t copy_object = first_object->copy;
		vm_offset_t copy_offset;
		vm_page_t copy_m;

		/*
		 *	We only need to copy if we want to write it.
		 */
		if ((fault_type & VM_PROT_WRITE) == 0) {
			prot &= ~VM_PROT_WRITE;
			m->flags |= PG_COPYONWRITE;
		}
		else {
			/*
			 *	Try to get the lock on the copy_object.
			 */
			if (!vm_object_lock_try(copy_object)) {
				vm_object_unlock(object);
				/* should spin a bit here... */
				vm_object_lock(object);
				goto RetryCopy;
			}

			/*
			 *	Make another reference to the copy-object,
			 *	to keep it from disappearing during the
			 *	copy.
			 */
			copy_object->ref_count++;

			/*
			 *	Does the page exist in the copy?
			 */
			copy_offset = first_offset
				- copy_object->shadow_offset;
			copy_m = vm_page_lookup(copy_object, copy_offset);
			if (page_exists = (copy_m != NULL)) {
				if (copy_m->flags & PG_BUSY) {
#ifdef DOTHREADS
					int	wait_result;

					/*
					 *	If the page is being brought
					 *	in, wait for it and then retry.
					 */
					PAGE_ASSERT_WAIT(copy_m, !change_wiring);
					RELEASE_PAGE(m);
					copy_object->ref_count--;
					vm_object_unlock(copy_object);
					UNLOCK_THINGS;
					thread_block();
					wait_result = current_thread()->wait_result;
					vm_object_deallocate(first_object);
					if (wait_result != THREAD_AWAKENED)
						return(KERN_SUCCESS);
					goto RetryFault;
#else
					/*
					 *	If the page is being brought
					 *	in, wait for it and then retry.
					 */
					PAGE_ASSERT_WAIT(copy_m, !change_wiring);
					RELEASE_PAGE(m);
					copy_object->ref_count--;
					vm_object_unlock(copy_object);
					UNLOCK_THINGS;
					thread_block();
					vm_object_deallocate(first_object);
					goto RetryFault;
#endif
				}
			}

			/*
			 *	If the page is not in memory (in the object)
			 *	and the object has a pager, we have to check
			 *	if the pager has the data in secondary
			 *	storage.
			 */
			if (!page_exists) {

				/*
				 *	If we don't allocate a (blank) page
				 *	here... another thread could try
				 *	to page it in, allocate a page, and
				 *	then block on the busy page in its
				 *	shadow (first_object).  Then we'd
				 *	trip over the busy page after we
				 *	found that the copy_object's pager
				 *	doesn't have the page...
				 */
				copy_m = vm_page_alloc(copy_object,
								copy_offset);
				if (copy_m == NULL) {
					/*
					 *	Wait for a page, then retry.
					 */
					RELEASE_PAGE(m);
					copy_object->ref_count--;
					vm_object_unlock(copy_object);
					UNLOCK_AND_DEALLOCATE;
					VM_WAIT;
					goto RetryFault;
				}

			 	if (copy_object->pager != NULL) {
					vm_object_unlock(object);
					vm_object_unlock(copy_object);
					UNLOCK_MAP;

					page_exists = vm_pager_has_page(
							copy_object->pager,
							(copy_offset + copy_object->paging_offset));

					vm_object_lock(copy_object);

					/*
					 * Since the map is unlocked, someone
					 * else could have copied this object
					 * and put a different copy_object
					 * between the two.  Or, the last
					 * reference to the copy-object (other
					 * than the one we have) may have
					 * disappeared - if that has happened,
					 * we don't need to make the copy.
					 */
					if (copy_object->shadow != object ||
					    copy_object->ref_count == 1) {
						/*
						 *	Gaah... start over!
						 */
						FREE_PAGE(copy_m);
						vm_object_unlock(copy_object);
						vm_object_deallocate(copy_object);
							/* may block */
						vm_object_lock(object);
						goto RetryCopy;
					}
					vm_object_lock(object);

					if (page_exists) {
						/*
						 *	We didn't need the page
						 */
						FREE_PAGE(copy_m);
					}
				}
			}
			if (!page_exists) {
				/*
				 *	Must copy page into copy-object.
				 */
				vm_page_copy(m, copy_m);
				copy_m->flags &= ~PG_FAKE;

				/*
				 * Things to remember:
				 * 1. The copied page must be marked 'dirty'
				 *    so it will be paged out to the copy
				 *    object.
				 * 2. If the old page was in use by any users
				 *    of the copy-object, it must be removed
				 *    from all pmaps.  (We can't know which
				 *    pmaps use it.)
				 */
				vm_page_lock_queues();
				pmap_page_protect(VM_PAGE_TO_PHYS(old_m),
						  VM_PROT_NONE);
				copy_m->flags &= ~PG_CLEAN;
				vm_page_activate(copy_m);	/* XXX */
				vm_page_unlock_queues();

				PAGE_WAKEUP(copy_m);
			}
			/*
			 *	The reference count on copy_object must be
			 *	at least 2: one for our extra reference,
			 *	and at least one from the outside world
			 *	(we checked that when we last locked
			 *	copy_object).
			 */
			copy_object->ref_count--;
			vm_object_unlock(copy_object);
			m->flags &= ~PG_COPYONWRITE;
		}
	}

	if (m->flags & (PG_ACTIVE | PG_INACTIVE))
		panic("vm_fault: active or inactive before retrying lookup");

	/*
	 *	We must verify that the maps have not changed
	 *	since our last lookup.
	 */

	if (!lookup_still_valid) {
		vm_object_t	retry_object;
		vm_offset_t	retry_offset;
		vm_prot_t	retry_prot;

		/*
		 *	Since map entries may be pageable, make sure we can
		 *	take a page fault on them.
		 */
		vm_object_unlock(object);

		/*
		 *	To avoid trying to write_lock the map while another
		 *	thread has it read_locked (in vm_map_pageable), we
		 *	do not try for write permission.  If the page is
		 *	still writable, we will get write permission.  If it
		 *	is not, or has been marked needs_copy, we enter the
		 *	mapping without write permission, and will merely
		 *	take another fault.
		 */
		result = vm_map_lookup(&map, vaddr,
				fault_type & ~VM_PROT_WRITE, &entry,
				&retry_object, &retry_offset, &retry_prot,
				&wired, &su);

		vm_object_lock(object);

		/*
		 *	If we don't need the page any longer, put it on the
		 *	active list (the easiest thing to do here).  If no
		 *	one needs it, pageout will grab it eventually.
		 */

		if (result != KERN_SUCCESS) {
			RELEASE_PAGE(m);
			UNLOCK_AND_DEALLOCATE;
			return(result);
		}

		lookup_still_valid = TRUE;

		if ((retry_object != first_object) ||
				(retry_offset != first_offset)) {
			RELEASE_PAGE(m);
			UNLOCK_AND_DEALLOCATE;
			goto RetryFault;
		}

		/*
		 *	Check whether the protection has changed or the object
		 *	has been copied while we left the map unlocked.
		 *	Changing from read to write permission is OK - we leave
		 *	the page write-protected, and catch the write fault.
		 *	Changing from write to read permission means that we
		 *	can't mark the page write-enabled after all.
		 */
		prot &= retry_prot;
		if (m->flags & PG_COPYONWRITE)
			prot &= ~VM_PROT_WRITE;
	}

	/*
	 * (the various bits we're fiddling with here are locked by
	 * the object's lock)
	 */

	/* XXX This distorts the meaning of the copy_on_write bit */

	if (prot & VM_PROT_WRITE)
		m->flags &= ~PG_COPYONWRITE;

	/*
	 *	It's critically important that a wired-down page be faulted
	 *	only once in each map for which it is wired.
	 */

	if (m->flags & (PG_ACTIVE | PG_INACTIVE))
		panic("vm_fault: active or inactive before pmap_enter");

	vm_object_unlock(object);

	/*
	 *	Put this page into the physical map.
	 *	We had to do the unlock above because pmap_enter
	 *	may cause other faults.   We don't put the
	 *	page back on the active queue until later so
	 *	that the page-out daemon won't find us (yet).
	 */

	pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), prot, wired);

	/*
	 *	If the page is not wired down, then put it where the
	 *	pageout daemon can find it.
	 */
	vm_object_lock(object);
	vm_page_lock_queues();
	if (change_wiring) {
		if (wired)
			vm_page_wire(m);
		else
			vm_page_unwire(m);
	}
	else
		vm_page_activate(m);
	vm_page_unlock_queues();

	/*
	 *	Unlock everything, and return
	 */

	PAGE_WAKEUP(m);
	UNLOCK_AND_DEALLOCATE;

	return(KERN_SUCCESS);

}

/*
 *	vm_fault_wire:
 *
 *	Wire down a range of virtual addresses in a map.
 */
int
vm_fault_wire(map, start, end)
	vm_map_t	map;
	vm_offset_t	start, end;
{
	register vm_offset_t	va;
	register pmap_t		pmap;
	int			rv;

	pmap = vm_map_pmap(map);

	/*
	 *	Inform the physical mapping system that the
	 *	range of addresses may not fault, so that
	 *	page tables and such can be locked down as well.
	 */

	pmap_pageable(pmap, start, end, FALSE);

	/*
	 *	We simulate a fault to get the page and enter it
	 *	in the physical map.
	 */

	for (va = start; va < end; va += PAGE_SIZE) {
		rv = vm_fault(map, va, VM_PROT_NONE, TRUE);
		if (rv) {
			if (va != start)
				vm_fault_unwire(map, start, va);
			return(rv);
		}
	}
	return(KERN_SUCCESS);
}


/*
 *	vm_fault_unwire:
 *
 *	Unwire a range of virtual addresses in a map.
 */
void
vm_fault_unwire(map, start, end)
	vm_map_t	map;
	vm_offset_t	start, end;
{

	register vm_offset_t	va, pa;
	register pmap_t		pmap;

	pmap = vm_map_pmap(map);

	/*
	 *	Since the pages are wired down, we must be able to
	 *	get their mappings from the physical map system.
	 */

	vm_page_lock_queues();

	for (va = start; va < end; va += PAGE_SIZE) {
		pa = pmap_extract(pmap, va);
		if (pa == (vm_offset_t) 0) {
			panic("unwire: page not in pmap");
		}
		pmap_change_wiring(pmap, va, FALSE);
		vm_page_unwire(PHYS_TO_VM_PAGE(pa));
	}
	vm_page_unlock_queues();

	/*
	 *	Inform the physical mapping system that the range
	 *	of addresses may fault, so that page tables and
	 *	such may be unwired themselves.
	 */

	pmap_pageable(pmap, start, end, TRUE);

}

/*
 *	Routine:
 *		vm_fault_copy_entry
 *	Function:
 *		Copy all of the pages from a wired-down map entry to another.
 *
 *	In/out conditions:
 *		The source and destination maps must be locked for write.
 *		The source map entry must be wired down (or be a sharing map
 *		entry corresponding to a main map entry that is wired down).
 */

void
vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
	vm_map_t	dst_map;
	vm_map_t	src_map;
	vm_map_entry_t	dst_entry;
	vm_map_entry_t	src_entry;
{

	vm_object_t	dst_object;
	vm_object_t	src_object;
	vm_offset_t	dst_offset;
	vm_offset_t	src_offset;
	vm_prot_t	prot;
	vm_offset_t	vaddr;
	vm_page_t	dst_m;
	vm_page_t	src_m;

#ifdef	lint
	src_map++;
#endif

	src_object = src_entry->object.vm_object;
	src_offset = src_entry->offset;

	/*
	 *	Create the top-level object for the destination entry.
	 *	(Doesn't actually shadow anything - we copy the pages
	 *	directly.)
	 */
	dst_object = vm_object_allocate(
			(vm_size_t) (dst_entry->end - dst_entry->start));

	dst_entry->object.vm_object = dst_object;
	dst_entry->offset = 0;

	prot  = dst_entry->max_protection;

	/*
	 *	Loop through all of the pages in the entry's range, copying
	 *	each one from the source object (it should be there) to the
	 *	destination object.
	 */
	for (vaddr = dst_entry->start, dst_offset = 0;
	     vaddr < dst_entry->end;
	     vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {

		/*
		 *	Allocate a page in the destination object
		 */
		vm_object_lock(dst_object);
		do {
			dst_m = vm_page_alloc(dst_object, dst_offset);
			if (dst_m == NULL) {
				vm_object_unlock(dst_object);
				VM_WAIT;
				vm_object_lock(dst_object);
			}
		} while (dst_m == NULL);

		/*
		 *	Find the page in the source object, and copy it in.
		 *	(Because the source is wired down, the page will be
		 *	in memory.)
		 */
		vm_object_lock(src_object);
		src_m = vm_page_lookup(src_object, dst_offset + src_offset);
		if (src_m == NULL)
			panic("vm_fault_copy_wired: page missing");

		vm_page_copy(src_m, dst_m);

		/*
		 *	Enter it in the pmap...
		 */
		vm_object_unlock(src_object);
		vm_object_unlock(dst_object);

		pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m),
				prot, FALSE);

		/*
		 *	Mark it no longer busy, and put it on the active list.
		 */
		vm_object_lock(dst_object);
		vm_page_lock_queues();
		vm_page_activate(dst_m);
		vm_page_unlock_queues();
		PAGE_WAKEUP(dst_m);
		vm_object_unlock(dst_object);
	}

}