as-iosched.c

	}
	ad->ioc_finished = 0;

	ad->next_rq[data_dir] = as_find_next_rq(ad, rq);

	/*
	 * take it off the sort and fifo list, add to dispatch queue
	 */
	as_remove_queued_request(ad->q, rq);
	WARN_ON(RQ_STATE(rq) != AS_RQ_QUEUED);

	elv_dispatch_sort(ad->q, rq);

	RQ_SET_STATE(rq, AS_RQ_DISPATCHED);
	if (RQ_IOC(rq) && RQ_IOC(rq)->aic)
		atomic_inc(&RQ_IOC(rq)->aic->nr_dispatched);
	ad->nr_dispatched++;
}

/*
 * as_dispatch_request selects the best request according to
 * read/write expire, batch expire, etc, and moves it to the dispatch
 * queue. Returns 1 if a request was found, 0 otherwise.
 */
static int as_dispatch_request(struct request_queue *q, int force)
{
	struct as_data *ad = q->elevator->elevator_data;
	const int reads = !list_empty(&ad->fifo_list[REQ_SYNC]);
	const int writes = !list_empty(&ad->fifo_list[REQ_ASYNC]);
	struct request *rq;

	if (unlikely(force)) {
		/*
		 * Forced dispatch, accounting is useless.  Reset
		 * accounting states and dump fifo_lists.  Note that
		 * batch_data_dir is reset to REQ_SYNC to avoid
		 * screwing write batch accounting as write batch
		 * accounting occurs on W->R transition.
		 */
		int dispatched = 0;

		ad->batch_data_dir = REQ_SYNC;
		ad->changed_batch = 0;
		ad->new_batch = 0;

		while (ad->next_rq[REQ_SYNC]) {
			as_move_to_dispatch(ad, ad->next_rq[REQ_SYNC]);
			dispatched++;
		}
		ad->last_check_fifo[REQ_SYNC] = jiffies;

		while (ad->next_rq[REQ_ASYNC]) {
			as_move_to_dispatch(ad, ad->next_rq[REQ_ASYNC]);
			dispatched++;
		}
		ad->last_check_fifo[REQ_ASYNC] = jiffies;

		return dispatched;
	}

	/* Signal that the write batch was uncontended, so we can't time it */
	if (ad->batch_data_dir == REQ_ASYNC && !reads) {
		if (ad->current_write_count == 0 || !writes)
			ad->write_batch_idled = 1;
	}

	if (!(reads || writes)
		|| ad->antic_status == ANTIC_WAIT_REQ
		|| ad->antic_status == ANTIC_WAIT_NEXT
		|| ad->changed_batch)
		return 0;

	if (!(reads && writes && as_batch_expired(ad))) {
		/*
		 * batch is still running or no reads or no writes
		 */
		rq = ad->next_rq[ad->batch_data_dir];

		if (ad->batch_data_dir == REQ_SYNC && ad->antic_expire) {
			if (as_fifo_expired(ad, REQ_SYNC))
				goto fifo_expired;

			if (as_can_anticipate(ad, rq)) {
				as_antic_waitreq(ad);
				return 0;
			}
		}

		if (rq) {
			/* we have a "next request" */
			if (reads && !writes)
				ad->current_batch_expires =
					jiffies + ad->batch_expire[REQ_SYNC];
			goto dispatch_request;
		}
	}

	/*
	 * at this point we are not running a batch. select the appropriate
	 * data direction (read / write)
	 */

	if (reads) {
		BUG_ON(RB_EMPTY_ROOT(&ad->sort_list[REQ_SYNC]));

		if (writes && ad->batch_data_dir == REQ_SYNC)
			/*
			 * Last batch was a read, switch to writes
			 */
			goto dispatch_writes;

		if (ad->batch_data_dir == REQ_ASYNC) {
			WARN_ON(ad->new_batch);
			ad->changed_batch = 1;
		}
		ad->batch_data_dir = REQ_SYNC;
		rq = rq_entry_fifo(ad->fifo_list[REQ_SYNC].next);
		ad->last_check_fifo[ad->batch_data_dir] = jiffies;
		goto dispatch_request;
	}

	/*
	 * the last batch was a read
	 */

	if (writes) {
dispatch_writes:
		BUG_ON(RB_EMPTY_ROOT(&ad->sort_list[REQ_ASYNC]));

		if (ad->batch_data_dir == REQ_SYNC) {
			ad->changed_batch = 1;

			/*
			 * new_batch might be 1 when the queue runs out of
			 * reads. A subsequent submission of a write might
			 * cause a change of batch before the read is finished.
			 */
			ad->new_batch = 0;
		}
		ad->batch_data_dir = REQ_ASYNC;
		ad->current_write_count = ad->write_batch_count;
		ad->write_batch_idled = 0;
		rq = rq_entry_fifo(ad->fifo_list[REQ_ASYNC].next);
		ad->last_check_fifo[REQ_ASYNC] = jiffies;
		goto dispatch_request;
	}

	BUG();
	return 0;

dispatch_request:
	/*
	 * If a request has expired, service it.
	 */

	if (as_fifo_expired(ad, ad->batch_data_dir)) {
fifo_expired:
		rq = rq_entry_fifo(ad->fifo_list[ad->batch_data_dir].next);
	}

	if (ad->changed_batch) {
		WARN_ON(ad->new_batch);

		if (ad->nr_dispatched)
			return 0;

		if (ad->batch_data_dir == REQ_ASYNC)
			ad->current_batch_expires = jiffies +
					ad->batch_expire[REQ_ASYNC];
		else
			ad->new_batch = 1;

		ad->changed_batch = 0;
	}

	/*
	 * rq is the selected appropriate request.
	 */
	as_move_to_dispatch(ad, rq);

	return 1;
}

/*
 * add rq to rbtree and fifo
 */
static void as_add_request(struct request_queue *q, struct request *rq)
{
	struct as_data *ad = q->elevator->elevator_data;
	int data_dir;

	RQ_SET_STATE(rq, AS_RQ_NEW);

	data_dir = rq_is_sync(rq);

	rq->elevator_private = as_get_io_context(q->node);

	if (RQ_IOC(rq)) {
		as_update_iohist(ad, RQ_IOC(rq)->aic, rq);
		atomic_inc(&RQ_IOC(rq)->aic->nr_queued);
	}

	as_add_rq_rb(ad, rq);

	/*
	 * set expire time and add to fifo list
	 */
	rq_set_fifo_time(rq, jiffies + ad->fifo_expire[data_dir]);
	list_add_tail(&rq->queuelist, &ad->fifo_list[data_dir]);

	as_update_rq(ad, rq); /* keep state machine up to date */
	RQ_SET_STATE(rq, AS_RQ_QUEUED);
}

static void as_activate_request(struct request_queue *q, struct request *rq)
{
	WARN_ON(RQ_STATE(rq) != AS_RQ_DISPATCHED);
	RQ_SET_STATE(rq, AS_RQ_REMOVED);
	if (RQ_IOC(rq) && RQ_IOC(rq)->aic)
		atomic_dec(&RQ_IOC(rq)->aic->nr_dispatched);
}

static void as_deactivate_request(struct request_queue *q, struct request *rq)
{
	WARN_ON(RQ_STATE(rq) != AS_RQ_REMOVED);
	RQ_SET_STATE(rq, AS_RQ_DISPATCHED);
	if (RQ_IOC(rq) && RQ_IOC(rq)->aic)
		atomic_inc(&RQ_IOC(rq)->aic->nr_dispatched);
}

/*
 * as_queue_empty tells us if there are requests left in the device. It may
 * not be the case that a driver can get the next request even if the queue
 * is not empty - it is used in the block layer to check for plugging and
 * merging opportunities
 */
static int as_queue_empty(struct request_queue *q)
{
	struct as_data *ad = q->elevator->elevator_data;

	return list_empty(&ad->fifo_list[REQ_ASYNC])
		&& list_empty(&ad->fifo_list[REQ_SYNC]);
}

static int
as_merge(struct request_queue *q, struct request **req, struct bio *bio)
{
	struct as_data *ad = q->elevator->elevator_data;
	sector_t rb_key = bio->bi_sector + bio_sectors(bio);
	struct request *__rq;

	/*
	 * check for front merge
	 */
	__rq = elv_rb_find(&ad->sort_list[bio_data_dir(bio)], rb_key);
	if (__rq && elv_rq_merge_ok(__rq, bio)) {
		*req = __rq;
		return ELEVATOR_FRONT_MERGE;
	}

	return ELEVATOR_NO_MERGE;
}

static void as_merged_request(struct request_queue *q, struct request *req,
			      int type)
{
	struct as_data *ad = q->elevator->elevator_data;

	/*
	 * if the merge was a front merge, we need to reposition request
	 */
	if (type == ELEVATOR_FRONT_MERGE) {
		as_del_rq_rb(ad, req);
		as_add_rq_rb(ad, req);
		/*
		 * Note! At this stage of this and the next function, our next
		 * request may not be optimal - eg the request may have "grown"
		 * behind the disk head. We currently don't bother adjusting.
		 */
	}
}

static void as_merged_requests(struct request_queue *q, struct request *req,
			 	struct request *next)
{
	/*
	 * if next expires before rq, assign its expire time to arq
	 * and move into next position (next will be deleted) in fifo
	 */
	if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) {
		if (time_before(rq_fifo_time(next), rq_fifo_time(req))) {
			list_move(&req->queuelist, &next->queuelist);
			rq_set_fifo_time(req, rq_fifo_time(next));
		}
	}

	/*
	 * kill knowledge of next, this one is a goner
	 */
	as_remove_queued_request(q, next);
	as_put_io_context(next);

	RQ_SET_STATE(next, AS_RQ_MERGED);
}

/*
 * This is executed in a "deferred" process context, by kblockd. It calls the
 * driver's request_fn so the driver can submit that request.
 *
 * IMPORTANT! This guy will reenter the elevator, so set up all queue global
 * state before calling, and don't rely on any state over calls.
 *
 * FIXME! dispatch queue is not a queue at all!
 */
static void as_work_handler(struct work_struct *work)
{
	struct as_data *ad = container_of(work, struct as_data, antic_work);
	struct request_queue *q = ad->q;
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	blk_start_queueing(q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}

static int as_may_queue(struct request_queue *q, int rw)
{
	int ret = ELV_MQUEUE_MAY;
	struct as_data *ad = q->elevator->elevator_data;
	struct io_context *ioc;
	if (ad->antic_status == ANTIC_WAIT_REQ ||
			ad->antic_status == ANTIC_WAIT_NEXT) {
		ioc = as_get_io_context(q->node);
		if (ad->io_context == ioc)
			ret = ELV_MQUEUE_MUST;
		put_io_context(ioc);
	}

	return ret;
}

static void as_exit_queue(elevator_t *e)
{
	struct as_data *ad = e->elevator_data;

	del_timer_sync(&ad->antic_timer);
	kblockd_flush_work(&ad->antic_work);

	BUG_ON(!list_empty(&ad->fifo_list[REQ_SYNC]));
	BUG_ON(!list_empty(&ad->fifo_list[REQ_ASYNC]));

	put_io_context(ad->io_context);
	kfree(ad);
}

/*
 * initialize elevator private data (as_data).
 */
static void *as_init_queue(struct request_queue *q)
{
	struct as_data *ad;

	ad = kmalloc_node(sizeof(*ad), GFP_KERNEL | __GFP_ZERO, q->node);
	if (!ad)
		return NULL;

	ad->q = q; /* Identify what queue the data belongs to */

	/* anticipatory scheduling helpers */
	ad->antic_timer.function = as_antic_timeout;
	ad->antic_timer.data = (unsigned long)q;
	init_timer(&ad->antic_timer);
	INIT_WORK(&ad->antic_work, as_work_handler);

	INIT_LIST_HEAD(&ad->fifo_list[REQ_SYNC]);
	INIT_LIST_HEAD(&ad->fifo_list[REQ_ASYNC]);
	ad->sort_list[REQ_SYNC] = RB_ROOT;
	ad->sort_list[REQ_ASYNC] = RB_ROOT;
	ad->fifo_expire[REQ_SYNC] = default_read_expire;
	ad->fifo_expire[REQ_ASYNC] = default_write_expire;
	ad->antic_expire = default_antic_expire;
	ad->batch_expire[REQ_SYNC] = default_read_batch_expire;
	ad->batch_expire[REQ_ASYNC] = default_write_batch_expire;

	ad->current_batch_expires = jiffies + ad->batch_expire[REQ_SYNC];
	ad->write_batch_count = ad->batch_expire[REQ_ASYNC] / 10;
	if (ad->write_batch_count < 2)
		ad->write_batch_count = 2;

	return ad;
}

/*
 * sysfs parts below
 */

static ssize_t
as_var_show(unsigned int var, char *page)
{
	return sprintf(page, "%d\n", var);
}

static ssize_t
as_var_store(unsigned long *var, const char *page, size_t count)
{
	char *p = (char *) page;

	*var = simple_strtoul(p, &p, 10);
	return count;
}

static ssize_t est_time_show(elevator_t *e, char *page)
{
	struct as_data *ad = e->elevator_data;
	int pos = 0;

	pos += sprintf(page+pos, "%lu %% exit probability\n",
				100*ad->exit_prob/256);
	pos += sprintf(page+pos, "%lu %% probability of exiting without a "
				"cooperating process submitting IO\n",
				100*ad->exit_no_coop/256);
	pos += sprintf(page+pos, "%lu ms new thinktime\n", ad->new_ttime_mean);
	pos += sprintf(page+pos, "%llu sectors new seek distance\n",
				(unsigned long long)ad->new_seek_mean);

	return pos;
}

#define SHOW_FUNCTION(__FUNC, __VAR)				\
static ssize_t __FUNC(elevator_t *e, char *page)		\
{								\
	struct as_data *ad = e->elevator_data;			\
	return as_var_show(jiffies_to_msecs((__VAR)), (page));	\
}
SHOW_FUNCTION(as_read_expire_show, ad->fifo_expire[REQ_SYNC]);
SHOW_FUNCTION(as_write_expire_show, ad->fifo_expire[REQ_ASYNC]);
SHOW_FUNCTION(as_antic_expire_show, ad->antic_expire);
SHOW_FUNCTION(as_read_batch_expire_show, ad->batch_expire[REQ_SYNC]);
SHOW_FUNCTION(as_write_batch_expire_show, ad->batch_expire[REQ_ASYNC]);
#undef SHOW_FUNCTION

#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX)				\
static ssize_t __FUNC(elevator_t *e, const char *page, size_t count)	\
{									\
	struct as_data *ad = e->elevator_data;				\
	int ret = as_var_store(__PTR, (page), count);			\
	if (*(__PTR) < (MIN))						\
		*(__PTR) = (MIN);					\
	else if (*(__PTR) > (MAX))					\
		*(__PTR) = (MAX);					\
	*(__PTR) = msecs_to_jiffies(*(__PTR));				\
	return ret;							\
}
STORE_FUNCTION(as_read_expire_store, &ad->fifo_expire[REQ_SYNC], 0, INT_MAX);
STORE_FUNCTION(as_write_expire_store, &ad->fifo_expire[REQ_ASYNC], 0, INT_MAX);
STORE_FUNCTION(as_antic_expire_store, &ad->antic_expire, 0, INT_MAX);
STORE_FUNCTION(as_read_batch_expire_store,
			&ad->batch_expire[REQ_SYNC], 0, INT_MAX);
STORE_FUNCTION(as_write_batch_expire_store,
			&ad->batch_expire[REQ_ASYNC], 0, INT_MAX);
#undef STORE_FUNCTION

#define AS_ATTR(name) \
	__ATTR(name, S_IRUGO|S_IWUSR, as_##name##_show, as_##name##_store)

static struct elv_fs_entry as_attrs[] = {
	__ATTR_RO(est_time),
	AS_ATTR(read_expire),
	AS_ATTR(write_expire),
	AS_ATTR(antic_expire),
	AS_ATTR(read_batch_expire),
	AS_ATTR(write_batch_expire),
	__ATTR_NULL
};

static struct elevator_type iosched_as = {
	.ops = {
		.elevator_merge_fn = 		as_merge,
		.elevator_merged_fn =		as_merged_request,
		.elevator_merge_req_fn =	as_merged_requests,
		.elevator_dispatch_fn =		as_dispatch_request,
		.elevator_add_req_fn =		as_add_request,
		.elevator_activate_req_fn =	as_activate_request,
		.elevator_deactivate_req_fn = 	as_deactivate_request,
		.elevator_queue_empty_fn =	as_queue_empty,
		.elevator_completed_req_fn =	as_completed_request,
		.elevator_former_req_fn =	elv_rb_former_request,
		.elevator_latter_req_fn =	elv_rb_latter_request,
		.elevator_may_queue_fn =	as_may_queue,
		.elevator_init_fn =		as_init_queue,
		.elevator_exit_fn =		as_exit_queue,
		.trim =				as_trim,
	},

	.elevator_attrs = as_attrs,
	.elevator_name = "anticipatory",
	.elevator_owner = THIS_MODULE,
};

static int __init as_init(void)
{
	elv_register(&iosched_as);

	return 0;
}

static void __exit as_exit(void)
{
	DECLARE_COMPLETION_ONSTACK(all_gone);
	elv_unregister(&iosched_as);
	ioc_gone = &all_gone;
	/* ioc_gone's update must be visible before reading ioc_count */
	smp_wmb();
	if (elv_ioc_count_read(ioc_count))
		wait_for_completion(&all_gone);
	synchronize_rcu();
}

module_init(as_init);
module_exit(as_exit);

MODULE_AUTHOR("Nick Piggin");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("anticipatory IO scheduler");