// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"

#include "alloc/accounting.h"

#include "btree/bkey_buf.h"
#include "btree/bset.h"
#include "btree/cache.h"
#include "btree/journal_overlay.h"

#include "journal/read.h"

#include <linux/sort.h>

/*
 * For managing keys we read from the journal: until journal replay works normal
 * btree lookups need to be able to find and return keys from the journal where
 * they overwrite what's in the btree, so we have a special iterator and
 * operations for the regular btree iter code to use:
 */

static inline size_t pos_to_idx(struct journal_keys *keys, size_t pos)
{
	size_t gap_size = keys->size - keys->nr;

	BUG_ON(pos >= keys->gap && pos < keys->gap + gap_size);

	if (pos >= keys->gap)
		pos -= gap_size;
	return pos;
}

static inline size_t idx_to_pos(struct journal_keys *keys, size_t idx)
{
	size_t gap_size = keys->size - keys->nr;

	if (idx >= keys->gap)
		idx += gap_size;
	return idx;
}

static inline struct journal_key *idx_to_key(struct journal_keys *keys, size_t idx)
{
	return keys->data + idx_to_pos(keys, idx);
}

static size_t __bch2_journal_key_search(struct journal_keys *keys,
					enum btree_id id, unsigned level,
					struct bpos pos)
{
	struct bch_fs *c = container_of(keys, struct bch_fs, journal_keys);
	size_t l = 0, r = keys->nr, m;

	while (l < r) {
		m = l + ((r - l) >> 1);
		if (__journal_key_cmp(c, id, level, pos, idx_to_key(keys, m)) > 0)
			l = m + 1;
		else
			r = m;
	}

	BUG_ON(l < keys->nr &&
	       __journal_key_cmp(c, id, level, pos, idx_to_key(keys, l)) > 0);

	BUG_ON(l &&
	       __journal_key_cmp(c, id, level, pos, idx_to_key(keys, l - 1)) <= 0);

	return l;
}

static size_t bch2_journal_key_search(struct journal_keys *keys,
				      enum btree_id id, unsigned level,
				      struct bpos pos)
{
	return idx_to_pos(keys, __bch2_journal_key_search(keys, id, level, pos));
}

static inline struct journal_key_range_overwritten *__overwrite_range(struct journal_keys *keys, u32 idx)
{
	return idx ? keys->overwrites.data + idx : NULL;
}

static inline struct journal_key_range_overwritten *overwrite_range(struct journal_keys *keys, u32 idx)
{
	return idx ? rcu_dereference(keys->overwrites.data) + idx : NULL;
}

/* Returns first non-overwritten key >= search key: */
const struct bkey_i *bch2_journal_keys_peek_max(struct bch_fs *c, enum btree_id btree_id,
						unsigned level, struct bpos pos,
						struct bpos end_pos, size_t *idx)
{
	struct journal_keys *keys = &c->journal_keys;
	unsigned iters = 0;
	struct journal_key *k;

	BUG_ON(*idx > keys->nr);
search:
	if (!*idx)
		*idx = __bch2_journal_key_search(keys, btree_id, level, pos);

	while (*idx &&
	       __journal_key_cmp(c, btree_id, level, end_pos, idx_to_key(keys, *idx - 1)) <= 0) {
		--(*idx);
		iters++;
		if (iters == 10) {
			*idx = 0;
			goto search;
		}
	}

	const struct bkey_i *ret = NULL;
	rcu_read_lock(); /* for overwritten_ranges */

	while ((k = *idx < keys->nr ? idx_to_key(keys, *idx) : NULL)) {
		if (__journal_key_cmp(c, btree_id, level, end_pos, k) < 0)
			break;

		if (k->overwritten) {
			if (k->overwritten_range)
				*idx = overwrite_range(keys, k->overwritten_range)->end;
			else
				*idx += 1;
			continue;
		}

		if (__journal_key_cmp(c, btree_id, level, pos, k) <= 0) {
			ret = journal_key_k(c, k);
			break;
		}

		(*idx)++;
		iters++;
		if (iters == 10) {
			*idx = 0;
			rcu_read_unlock();
			goto search;
		}
	}

	rcu_read_unlock();
	return ret;
}

const struct bkey_i *bch2_journal_keys_peek_prev_min(struct bch_fs *c, enum btree_id btree_id,
						     unsigned level, struct bpos pos,
						     struct bpos end_pos, size_t *idx)
{
	struct journal_keys *keys = &c->journal_keys;
	unsigned iters = 0;
	struct journal_key *k;

	BUG_ON(*idx > keys->nr);

	if (!keys->nr)
		return NULL;
search:
	if (!*idx)
		*idx = __bch2_journal_key_search(keys, btree_id, level, pos);

	while (*idx < keys->nr &&
	       __journal_key_cmp(c, btree_id, level, end_pos, idx_to_key(keys, *idx)) >= 0) {
		(*idx)++;
		iters++;
		if (iters == 10) {
			*idx = 0;
			goto search;
		}
	}

	if (*idx == keys->nr)
		--(*idx);

	const struct bkey_i *ret = NULL;
	rcu_read_lock(); /* for overwritten_ranges */

	while (true) {
		k = idx_to_key(keys, *idx);
		if (__journal_key_cmp(c, btree_id, level, end_pos, k) > 0)
			break;

		if (k->overwritten) {
			if (k->overwritten_range)
				*idx = overwrite_range(keys, k->overwritten_range)->start;
			if (!*idx)
				break;
			--(*idx);
			continue;
		}

		if (__journal_key_cmp(c, btree_id, level, pos, k) >= 0) {
			ret = journal_key_k(c, k);
			break;
		}

		if (!*idx)
			break;
		--(*idx);
		iters++;
		if (iters == 10) {
			*idx = 0;
			goto search;
		}
	}

	rcu_read_unlock();
	return ret;
}

const struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *c, enum btree_id btree_id,
						 unsigned level, struct bpos pos)
{
	size_t idx = 0;

	return bch2_journal_keys_peek_max(c, btree_id, level, pos, pos, &idx);
}

static void journal_iter_verify(struct journal_iter *iter)
{
#ifdef CONFIG_BCACHEFS_DEBUG
	struct journal_keys *keys = iter->keys;
	size_t gap_size = keys->size - keys->nr;

	BUG_ON(iter->idx >= keys->gap &&
	       iter->idx <  keys->gap + gap_size);

	if (iter->idx < keys->size) {
		struct journal_key *k = keys->data + iter->idx;

		int cmp = __journal_key_btree_cmp(iter->btree_id, iter->level, k);
		BUG_ON(cmp > 0);
	}
#endif
}

static void journal_iters_fix(struct bch_fs *c)
{
	struct journal_keys *keys = &c->journal_keys;
	/* The key we just inserted is immediately before the gap: */
	size_t gap_end = keys->gap + (keys->size - keys->nr);
	struct journal_key *new_key = &keys->data[keys->gap - 1];
	struct journal_iter *iter;

	/*
	 * If an iterator points one after the key we just inserted, decrement
	 * the iterator so it points at the key we just inserted - if the
	 * decrement was unnecessary, bch2_btree_and_journal_iter_peek() will
	 * handle that:
	 */
	list_for_each_entry(iter, &c->journal_iters, list) {
		journal_iter_verify(iter);
		if (iter->idx		== gap_end &&
		    new_key->btree_id	== iter->btree_id &&
		    new_key->level	== iter->level)
			iter->idx = keys->gap - 1;
		journal_iter_verify(iter);
	}
}

static void journal_iters_move_gap(struct bch_fs *c, size_t old_gap, size_t new_gap)
{
	struct journal_keys *keys = &c->journal_keys;
	struct journal_iter *iter;
	size_t gap_size = keys->size - keys->nr;

	list_for_each_entry(iter, &c->journal_iters, list) {
		if (iter->idx > old_gap)
			iter->idx -= gap_size;
		if (iter->idx >= new_gap)
			iter->idx += gap_size;
	}
}

int bch2_journal_key_insert_take(struct bch_fs *c, enum btree_id id,
				 unsigned level, struct bkey_i *k)
{
	struct journal_key n = {
		.btree_id	= id,
		.level		= level,
		.allocated	= true,
		.allocated_k	= k,
	};
	struct journal_keys *keys = &c->journal_keys;

	/*
	 * Before journal_keys_sort (btree not yet running), stash keys
	 * in pre_sort so they survive the reset. They get merged into
	 * the main array after sorting.
	 */
	if (!test_bit(BCH_FS_btree_running, &c->flags))
		return darray_push(&keys->pre_sort, n)
			? bch_err_throw(c, ENOMEM_journal_key_insert) : 0;

	size_t idx = bch2_journal_key_search(keys, id, level, k->k.p);

	BUG_ON(test_bit(BCH_FS_may_go_rw, &c->flags));
	BUG_ON(test_bit(BCH_FS_rw, &c->flags));
	BUG_ON(current != c->recovery_task);

	if (idx < keys->size &&
	    journal_key_cmp(c, &n, &keys->data[idx]) == 0) {
		struct bkey_i *o = journal_key_k(c, &keys->data[idx]);

		if (k->k.type == KEY_TYPE_accounting &&
		    o->k.type == KEY_TYPE_accounting) {
			if (!keys->data[idx].allocated)
				goto insert;

			bch2_accounting_accumulate(bkey_i_to_accounting(k),
						   bkey_i_to_s_c_accounting(o));
		}

		if (keys->data[idx].allocated)
			kfree(keys->data[idx].allocated_k);
		keys->data[idx] = n;
		return 0;
	}
insert:
	if (idx > keys->gap)
		idx -= keys->size - keys->nr;

	size_t old_gap = keys->gap;

	if (keys->nr == keys->size) {
		journal_iters_move_gap(c, old_gap, keys->size);
		old_gap = keys->size;

		struct journal_keys new_keys = {
			.nr			= keys->nr,
			.size			= max_t(size_t, keys->size, 8) * 2,
		};

		new_keys.data = bch2_kvmalloc(new_keys.size * sizeof(new_keys.data[0]), GFP_KERNEL);
		if (!new_keys.data) {
			bch_err(c, "%s: error allocating new key array (size %zu)",
				__func__, new_keys.size);
			return bch_err_throw(c, ENOMEM_journal_key_insert);
		}

		/* Since @keys was full, there was no gap: */
		memcpy(new_keys.data, keys->data, sizeof(keys->data[0]) * keys->nr);
		kvfree(keys->data);
		keys->data	= new_keys.data;
		keys->nr	= new_keys.nr;
		keys->size	= new_keys.size;

		/* And now the gap is at the end: */
		keys->gap	= keys->nr;
	}

	journal_iters_move_gap(c, old_gap, idx);

	move_gap(keys, idx);

	keys->nr++;
	keys->data[keys->gap++] = n;

	journal_iters_fix(c);

	return 0;
}

/*
 * Can only be used from the recovery thread while we're still RO - can't be
 * used once we've got RW, as journal_keys is at that point used by multiple
 * threads:
 */
int bch2_journal_key_insert(struct bch_fs *c, enum btree_id id,
			    unsigned level, struct bkey_i *k)
{
	struct bkey_i *n;
	int ret;

	n = kmalloc(bkey_bytes(&k->k), GFP_KERNEL);
	if (!n)
		return bch_err_throw(c, ENOMEM_journal_key_insert);

	bkey_copy(n, k);
	ret = bch2_journal_key_insert_take(c, id, level, n);
	if (ret)
		kfree(n);
	return ret;
}

int bch2_journal_key_delete(struct bch_fs *c, enum btree_id id,
			    unsigned level, struct bpos pos)
{
	struct bkey_i whiteout;

	bkey_init(&whiteout.k);
	whiteout.k.p = pos;

	return bch2_journal_key_insert(c, id, level, &whiteout);
}

bool bch2_key_deleted_in_journal(struct btree_trans *trans, enum btree_id btree,
				 unsigned level, struct bpos pos)
{
	if (!trans->journal_replay_not_finished)
		return false;

	struct bch_fs *c = trans->c;
	struct journal_keys *keys = &c->journal_keys;
	size_t idx = bch2_journal_key_search(keys, btree, level, pos);

	if (idx >= keys->size ||
	    keys->data[idx].btree_id	!= btree ||
	    keys->data[idx].level	!= level)
		return false;

	struct bkey_i *k = journal_key_k(c, &keys->data[idx]);
	return bpos_eq(k->k.p, pos) && bkey_deleted(&k->k);
}

static void __bch2_journal_key_overwritten(struct journal_keys *keys, size_t pos)
{
	struct journal_key *k = keys->data + pos;
	size_t idx = pos_to_idx(keys, pos);

	lockdep_assert_held(&keys->overwrite_lock);

	k->overwritten = true;

	struct journal_key *prev = idx > 0 ? keys->data + idx_to_pos(keys, idx - 1) : NULL;
	struct journal_key *next = idx + 1 < keys->nr ? keys->data + idx_to_pos(keys, idx + 1) : NULL;

	bool prev_overwritten = prev && prev->overwritten;
	bool next_overwritten = next && next->overwritten;

	struct journal_key_range_overwritten *prev_range =
		prev_overwritten ? __overwrite_range(keys, prev->overwritten_range) : NULL;
	struct journal_key_range_overwritten *next_range =
		next_overwritten ? __overwrite_range(keys, next->overwritten_range) : NULL;

	BUG_ON(prev_range && prev_range->end != idx);
	BUG_ON(next_range && next_range->start != idx + 1);

	if (prev_range && next_range) {
		prev_range->end = next_range->end;

		keys->data[pos].overwritten_range = prev->overwritten_range;

		u32 old = next->overwritten_range;

		for (size_t i = next_range->start; i < next_range->end; i++) {
			struct journal_key *ip = keys->data + idx_to_pos(keys, i);
			BUG_ON(ip->overwritten_range != old);
			ip->overwritten_range = prev->overwritten_range;
		}
	} else if (prev_range) {
		prev_range->end++;
		k->overwritten_range = prev->overwritten_range;
		if (next_overwritten) {
			prev_range->end++;
			next->overwritten_range = prev->overwritten_range;
		}
	} else if (next_range) {
		next_range->start--;
		k->overwritten_range = next->overwritten_range;
		if (prev_overwritten) {
			next_range->start--;
			prev->overwritten_range = next->overwritten_range;
		}
	} else if (prev_overwritten || next_overwritten) {
		/* 0 is a sentinel value */
		if (darray_resize_rcu(&keys->overwrites, max(keys->overwrites.nr + 1, 2)))
			return;

		if (!keys->overwrites.nr)
			darray_push(&keys->overwrites, (struct journal_key_range_overwritten) {});

		darray_push(&keys->overwrites, ((struct journal_key_range_overwritten) {
			.start	= idx - (size_t) prev_overwritten,
			.end	= idx + 1 + (size_t) next_overwritten,
		}));

		smp_wmb();
		u32 r = keys->overwrites.nr - 1;

		k->overwritten_range = r;

		if (prev_overwritten)
			prev->overwritten_range = r;
		if (next_overwritten)
			next->overwritten_range = r;
	}
}

int bch2_journal_key_check_or_overwrite(struct bch_fs *c, enum btree_id btree,
					 unsigned level, struct bpos pos,
					 bool check)
{
	struct journal_keys *keys = &c->journal_keys;
	size_t idx = bch2_journal_key_search(keys, btree, level, pos);

	if (idx				>= keys->size ||
	    keys->data[idx].btree_id	!= btree ||
	    keys->data[idx].level	!= level ||
	    keys->data[idx].overwritten)
		return 0;

	struct bkey_i *k = journal_key_k(c, &keys->data[idx]);

	if (!bpos_eq(k->k.p, pos))
		return 0;

	if (check)
		return keys->data[idx].overwritten ? -1 : 0;

	__bch2_journal_key_overwritten(keys, idx);
	return 0;
}

static void bch2_journal_iter_advance(struct journal_iter *iter)
{
	if (iter->idx < iter->keys->size) {
		iter->idx++;
		if (iter->idx == iter->keys->gap)
			iter->idx += iter->keys->size - iter->keys->nr;
	}
}

static struct bkey_s_c bch2_journal_iter_peek(struct bch_fs *c, struct journal_iter *iter)
{
	journal_iter_verify(iter);

	guard(rcu)();
	while (iter->idx < iter->keys->size) {
		struct journal_key *k = iter->keys->data + iter->idx;

		int cmp = __journal_key_btree_cmp(iter->btree_id, iter->level, k);
		if (cmp < 0)
			break;
		BUG_ON(cmp);

		if (!k->overwritten)
			return bkey_i_to_s_c(journal_key_k(c, k));

		if (k->overwritten_range)
			iter->idx = idx_to_pos(iter->keys, overwrite_range(iter->keys, k->overwritten_range)->end);
		else
			bch2_journal_iter_advance(iter);
	}

	return bkey_s_c_null;
}

static void bch2_journal_iter_exit(struct journal_iter *iter)
{
	list_del(&iter->list);
}

static void bch2_journal_iter_init(struct bch_fs *c,
				   struct journal_iter *iter,
				   enum btree_id id, unsigned level,
				   struct bpos pos)
{
	iter->btree_id	= id;
	iter->level	= level;
	iter->keys	= &c->journal_keys;
	iter->idx	= bch2_journal_key_search(&c->journal_keys, id, level, pos);

	journal_iter_verify(iter);
}

static struct bkey_s_c bch2_journal_iter_peek_btree(struct btree_and_journal_iter *iter)
{
	return bch2_btree_node_iter_peek_unpack(&iter->node_iter,
						iter->b, &iter->unpacked);
}

static void bch2_journal_iter_advance_btree(struct btree_and_journal_iter *iter)
{
	bch2_btree_node_iter_advance(&iter->node_iter, iter->b);
}

void bch2_btree_and_journal_iter_advance(struct btree_and_journal_iter *iter)
{
	if (bpos_eq(iter->pos, SPOS_MAX))
		iter->at_end = true;
	else
		iter->pos = bpos_successor(iter->pos);
}

static void btree_and_journal_iter_prefetch(struct btree_and_journal_iter *_iter)
{
	struct btree_and_journal_iter iter = *_iter;
	struct bch_fs *c = iter.trans->c;
	unsigned level = iter.journal.level;
	unsigned nr = test_bit(BCH_FS_started, &c->flags)
		? (level > 1 ? 0 :  2)
		: (level > 1 ? 1 : 16);

	iter.prefetch = false;
	iter.fail_if_too_many_whiteouts = true;

	struct bkey_buf tmp __cleanup(bch2_bkey_buf_exit);
	bch2_bkey_buf_init(&tmp);

	while (nr--) {
		bch2_btree_and_journal_iter_advance(&iter);
		struct bkey_s_c k = bch2_btree_and_journal_iter_peek(c, &iter);
		if (!k.k)
			break;

		bch2_bkey_buf_reassemble(&tmp, k);
		bch2_btree_node_prefetch(iter.trans, NULL, tmp.k, iter.journal.btree_id, level - 1);
	}
}

struct bkey_s_c bch2_btree_and_journal_iter_peek(struct bch_fs *c, struct btree_and_journal_iter *iter)
{
	struct bkey_s_c btree_k, journal_k = bkey_s_c_null, ret;
	size_t iters = 0;

	if (iter->prefetch && iter->journal.level)
		btree_and_journal_iter_prefetch(iter);
again:
	if (iter->at_end)
		return bkey_s_c_null;

	iters++;

	if (iters > 20 && iter->fail_if_too_many_whiteouts)
		return bkey_s_c_null;

	while ((btree_k = bch2_journal_iter_peek_btree(iter)).k &&
	       bpos_lt(btree_k.k->p, iter->pos))
		bch2_journal_iter_advance_btree(iter);

	if (iter->trans->journal_replay_not_finished)
		while ((journal_k = bch2_journal_iter_peek(c, &iter->journal)).k &&
		       bpos_lt(journal_k.k->p, iter->pos))
			bch2_journal_iter_advance(&iter->journal);

	ret = journal_k.k &&
		(!btree_k.k || bpos_le(journal_k.k->p, btree_k.k->p))
		? journal_k
		: btree_k;

	if (ret.k && iter->b && bpos_gt(ret.k->p, iter->b->data->max_key))
		ret = bkey_s_c_null;

	if (ret.k) {
		iter->pos = ret.k->p;
		if (bkey_deleted(ret.k)) {
			bch2_btree_and_journal_iter_advance(iter);
			goto again;
		}
	} else {
		iter->pos = SPOS_MAX;
		iter->at_end = true;
	}

	return ret;
}

void bch2_btree_and_journal_iter_exit(struct btree_and_journal_iter *iter)
{
	bch2_journal_iter_exit(&iter->journal);
}

void __bch2_btree_and_journal_iter_init_node_iter(struct btree_trans *trans,
						  struct btree_and_journal_iter *iter,
						  struct btree *b,
						  struct btree_node_iter node_iter,
						  struct bpos pos)
{
	memset(iter, 0, sizeof(*iter));

	iter->trans = trans;
	iter->b = b;
	iter->node_iter = node_iter;
	iter->pos = b->data->min_key;
	iter->at_end = false;
	INIT_LIST_HEAD(&iter->journal.list);

	if (trans->journal_replay_not_finished) {
		bch2_journal_iter_init(trans->c, &iter->journal, b->c.btree_id, b->c.level, pos);
		if (!test_bit(BCH_FS_may_go_rw, &trans->c->flags)) {
			BUG_ON(current != trans->c->recovery_task);
			list_add(&iter->journal.list, &trans->c->journal_iters);
		}
	}
}

/*
 * this version is used by btree_gc before filesystem has gone RW and
 * multithreaded, so uses the journal_iters list:
 */
void bch2_btree_and_journal_iter_init_node_iter(struct btree_trans *trans,
						struct btree_and_journal_iter *iter,
						struct btree *b)
{
	struct btree_node_iter node_iter;

	bch2_btree_node_iter_init_from_start(&node_iter, b);
	__bch2_btree_and_journal_iter_init_node_iter(trans, iter, b, node_iter, b->data->min_key);
}

/* sort and dedup all keys in the journal: */

/*
 * When keys compare equal, oldest compares first:
 */
static int journal_sort_key_cmp(const void *_l, const void *_r, const void *priv)
{
	struct bch_fs *c = (void *) priv;
	const struct journal_key *l = _l;
	const struct journal_key *r = _r;
	int rewind = l->rewind && r->rewind ? -1 : 1;

	int cmp = journal_key_cmp(c, l, r);
	if (cmp)
		return cmp;

	if (l->allocated || r->allocated)
		return cmp_int(l->allocated, r->allocated);

	return ((cmp_int(l->journal_seq_offset, r->journal_seq_offset) ?:
		 cmp_int(l->journal_offset, r->journal_offset)) * rewind);
}

void bch2_journal_keys_put(struct bch_fs *c)
{
	struct journal_keys *keys = &c->journal_keys;

	BUG_ON(atomic_read(&keys->ref) <= 0);

	if (!atomic_dec_and_test(&keys->ref))
		return;

	move_gap(keys, keys->nr);

	darray_for_each(*keys, i)
		if (i->allocated)
			kfree(i->allocated_k);

	kvfree(keys->data);
	keys->data = NULL;
	keys->nr = keys->gap = keys->size = 0;

	darray_for_each(keys->pre_sort, i)
		kfree(i->allocated_k);
	darray_exit(&keys->pre_sort);
	darray_exit(&keys->overwrites);

	struct journal_replay **i;
	struct genradix_iter iter;

	genradix_for_each(&c->journal_entries, iter, i)
		kvfree(*i);
	genradix_free(&c->journal_entries);
}

/*
 * Free the sorted journal keys array without freeing journal_entries,
 * so that bch2_journal_keys_sort() can re-sort from the raw entries
 * (e.g. after changing journal_rewind).
 */
static void bch2_journal_keys_reset(struct bch_fs *c)
{
	struct journal_keys *keys = &c->journal_keys;

	move_gap(keys, keys->nr);

	darray_for_each(*keys, i)
		if (i->allocated)
			kfree(i->allocated_k);

	kvfree(keys->data);
	keys->data = NULL;
	keys->nr = keys->gap = keys->size = 0;

	darray_exit(&keys->overwrites);
}

static void __journal_keys_sort(struct journal_keys *keys)
{
	struct bch_fs *c = container_of(keys, struct bch_fs, journal_keys);

	sort_r_nonatomic(keys->data, keys->nr, sizeof(keys->data[0]),
			 journal_sort_key_cmp, NULL, c);

	cond_resched();

	struct journal_key *dst = keys->data;

	darray_for_each(*keys, src) {
		/*
		 * We don't accumulate accounting keys here because we have to
		 * compare each individual accounting key against the version in
		 * the btree during replay:
		 */
		struct bkey_i *k = journal_key_k(c, src);
		if (k->k.type != KEY_TYPE_accounting &&
		    src + 1 < &darray_top(*keys) &&
		    !journal_key_cmp(c, src, src + 1))
			continue;

		*dst++ = *src;
	}

	keys->nr = dst - keys->data;
}

static bool journal_seq_is_rewound(struct bch_fs *c, u64 seq)
{
	darray_for_each(c->journal.rewind_ranges, range)
		if (seq > range->to && seq <= range->from)
			return true;

	return false;
}

int bch2_journal_keys_sort(struct bch_fs *c)
{
	struct genradix_iter iter;
	struct journal_replay *i, **_i;
	struct journal_keys *keys = &c->journal_keys;
	size_t nr_read = 0;
	size_t nr_extra_sorts = 0;

	/* We may be called more than once - when deciding to rewind because of
	 * opts.scrub_recent_journal_entries */
	bch2_journal_keys_reset(c);

	genradix_for_each(&c->journal_entries, iter, _i) {
		i = *_i;

		if (journal_replay_ignore(i))
			continue;

		cond_resched();

		bool seq_rewound = journal_seq_is_rewound(c, le64_to_cpu(i->j.seq));

		if (seq_rewound && !JSET_HAS_OVERWRITES(&i->j)) {
			bch_err(c, "cannot rewind journal seq %llu: no overwrite entries (journal_transaction_names was off)",
				le64_to_cpu(i->j.seq));
			return bch_err_throw(c, journal_rewind_no_overwrites);
		}

		vstruct_for_each(&i->j, entry) {
			bool rewind = seq_rewound &&
				!entry->level &&
				!btree_id_is_alloc(entry->btree_id);

			if (entry->type != (rewind
					    ? BCH_JSET_ENTRY_overwrite
					    : BCH_JSET_ENTRY_btree_keys))
				continue;

			if (!rewind && le64_to_cpu(i->j.seq) < c->journal_replay_seq_start)
				continue;

			jset_entry_for_each_key(entry, k) {
				struct journal_key n = (struct journal_key) {
					.btree_id	= entry->btree_id,
					.level		= entry->level,
					.rewind		= rewind,
					.journal_seq_offset = journal_entry_radix_idx(c, le64_to_cpu(i->j.seq)),
					.journal_offset	= k->_data - i->j._data,
				};

				if (darray_push(keys, n)) {
					__journal_keys_sort(keys);
					nr_extra_sorts++;

					if (keys->nr * 8 > keys->size * 7) {
						bch_err(c, "Too many journal keys for slowpath; have %zu compacted, buf size %zu, processed %zu keys at seq %llu",
							keys->nr, keys->size, nr_read, le64_to_cpu(i->j.seq));
						return bch_err_throw(c, ENOMEM_journal_keys_sort);
					}

					BUG_ON(darray_push(keys, n));
				}

				nr_read++;
			}
		}
	}

	/*
	 * Merge in any pre-sort keys (inserted before journal_keys_sort,
	 * e.g. from dev_usage_init during offline device add). These
	 * were saved by bch2_journal_keys_reset instead of being freed.
	 * Ownership transfers to the main array.
	 */
	darray_for_each(keys->pre_sort, i)
		darray_push(keys, *i);
	keys->pre_sort.nr = 0;

	__journal_keys_sort(keys);
	keys->gap = keys->nr;

	darray_for_each(*keys, i) {
		struct bkey_i *k = journal_key_k(c, i);
		if (k->k.type == KEY_TYPE_btree_ptr_v2)
			bkey_i_to_btree_ptr_v2(k)->v.mem_ptr = 0;
	}

	CLASS(bch_log_msg_level, msg)(c, nr_extra_sorts ? LOGLEVEL_debug : LOGLEVEL_notice);
	prt_printf(&msg.m, "Journal keys: %zu read, %zu after sorting and compacting", nr_read, keys->nr);
	if (nr_extra_sorts)
		prt_printf(&msg.m, "Required %zu extra sorts due to low memory", nr_extra_sorts);
	return 0;
}

void bch2_shoot_down_journal_keys(struct bch_fs *c, enum btree_id btree,
				  unsigned level_min, unsigned level_max,
				  struct bpos start, struct bpos end)
{
	struct journal_keys *keys = &c->journal_keys;
	size_t dst = 0;

	move_gap(keys, keys->nr);

	darray_for_each(*keys, i) {
		struct bkey_i *k = journal_key_k(c, i);

		if (!(i->btree_id == btree &&
		      i->level >= level_min &&
		      i->level <= level_max &&
		      bpos_ge(k->k.p, start) &&
		      bpos_le(k->k.p, end)))
			keys->data[dst++] = *i;
		else if (i->allocated) {
			kfree(i->allocated_k);
			i->allocated_k = NULL;
		}
	}
	keys->nr = keys->gap = dst;
}

void bch2_journal_keys_dump(struct bch_fs *c)
{
	struct journal_keys *keys = &c->journal_keys;
	CLASS(printbuf, buf)();

	pr_info("%zu keys:", keys->nr);

	move_gap(keys, keys->nr);

	darray_for_each(*keys, i) {
		printbuf_reset(&buf);
		prt_printf(&buf, "btree=");
		bch2_btree_id_to_text(&buf, i->btree_id);
		prt_printf(&buf, " l=%u ", i->level);
		bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(journal_key_k(c, i)));
		pr_err("%s", buf.buf);
	}
}

void bch2_fs_journal_keys_init(struct bch_fs *c)
{
	struct journal_keys *keys = &c->journal_keys;

	atomic_set(&keys->ref, 1);
	keys->initial_ref_held = true;
	mutex_init(&keys->overwrite_lock);
}
