Skip to content

zvol: Enable zvol threading functionality on FreeBSD #17169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 8, 2025
Merged

zvol: Enable zvol threading functionality on FreeBSD #17169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions include/sys/zvol_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,32 @@ typedef struct zvol_state {
boolean_t zv_threading; /* volthreading property */
} zvol_state_t;

/*
* zvol taskqs
*/
typedef struct zv_taskq {
uint_t tqs_cnt;
taskq_t **tqs_taskq;
} zv_taskq_t;

typedef struct zv_request_stack {
zvol_state_t *zv;
struct bio *bio;
#ifdef __linux__
struct request *rq;
#endif
} zv_request_t;

typedef struct zv_request_task {
zv_request_t zvr;
taskq_ent_t ent;
} zv_request_task_t;

/*
* Switch taskq at multiple of 512 MB offset. This can be set to a lower value
* to utilize more threads for small files but may affect prefetch hits.
*/
#define ZVOL_TASKQ_OFFSET_SHIFT 29

extern krwlock_t zvol_state_lock;
#define ZVOL_HT_SIZE 1024
Expand All @@ -69,6 +95,10 @@ extern zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE];

extern unsigned int zvol_volmode;
extern unsigned int zvol_inhibit_dev;
extern unsigned int zvol_threads;
extern unsigned int zvol_num_taskqs;
extern unsigned int zvol_request_sync;
extern zv_taskq_t zvol_taskqs;

/*
* platform independent functions exported to platform code
Expand All @@ -94,6 +124,8 @@ int zvol_clone_range(zvol_state_handle_t *, uint64_t,
void zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype,
uint64_t off, uint64_t len, uint64_t blksz, const blkptr_t *bps,
size_t nbps);
zv_request_task_t *zv_request_task_create(zv_request_t zvr);
void zv_request_task_free(zv_request_task_t *task);

/*
* platform dependent functions exported to platform independent code
Expand Down
167 changes: 69 additions & 98 deletions module/os/freebsd/zfs/zvol_os.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
#include <geom/geom.h>
#include <sys/zvol.h>
#include <sys/zvol_impl.h>
#include <cityhash.h>

#include "zfs_namecheck.h"

Expand All @@ -112,12 +113,6 @@
#define ZVOL_RW_READ_HELD RW_READ_HELD
#endif

enum zvol_geom_state {
ZVOL_GEOM_UNINIT,
ZVOL_GEOM_STOPPED,
ZVOL_GEOM_RUNNING,
};

struct zvol_state_os {
#define zso_dev _zso_state._zso_dev
#define zso_geom _zso_state._zso_geom
Expand All @@ -131,9 +126,6 @@ struct zvol_state_os {
/* volmode=geom */
struct zvol_state_geom {
struct g_provider *zsg_provider;
struct bio_queue_head zsg_queue;
struct mtx zsg_queue_mtx;
enum zvol_geom_state zsg_state;
} _zso_geom;
} _zso_state;
int zso_dying;
Expand Down Expand Up @@ -169,7 +161,7 @@ static d_close_t zvol_cdev_close;
static d_ioctl_t zvol_cdev_ioctl;
static d_read_t zvol_cdev_read;
static d_write_t zvol_cdev_write;
static d_strategy_t zvol_geom_bio_strategy;
static d_strategy_t zvol_cdev_bio_strategy;
static d_kqfilter_t zvol_cdev_kqfilter;

static struct cdevsw zvol_cdevsw = {
Expand All @@ -181,7 +173,7 @@ static struct cdevsw zvol_cdevsw = {
.d_ioctl = zvol_cdev_ioctl,
.d_read = zvol_cdev_read,
.d_write = zvol_cdev_write,
.d_strategy = zvol_geom_bio_strategy,
.d_strategy = zvol_cdev_bio_strategy,
.d_kqfilter = zvol_cdev_kqfilter,
};

Expand All @@ -205,13 +197,11 @@ DECLARE_GEOM_CLASS(zfs_zvol_class, zfs_zvol);

static int zvol_geom_open(struct g_provider *pp, int flag, int count);
static int zvol_geom_close(struct g_provider *pp, int flag, int count);
static void zvol_geom_run(zvol_state_t *zv);
static void zvol_geom_destroy(zvol_state_t *zv);
static int zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace);
static void zvol_geom_worker(void *arg);
static void zvol_geom_bio_start(struct bio *bp);
static int zvol_geom_bio_getattr(struct bio *bp);
/* static d_strategy_t zvol_geom_bio_strategy; (declared elsewhere) */
static void zvol_geom_bio_strategy(struct bio *bp, boolean_t sync);

/*
* GEOM mode implementation
Expand Down Expand Up @@ -419,20 +409,6 @@ zvol_geom_close(struct g_provider *pp, int flag, int count)
return (0);
}

static void
zvol_geom_run(zvol_state_t *zv)
{
struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
struct g_provider *pp = zsg->zsg_provider;

ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

g_error_provider(pp, 0);

kproc_kthread_add(zvol_geom_worker, zv, &system_proc, NULL, 0, 0,
"zfskern", "zvol %s", pp->name + sizeof (ZVOL_DRIVER));
}

static void
zvol_geom_destroy(zvol_state_t *zv)
{
Expand All @@ -443,9 +419,6 @@ zvol_geom_destroy(zvol_state_t *zv)

g_topology_assert();

mutex_enter(&zv->zv_state_lock);
VERIFY3S(zsg->zsg_state, ==, ZVOL_GEOM_RUNNING);
mutex_exit(&zv->zv_state_lock);
zsg->zsg_provider = NULL;
g_wither_geom(pp->geom, ENXIO);
}
Expand Down Expand Up @@ -516,44 +489,10 @@ zvol_geom_access(struct g_provider *pp, int acr, int acw, int ace)
return (error);
}

static void
zvol_geom_worker(void *arg)
{
zvol_state_t *zv = arg;
struct zvol_state_geom *zsg = &zv->zv_zso->zso_geom;
struct bio *bp;

ASSERT3S(zv->zv_volmode, ==, ZFS_VOLMODE_GEOM);

thread_lock(curthread);
sched_prio(curthread, PRIBIO);
thread_unlock(curthread);

for (;;) {
mtx_lock(&zsg->zsg_queue_mtx);
bp = bioq_takefirst(&zsg->zsg_queue);
if (bp == NULL) {
if (zsg->zsg_state == ZVOL_GEOM_STOPPED) {
zsg->zsg_state = ZVOL_GEOM_RUNNING;
wakeup(&zsg->zsg_state);
mtx_unlock(&zsg->zsg_queue_mtx);
kthread_exit();
}
msleep(&zsg->zsg_queue, &zsg->zsg_queue_mtx,
PRIBIO | PDROP, "zvol:io", 0);
continue;
}
mtx_unlock(&zsg->zsg_queue_mtx);
zvol_geom_bio_strategy(bp);
}
}

static void
zvol_geom_bio_start(struct bio *bp)
{
zvol_state_t *zv = bp->bio_to->private;
struct zvol_state_geom *zsg;
boolean_t first;

if (zv == NULL) {
g_io_deliver(bp, ENXIO);
Expand All @@ -565,18 +504,8 @@ zvol_geom_bio_start(struct bio *bp)
return;
}

if (!THREAD_CAN_SLEEP()) {
zsg = &zv->zv_zso->zso_geom;
mtx_lock(&zsg->zsg_queue_mtx);
first = (bioq_first(&zsg->zsg_queue) == NULL);
bioq_insert_tail(&zsg->zsg_queue, bp);
mtx_unlock(&zsg->zsg_queue_mtx);
if (first)
wakeup_one(&zsg->zsg_queue);
return;
}

zvol_geom_bio_strategy(bp);
zvol_geom_bio_strategy(bp, !g_is_geom_thread(curthread) &&
THREAD_CAN_SLEEP());
}

static int
Expand Down Expand Up @@ -660,9 +589,10 @@ zvol_cdev_kqfilter(struct cdev *dev, struct knote *kn)
}

static void
zvol_geom_bio_strategy(struct bio *bp)
zvol_strategy_impl(zv_request_t *zvr)
{
zvol_state_t *zv;
struct bio *bp;
uint64_t off, volsize;
size_t resid;
char *addr;
Expand All @@ -673,11 +603,8 @@ zvol_geom_bio_strategy(struct bio *bp)
boolean_t is_dumpified;
boolean_t commit;

if (bp->bio_to)
zv = bp->bio_to->private;
else
zv = bp->bio_dev->si_drv2;

bp = zvr->bio;
zv = zvr->zv;
if (zv == NULL) {
error = SET_ERROR(ENXIO);
goto out;
Expand Down Expand Up @@ -813,6 +740,63 @@ zvol_geom_bio_strategy(struct bio *bp)
biofinish(bp, NULL, error);
}

static void
zvol_strategy_task(void *arg)
{
zv_request_task_t *task = arg;

zvol_strategy_impl(&task->zvr);
zv_request_task_free(task);
}

static void
zvol_geom_bio_strategy(struct bio *bp, boolean_t sync)
{
zv_taskq_t *ztqs = &zvol_taskqs;
zv_request_task_t *task;
zvol_state_t *zv;
uint_t tq_idx;
uint_t taskq_hash;
int error;

if (bp->bio_to)
zv = bp->bio_to->private;
else
zv = bp->bio_dev->si_drv2;

if (zv == NULL) {
error = SET_ERROR(ENXIO);
if (bp->bio_to)
g_io_deliver(bp, error);
else
biofinish(bp, NULL, error);
return;
}

zv_request_t zvr = {
.zv = zv,
.bio = bp,
};

if (sync || zvol_request_sync) {
zvol_strategy_impl(&zvr);
return;
}

taskq_hash = cityhash3((uintptr_t)zv, curcpu, bp->bio_offset >>
ZVOL_TASKQ_OFFSET_SHIFT);
tq_idx = taskq_hash % ztqs->tqs_cnt;
task = zv_request_task_create(zvr);
taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx], zvol_strategy_task, task,
0, &task->ent);
}

static void
zvol_cdev_bio_strategy(struct bio *bp)
{
zvol_geom_bio_strategy(bp, B_FALSE);
}

/*
* Character device mode implementation
*/
Expand Down Expand Up @@ -1352,7 +1336,6 @@ zvol_os_free(zvol_state_t *zv)
g_topology_lock();
zvol_geom_destroy(zv);
g_topology_unlock();
mtx_destroy(&zsg->zsg_queue_mtx);
} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
struct cdev *dev = zsd->zsd_cdev;
Expand Down Expand Up @@ -1432,9 +1415,6 @@ zvol_os_create_minor(const char *name)
struct g_provider *pp;
struct g_geom *gp;

zsg->zsg_state = ZVOL_GEOM_UNINIT;
mtx_init(&zsg->zsg_queue_mtx, "zvol", NULL, MTX_DEF);

g_topology_lock();
gp = g_new_geomf(&zfs_zvol_class, "zfs::zvol::%s", name);
gp->start = zvol_geom_bio_start;
Expand All @@ -1446,7 +1426,6 @@ zvol_os_create_minor(const char *name)
pp->private = zv;

zsg->zsg_provider = pp;
bioq_init(&zsg->zsg_queue);
} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
struct cdev *dev;
Expand Down Expand Up @@ -1502,7 +1481,7 @@ zvol_os_create_minor(const char *name)
dmu_objset_disown(os, B_TRUE, FTAG);

if (error == 0 && volmode == ZFS_VOLMODE_GEOM) {
zvol_geom_run(zv);
g_error_provider(zv->zv_zso->zso_geom.zsg_provider, 0);
g_topology_unlock();
}
out_doi:
Expand All @@ -1529,14 +1508,7 @@ zvol_os_clear_private(zvol_state_t *zv)
if (pp->private == NULL) /* already cleared */
return;

mtx_lock(&zsg->zsg_queue_mtx);
zsg->zsg_state = ZVOL_GEOM_STOPPED;
pp->private = NULL;
wakeup_one(&zsg->zsg_queue);
while (zsg->zsg_state != ZVOL_GEOM_RUNNING)
msleep(&zsg->zsg_state, &zsg->zsg_queue_mtx,
0, "zvol:w", 0);
mtx_unlock(&zsg->zsg_queue_mtx);
ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
} else if (zv->zv_volmode == ZFS_VOLMODE_DEV) {
struct zvol_state_dev *zsd = &zv->zv_zso->zso_dev;
Expand Down Expand Up @@ -1606,8 +1578,7 @@ zvol_busy(void)
int
zvol_init(void)
{
zvol_init_impl();
return (0);
return (zvol_init_impl());
}

void
Expand Down
Loading
Loading