/*
* midcomms.c
*
* This is the appallingly named «mid-level» comms layer.
*
* Its purpose is to take packets from the «real» comms layer,
* split them up into packets and pass them to the interested
* part of the locking mechanism.
*
* It also takes messages from the locking layer, formats them
* into packets and sends them to the comms layer.
*/

#include «dlm_internal.h»
#include «lowcomms.h»
#include «config.h»
#include «lock.h»
#include «midcomms.h»

static void copy_from_cb(void *dst, const void *base, unsigned offset,
unsigned len, unsigned limit)
{
unsigned copy = len;

if ((copy + offset) > limit)
copy = limit – offset;
memcpy(dst, base + offset, copy);
len -= copy;
if (len)
memcpy(dst + copy, base, len);
}

/*
* Called from the low-level comms layer to process a buffer of
* commands.
*
* Only complete messages are processed here, any «spare» bytes from
* the end of a buffer are saved and tacked onto the front of the next
* message that comes in. I doubt this will happen very often but we
* need to be able to cope with it and I don’t want the task to be waiting
* for packets to come in when there is useful work to be done.
*/

int dlm_process_incoming_buffer(int nodeid, const void *base,
unsigned offset, unsigned len, unsigned limit)
{
union {
unsigned char __buf[DLM_INBUF_LEN];
/* this is to force proper alignment on some arches */
union dlm_packet p;
} __tmp;
union dlm_packet *p = &__tmp.p;
int ret = 0;
int err = 0;
uint16_t msglen;
uint32_t lockspace;

while (len > sizeof(struct dlm_header)) {

/* Copy just the header to check the total length. The
message may wrap around the end of the buffer back to the
start, so we need to use a temp buffer and copy_from_cb. */

copy_from_cb(p, base, offset, sizeof(struct dlm_header),
limit);

msglen = le16_to_cpu(p->header.h_length);
lockspace = p->header.h_lockspace;

err = -EINVAL;
if (msglen < sizeof(struct dlm_header))
break;
if (p->header.h_cmd == DLM_MSG) {
if (msglen < sizeof(struct dlm_message))
break;
} else {
if (msglen < sizeof(struct dlm_rcom))
break;
}
err = -E2BIG;
if (msglen > dlm_config.ci_buffer_size) {
log_print(»message size %d from %d too big, buf len %d»,
msglen, nodeid, len);
break;
}
err = 0;

/* If only part of the full message is contained in this
buffer, then do nothing and wait for lowcomms to call
us again later with more data. We return 0 meaning
we’ve consumed none of the input buffer. */

if (msglen > len)
break;

/* Allocate a larger temp buffer if the full message won’t fit
in the buffer on the stack (which should work for most
ordinary messages). */

if (msglen > sizeof(__tmp) && p == &__tmp.p) {
p = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS);
if (p == NULL)
return ret;
}

copy_from_cb(p, base, offset, msglen, limit);

BUG_ON(lockspace != p->header.h_lockspace);

ret += msglen;
offset += msglen;
offset &= (limit – 1);
len -= msglen;

dlm_receive_buffer(p, nodeid);
}

if (p != &__tmp.p)
kfree(p);

return err ? err : ret;
}

#ifndef __MEMORY_DOT_H__
#define __MEMORY_DOT_H__

int dlm_memory_init(void);
void dlm_memory_exit(void);
struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls, int namelen);
void dlm_free_rsb(struct dlm_rsb *r);
struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls);
void dlm_free_lkb(struct dlm_lkb *l);
char *dlm_allocate_lvb(struct dlm_ls *ls);
void dlm_free_lvb(char *l);

#endif /* __MEMORY_DOT_H__ */

#include «dlm_internal.h»
#include «config.h»
#include «memory.h»

static struct kmem_cache *lkb_cache;

int __init dlm_memory_init(void)
{
int ret = 0;

lkb_cache = kmem_cache_create(»dlm_lkb», sizeof(struct dlm_lkb),
__alignof__(struct dlm_lkb), 0, NULL);
if (!lkb_cache)
ret = -ENOMEM;
return ret;
}

void dlm_memory_exit(void)
{
if (lkb_cache)
kmem_cache_destroy(lkb_cache);
}

char *dlm_allocate_lvb(struct dlm_ls *ls)
{
char *p;

p = kzalloc(ls->ls_lvblen, ls->ls_allocation);
return p;
}

void dlm_free_lvb(char *p)
{
kfree(p);
}

/* FIXME: have some minimal space built-in to rsb for the name and
kmalloc a separate name if needed, like dentries are done */

struct dlm_rsb *dlm_allocate_rsb(struct dlm_ls *ls, int namelen)
{
struct dlm_rsb *r;

DLM_ASSERT(namelen <= DLM_RESNAME_MAXLEN,);

r = kzalloc(sizeof(*r) + namelen, ls->ls_allocation);
return r;
}

void dlm_free_rsb(struct dlm_rsb *r)
{
if (r->res_lvbptr)
dlm_free_lvb(r->res_lvbptr);
kfree(r);
}

struct dlm_lkb *dlm_allocate_lkb(struct dlm_ls *ls)
{
struct dlm_lkb *lkb;

lkb = kmem_cache_zalloc(lkb_cache, ls->ls_allocation);
return lkb;
}

void dlm_free_lkb(struct dlm_lkb *lkb)
{
if (lkb->lkb_flags & DLM_IFL_USER) {
struct dlm_user_args *ua;
ua = lkb->lkb_ua;
if (ua) {
if (ua->lksb.sb_lvbptr)
kfree(ua->lksb.sb_lvbptr);
kfree(ua);
}
}
kmem_cache_free(lkb_cache, lkb);
}

#ifndef __MEMBER_DOT_H__
#define __MEMBER_DOT_H__

int dlm_ls_stop(struct dlm_ls *ls);
int dlm_ls_start(struct dlm_ls *ls);
void dlm_clear_members(struct dlm_ls *ls);
void dlm_clear_members_gone(struct dlm_ls *ls);
int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv,int *neg_out);
int dlm_is_removed(struct dlm_ls *ls, int nodeid);
int dlm_is_member(struct dlm_ls *ls, int nodeid);

#endif /* __MEMBER_DOT_H__ */

#include «dlm_internal.h»
#include «lockspace.h»
#include «member.h»
#include «recoverd.h»
#include «recover.h»
#include «rcom.h»
#include «config.h»
#include «lowcomms.h»

static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new)
{
struct dlm_member *memb = NULL;
struct list_head *tmp;
struct list_head *newlist = &new->list;
struct list_head *head = &ls->ls_nodes;

list_for_each(tmp, head) {
memb = list_entry(tmp, struct dlm_member, list);
if (new->nodeid < memb->nodeid)
break;
}

if (!memb)
list_add_tail(newlist, head);
else {
/* FIXME: can use list macro here */
newlist->prev = tmp->prev;
newlist->next = tmp;
tmp->prev->next = newlist;
tmp->prev = newlist;
}
}

static int dlm_add_member(struct dlm_ls *ls, int nodeid)
{
struct dlm_member *memb;
int w, error;

memb = kzalloc(sizeof(struct dlm_member), ls->ls_allocation);
if (!memb)
return -ENOMEM;

w = dlm_node_weight(ls->ls_name, nodeid);
if (w < 0) {
kfree(memb);
return w;
}

error = dlm_lowcomms_connect_node(nodeid);
if (error < 0) {
kfree(memb);
return error;
}

memb->nodeid = nodeid;
memb->weight = w;
add_ordered_member(ls, memb);
ls->ls_num_nodes++;
return 0;
}

static void dlm_remove_member(struct dlm_ls *ls, struct dlm_member *memb)
{
list_move(&memb->list, &ls->ls_nodes_gone);
ls->ls_num_nodes–;
}

int dlm_is_member(struct dlm_ls *ls, int nodeid)
{
struct dlm_member *memb;

list_for_each_entry(memb, &ls->ls_nodes, list) {
if (memb->nodeid == nodeid)
return 1;
}
return 0;
}

int dlm_is_removed(struct dlm_ls *ls, int nodeid)
{
struct dlm_member *memb;

list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
if (memb->nodeid == nodeid)
return 1;
}
return 0;
}

static void clear_memb_list(struct list_head *head)
{
struct dlm_member *memb;

while (!list_empty(head)) {
memb = list_entry(head->next, struct dlm_member, list);
list_del(&memb->list);
kfree(memb);
}
}

void dlm_clear_members(struct dlm_ls *ls)
{
clear_memb_list(&ls->ls_nodes);
ls->ls_num_nodes = 0;
}

void dlm_clear_members_gone(struct dlm_ls *ls)
{
clear_memb_list(&ls->ls_nodes_gone);
}

static void make_member_array(struct dlm_ls *ls)
{
struct dlm_member *memb;
int i, w, x = 0, total = 0, all_zero = 0, *array;

kfree(ls->ls_node_array);
ls->ls_node_array = NULL;

list_for_each_entry(memb, &ls->ls_nodes, list) {
if (memb->weight)
total += memb->weight;
}

/* all nodes revert to weight of 1 if all have weight 0 */

if (!total) {
total = ls->ls_num_nodes;
all_zero = 1;
}

ls->ls_total_weight = total;

array = kmalloc(sizeof(int) * total, ls->ls_allocation);
if (!array)
return;

list_for_each_entry(memb, &ls->ls_nodes, list) {
if (!all_zero && !memb->weight)
continue;

if (all_zero)
w = 1;
else
w = memb->weight;

DLM_ASSERT(x < total, printk("total %d x %d\n", total, x););

for (i = 0; i < w; i++)
array[x++] = memb->nodeid;
}

ls->ls_node_array = array;
}

/* send a status request to all members just to establish comms connections */

static int ping_members(struct dlm_ls *ls)
{
struct dlm_member *memb;
int error = 0;

list_for_each_entry(memb, &ls->ls_nodes, list) {
error = dlm_recovery_stopped(ls);
if (error)
break;
error = dlm_rcom_status(ls, memb->nodeid);
if (error)
break;
}
if (error)
log_debug(ls, «ping_members aborted %d last nodeid %d»,
error, ls->ls_recover_nodeid);
return error;
}

int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out)
{
struct dlm_member *memb, *safe;
int i, error, found, pos = 0, neg = 0, low = -1;

/* previously removed members that we’ve not finished removing need to
count as a negative change so the «neg» recovery steps will happen */

list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
log_debug(ls, «prev removed member %d», memb->nodeid);
neg++;
}

/* move departed members from ls_nodes to ls_nodes_gone */

list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) {
found = 0;
for (i = 0; i < rv->node_count; i++) {
if (memb->nodeid == rv->nodeids[i]) {
found = 1;
break;
}
}

if (!found) {
neg++;
dlm_remove_member(ls, memb);
log_debug(ls, «remove member %d», memb->nodeid);
}
}

/* Add an entry to ls_nodes_gone for members that were removed and
then added again, so that previous state for these nodes will be
cleared during recovery. */

for (i = 0; i < rv->new_count; i++) {
if (!dlm_is_member(ls, rv->new[i]))
continue;
log_debug(ls, «new nodeid %d is a re-added member», rv->new[i]);

memb = kzalloc(sizeof(struct dlm_member), ls->ls_allocation);
if (!memb)
return -ENOMEM;
memb->nodeid = rv->new[i];
list_add_tail(&memb->list, &ls->ls_nodes_gone);
neg++;
}

/* add new members to ls_nodes */

for (i = 0; i < rv->node_count; i++) {
if (dlm_is_member(ls, rv->nodeids[i]))
continue;
dlm_add_member(ls, rv->nodeids[i]);
pos++;
log_debug(ls, «add member %d», rv->nodeids[i]);
}

list_for_each_entry(memb, &ls->ls_nodes, list) {
if (low == -1 || memb->nodeid < low)
low = memb->nodeid;
}
ls->ls_low_nodeid = low;

make_member_array(ls);
dlm_set_recover_status(ls, DLM_RS_NODES);
*neg_out = neg;

error = ping_members(ls);
if (!error || error == -EPROTO) {
/* new_lockspace() may be waiting to know if the config
is good or bad */
ls->ls_members_result = error;
complete(&ls->ls_members_done);
}
if (error)
goto out;

error = dlm_recover_members_wait(ls);
out:
log_debug(ls, «total members %d error %d», ls->ls_num_nodes, error);
return error;
}

/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before
dlm_ls_start() is called on any of them to start the new recovery. */

int dlm_ls_stop(struct dlm_ls *ls)
{
int new;

/*
* Prevent dlm_recv from being in the middle of something when we do
* the stop. This includes ensuring dlm_recv isn’t processing a
* recovery message (rcom), while dlm_recoverd is aborting and
* resetting things from an in-progress recovery. i.e. we want
* dlm_recoverd to abort its recovery without worrying about dlm_recv
* processing an rcom at the same time. Stopping dlm_recv also makes
* it easy for dlm_receive_message() to check locking stopped and add a
* message to the requestqueue without races.
*/

down_write(&ls->ls_recv_active);

/*
* Abort any recovery that’s in progress (see RECOVERY_STOP,
* dlm_recovery_stopped()) and tell any other threads running in the
* dlm to quit any processing (see RUNNING, dlm_locking_stopped()).
*/

spin_lock(&ls->ls_recover_lock);
set_bit(LSFL_RECOVERY_STOP, &ls->ls_flags);
new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags);
ls->ls_recover_seq++;
spin_unlock(&ls->ls_recover_lock);

/*
* Let dlm_recv run again, now any normal messages will be saved on the
* requestqueue for later.
*/

up_write(&ls->ls_recv_active);

/*
* This in_recovery lock does two things:
* 1) Keeps this function from returning until all threads are out
* of locking routines and locking is truely stopped.
* 2) Keeps any new requests from being processed until it’s unlocked
* when recovery is complete.
*/

if (new)
down_write(&ls->ls_in_recovery);

/*
* The recoverd suspend/resume makes sure that dlm_recoverd (if
* running) has noticed RECOVERY_STOP above and quit processing the
* previous recovery.
*/

dlm_recoverd_suspend(ls);
ls->ls_recover_status = 0;
dlm_recoverd_resume(ls);

if (!ls->ls_recover_begin)
ls->ls_recover_begin = jiffies;
return 0;
}

int dlm_ls_start(struct dlm_ls *ls)
{
struct dlm_recover *rv = NULL, *rv_old;
int *ids = NULL, *new = NULL;
int error, ids_count = 0, new_count = 0;

rv = kzalloc(sizeof(struct dlm_recover), ls->ls_allocation);
if (!rv)
return -ENOMEM;

error = dlm_nodeid_list(ls->ls_name, &ids, &ids_count,
&new, &new_count);
if (error < 0)
goto fail;

spin_lock(&ls->ls_recover_lock);

/* the lockspace needs to be stopped before it can be started */

if (!dlm_locking_stopped(ls)) {
spin_unlock(&ls->ls_recover_lock);
log_error(ls, «start ignored: lockspace running»);
error = -EINVAL;
goto fail;
}

rv->nodeids = ids;
rv->node_count = ids_count;
rv->new = new;
rv->new_count = new_count;
rv->seq = ++ls->ls_recover_seq;
rv_old = ls->ls_recover_args;
ls->ls_recover_args = rv;
spin_unlock(&ls->ls_recover_lock);

if (rv_old) {
log_error(ls, «unused recovery %llx %d»,
(unsigned long long)rv_old->seq, rv_old->node_count);
kfree(rv_old->nodeids);
kfree(rv_old->new);
kfree(rv_old);
}

dlm_recoverd_kick(ls);
return 0;

fail:
kfree(rv);
kfree(ids);
kfree(new);
return error;
}

#include «dlm_internal.h»
#include «lockspace.h»
#include «lock.h»
#include «user.h»
#include «memory.h»
#include «config.h»

static int __init init_dlm(void)
{
int error;

error = dlm_memory_init();
if (error)
goto out;

error = dlm_lockspace_init();
if (error)
goto out_mem;

error = dlm_config_init();
if (error)
goto out_lockspace;

error = dlm_register_debugfs();
if (error)
goto out_config;

error = dlm_user_init();
if (error)
goto out_debug;

error = dlm_netlink_init();
if (error)
goto out_user;

error = dlm_plock_init();
if (error)
goto out_netlink;

printk(»DLM (built %s %s) installed\n», __DATE__, __TIME__);

return 0;

out_netlink:
dlm_netlink_exit();
out_user:
dlm_user_exit();
out_debug:
dlm_unregister_debugfs();
out_config:
dlm_config_exit();
out_lockspace:
dlm_lockspace_exit();
out_mem:
dlm_memory_exit();
out:
return error;
}

static void __exit exit_dlm(void)
{
dlm_plock_exit();
dlm_netlink_exit();
dlm_user_exit();
dlm_config_exit();
dlm_memory_exit();
dlm_lockspace_exit();
dlm_unregister_debugfs();
}

module_init(init_dlm);
module_exit(exit_dlm);

MODULE_DESCRIPTION(»Distributed Lock Manager»);
MODULE_AUTHOR(»Red Hat, Inc.»);
MODULE_LICENSE(»GPL»);

EXPORT_SYMBOL_GPL(dlm_new_lockspace);
EXPORT_SYMBOL_GPL(dlm_release_lockspace);
EXPORT_SYMBOL_GPL(dlm_lock);
EXPORT_SYMBOL_GPL(dlm_unlock);

#ifndef __LVB_TABLE_DOT_H__
#define __LVB_TABLE_DOT_H__

extern const int dlm_lvb_operations[8][8];

#endif

#ifndef __LOWCOMMS_DOT_H__
#define __LOWCOMMS_DOT_H__

int dlm_lowcomms_start(void);
void dlm_lowcomms_stop(void);
int dlm_lowcomms_close(int nodeid);
void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc);
void dlm_lowcomms_commit_buffer(void *mh);
int dlm_lowcomms_connect_node(int nodeid);

#endif /* __LOWCOMMS_DOT_H__ */

/*
* lowcomms.c
*
* This is the «low-level» comms layer.
*
* It is responsible for sending/receiving messages
* from other nodes in the cluster.
*
* Cluster nodes are referred to by their nodeids. nodeids are
* simply 32 bit numbers to the locking module – if they need to
* be expanded for the cluster infrastructure then that is its
* responsibility. It is this layer’s
* responsibility to resolve these into IP address or
* whatever it needs for inter-node communication.
*
* The comms level is two kernel threads that deal mainly with
* the receiving of messages from other nodes and passing them
* up to the mid-level comms layer (which understands the
* message format) for execution by the locking core, and
* a send thread which does all the setting up of connections
* to remote nodes and the sending of data. Threads are not allowed
* to send their own data because it may cause them to wait in times
* of high load. Also, this way, the sending thread can collect together
* messages bound for one node and send them in one block.
*
* lowcomms will choose to use either TCP or SCTP as its transport layer
* depending on the configuration variable ‘protocol’. This should be set
* to 0 (default) for TCP or 1 for SCTP. It should be configured using a
* cluster-wide mechanism as it must be the same on all nodes of the cluster
* for the DLM to function.
*
*/

#include
#include
#include
#include #include #include #include #include
#include

#include «dlm_internal.h»
#include «lowcomms.h»
#include «midcomms.h»
#include «config.h»

#define NEEDED_RMEM (4*1024*1024)
#define CONN_HASH_SIZE 32

struct cbuf {
unsigned int base;
unsigned int len;
unsigned int mask;
};

static void cbuf_add(struct cbuf *cb, int n)
{
cb->len += n;
}

static int cbuf_data(struct cbuf *cb)
{
return ((cb->base + cb->len) & cb->mask);
}

static void cbuf_init(struct cbuf *cb, int size)
{
cb->base = cb->len = 0;
cb->mask = size-1;
}

static void cbuf_eat(struct cbuf *cb, int n)
{
cb->len -= n;
cb->base += n;
cb->base &= cb->mask;
}

static bool cbuf_empty(struct cbuf *cb)
{
return cb->len == 0;
}

struct connection {
struct socket *sock; /* NULL if not connected */
uint32_t nodeid; /* So we know who we are in the list */
struct mutex sock_mutex;
unsigned long flags;
#define CF_READ_PENDING 1
#define CF_WRITE_PENDING 2
#define CF_CONNECT_PENDING 3
#define CF_INIT_PENDING 4
#define CF_IS_OTHERCON 5
#define CF_CLOSE 6
struct list_head writequeue; /* List of outgoing writequeue_entries */
spinlock_t writequeue_lock;
int (*rx_action) (struct connection *); /* What to do when active */
void (*connect_action) (struct connection *); /* What to do to connect */
struct page *rx_page;
struct cbuf cb;
int retries;
#define MAX_CONNECT_RETRIES 3
int sctp_assoc;
struct hlist_node list;
struct connection *othercon;
struct work_struct rwork; /* Receive workqueue */
struct work_struct swork; /* Send workqueue */
};
#define sock2con(x) ((struct connection *)(x)->sk_user_data)

/* An entry waiting to be sent */
struct writequeue_entry {
struct list_head list;
struct page *page;
int offset;
int len;
int end;
int users;
struct connection *con;
};

static struct sockaddr_storage *dlm_local_addr[DLM_MAX_ADDR_COUNT];
static int dlm_local_count;

/* Work queues */
static struct workqueue_struct *recv_workqueue;
static struct workqueue_struct *send_workqueue;

static struct hlist_head connection_hash[CONN_HASH_SIZE];
static DEFINE_MUTEX(connections_lock);
static struct kmem_cache *con_cache;

static void process_recv_sockets(struct work_struct *work);
static void process_send_sockets(struct work_struct *work);

/* This is deliberately very simple because most clusters have simple
sequential nodeids, so we should be able to go straight to a connection
struct in the array */
static inline int nodeid_hash(int nodeid)
{
return nodeid & (CONN_HASH_SIZE-1);
}

static struct connection *__find_con(int nodeid)
{
int r;
struct hlist_node *h;
struct connection *con;

r = nodeid_hash(nodeid);

hlist_for_each_entry(con, h, &connection_hash[r], list) {
if (con->nodeid == nodeid)
return con;
}
return NULL;
}

/*
* If ‘allocation’ is zero then we don’t attempt to create a new
* connection structure for this node.
*/
static struct connection *__nodeid2con(int nodeid, gfp_t alloc)
{
struct connection *con = NULL;
int r;

con = __find_con(nodeid);
if (con || !alloc)
return con;

con = kmem_cache_zalloc(con_cache, alloc);
if (!con)
return NULL;

r = nodeid_hash(nodeid);
hlist_add_head(&con->list, &connection_hash[r]);

con->nodeid = nodeid;
mutex_init(&con->sock_mutex);
INIT_LIST_HEAD(&con->writequeue);
spin_lock_init(&con->writequeue_lock);
INIT_WORK(&con->swork, process_send_sockets);
INIT_WORK(&con->rwork, process_recv_sockets);

/* Setup action pointers for child sockets */
if (con->nodeid) {
struct connection *zerocon = __find_con(0);

con->connect_action = zerocon->connect_action;
if (!con->rx_action)
con->rx_action = zerocon->rx_action;
}

return con;
}

/* Loop round all connections */
static void foreach_conn(void (*conn_func)(struct connection *c))
{
int i;
struct hlist_node *h, *n;
struct connection *con;

for (i = 0; i < CONN_HASH_SIZE; i++) {
hlist_for_each_entry_safe(con, h, n, &connection_hash[i], list){
conn_func(con);
}
}
}

static struct connection *nodeid2con(int nodeid, gfp_t allocation)
{
struct connection *con;

mutex_lock(&connections_lock);
con = __nodeid2con(nodeid, allocation);
mutex_unlock(&connections_lock);

return con;
}

/* This is a bit drastic, but only called when things go wrong */
static struct connection *assoc2con(int assoc_id)
{
int i;
struct hlist_node *h;
struct connection *con;

mutex_lock(&connections_lock);

for (i = 0 ; i < CONN_HASH_SIZE; i++) {
hlist_for_each_entry(con, h, &connection_hash[i], list) {
if (con && con->sctp_assoc == assoc_id) {
mutex_unlock(&connections_lock);
return con;
}
}
}
mutex_unlock(&connections_lock);
return NULL;
}

static int nodeid_to_addr(int nodeid, struct sockaddr *retaddr)
{
struct sockaddr_storage addr;
int error;

if (!dlm_local_count)
return -1;

error = dlm_nodeid_to_addr(nodeid, &addr);
if (error)
return error;

if (dlm_local_addr[0]->ss_family == AF_INET) {
struct sockaddr_in *in4 = (struct sockaddr_in *) &addr;
struct sockaddr_in *ret4 = (struct sockaddr_in *) retaddr;
ret4->sin_addr.s_addr = in4->sin_addr.s_addr;
} else {
struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) &addr;
struct sockaddr_in6 *ret6 = (struct sockaddr_in6 *) retaddr;
ipv6_addr_copy(&ret6->sin6_addr, &in6->sin6_addr);
}

return 0;
}

/* Data available on socket or listen socket received a connect */
static void lowcomms_data_ready(struct sock *sk, int count_unused)
{
struct connection *con = sock2con(sk);
if (con && !test_and_set_bit(CF_READ_PENDING, &con->flags))
queue_work(recv_workqueue, &con->rwork);
}

static void lowcomms_write_space(struct sock *sk)
{
struct connection *con = sock2con(sk);

if (con && !test_and_set_bit(CF_WRITE_PENDING, &con->flags))
queue_work(send_workqueue, &con->swork);
}

static inline void lowcomms_connect_sock(struct connection *con)
{
if (test_bit(CF_CLOSE, &con->flags))
return;
if (!test_and_set_bit(CF_CONNECT_PENDING, &con->flags))
queue_work(send_workqueue, &con->swork);
}

static void lowcomms_state_change(struct sock *sk)
{
if (sk->sk_state == TCP_ESTABLISHED)
lowcomms_write_space(sk);
}

int dlm_lowcomms_connect_node(int nodeid)
{
struct connection *con;

/* with sctp there’s no connecting without sending */
if (dlm_config.ci_protocol != 0)
return 0;

if (nodeid == dlm_our_nodeid())
return 0;

con = nodeid2con(nodeid, GFP_NOFS);
if (!con)
return -ENOMEM;
lowcomms_connect_sock(con);
return 0;
}

/* Make a socket active */
static int add_sock(struct socket *sock, struct connection *con)
{
con->sock = sock;

/* Install a data_ready callback */
con->sock->sk->sk_data_ready = lowcomms_data_ready;
con->sock->sk->sk_write_space = lowcomms_write_space;
con->sock->sk->sk_state_change = lowcomms_state_change;
con->sock->sk->sk_user_data = con;
con->sock->sk->sk_allocation = GFP_NOFS;
return 0;
}

/* Add the port number to an IPv6 or 4 sockaddr and return the address
length */
static void make_sockaddr(struct sockaddr_storage *saddr, uint16_t port,
int *addr_len)
{
saddr->ss_family = dlm_local_addr[0]->ss_family;
if (saddr->ss_family == AF_INET) {
struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr;
in4_addr->sin_port = cpu_to_be16(port);
*addr_len = sizeof(struct sockaddr_in);
memset(&in4_addr->sin_zero, 0, sizeof(in4_addr->sin_zero));
} else {
struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)saddr;
in6_addr->sin6_port = cpu_to_be16(port);
*addr_len = sizeof(struct sockaddr_in6);
}
memset((char *)saddr + *addr_len, 0, sizeof(struct sockaddr_storage) – *addr_len);
}

/* Close a remote connection and tidy up */
static void close_connection(struct connection *con, bool and_other)
{
mutex_lock(&con->sock_mutex);

if (con->sock) {
sock_release(con->sock);
con->sock = NULL;
}
if (con->othercon && and_other) {
/* Will only re-enter once. */
close_connection(con->othercon, false);
}
if (con->rx_page) {
__free_page(con->rx_page);
con->rx_page = NULL;
}

con->retries = 0;
mutex_unlock(&con->sock_mutex);
}

/* We only send shutdown messages to nodes that are not part of the cluster */
static void sctp_send_shutdown(sctp_assoc_t associd)
{
static char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))];
struct msghdr outmessage;
struct cmsghdr *cmsg;
struct sctp_sndrcvinfo *sinfo;
int ret;
struct connection *con;

con = nodeid2con(0,0);
BUG_ON(con == NULL);

outmessage.msg_name = NULL;
outmessage.msg_namelen = 0;
outmessage.msg_control = outcmsg;
outmessage.msg_controllen = sizeof(outcmsg);
outmessage.msg_flags = MSG_EOR;

cmsg = CMSG_FIRSTHDR(&outmessage);
cmsg->cmsg_level = IPPROTO_SCTP;
cmsg->cmsg_type = SCTP_SNDRCV;
cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo));
outmessage.msg_controllen = cmsg->cmsg_len;
sinfo = CMSG_DATA(cmsg);
memset(sinfo, 0×00, sizeof(struct sctp_sndrcvinfo));

sinfo->sinfo_flags |= MSG_EOF;
sinfo->sinfo_assoc_id = associd;

ret = kernel_sendmsg(con->sock, &outmessage, NULL, 0, 0);

if (ret != 0)
log_print(»send EOF to node failed: %d», ret);
}

static void sctp_init_failed_foreach(struct connection *con)
{
con->sctp_assoc = 0;
if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) {
if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags))
queue_work(send_workqueue, &con->swork);
}
}

/* INIT failed but we don’t know which node…
restart INIT on all pending nodes */
static void sctp_init_failed(void)
{
mutex_lock(&connections_lock);

foreach_conn(sctp_init_failed_foreach);

mutex_unlock(&connections_lock);
}

/* Something happened to an association */
static void process_sctp_notification(struct connection *con,
struct msghdr *msg, char *buf)
{
union sctp_notification *sn = (union sctp_notification *)buf;

if (sn->sn_header.sn_type == SCTP_ASSOC_CHANGE) {
switch (sn->sn_assoc_change.sac_state) {

case SCTP_COMM_UP:
case SCTP_RESTART:
{
/* Check that the new node is in the lockspace */
struct sctp_prim prim;
int nodeid;
int prim_len, ret;
int addr_len;
struct connection *new_con;
sctp_peeloff_arg_t parg;
int parglen = sizeof(parg);
int err;

/*
* We get this before any data for an association.
* We verify that the node is in the cluster and
* then peel off a socket for it.
*/
if ((int)sn->sn_assoc_change.sac_assoc_id <= 0) {
log_print("COMM_UP for invalid assoc ID %d",
(int)sn->sn_assoc_change.sac_assoc_id);
sctp_init_failed();
return;
}
memset(&prim, 0, sizeof(struct sctp_prim));
prim_len = sizeof(struct sctp_prim);
prim.ssp_assoc_id = sn->sn_assoc_change.sac_assoc_id;

ret = kernel_getsockopt(con->sock,
IPPROTO_SCTP,
SCTP_PRIMARY_ADDR,
(char*)&prim,
&prim_len);
if (ret < 0) {
log_print("getsockopt/sctp_primary_addr on "
"new assoc %d failed : %d",
(int)sn->sn_assoc_change.sac_assoc_id,
ret);

/* Retry INIT later */
new_con = assoc2con(sn->sn_assoc_change.sac_assoc_id);
if (new_con)
clear_bit(CF_CONNECT_PENDING, &con->flags);
return;
}
make_sockaddr(&prim.ssp_addr, 0, &addr_len);
if (dlm_addr_to_nodeid(&prim.ssp_addr, &nodeid)) {
int i;
unsigned char *b=(unsigned char *)&prim.ssp_addr;
log_print(»reject connect from unknown addr»);
for (i=0; i printk("%02x ", b[i]);
printk("\n");
sctp_send_shutdown(prim.ssp_assoc_id);
return;
}

new_con = nodeid2con(nodeid, GFP_NOFS);
if (!new_con)
return;

/* Peel off a new sock */
parg.associd = sn->sn_assoc_change.sac_assoc_id;
ret = kernel_getsockopt(con->sock, IPPROTO_SCTP,
SCTP_SOCKOPT_PEELOFF,
(void *)&parg, &parglen);
if (ret < 0) {
log_print("Can't peel off a socket for "
"connection %d to node %d: err=%d",
parg.associd, nodeid, ret);
return;
}
new_con->sock = sockfd_lookup(parg.sd, &err);
if (!new_con->sock) {
log_print(»sockfd_lookup error %d», err);
return;
}
add_sock(new_con->sock, new_con);
sockfd_put(new_con->sock);

log_print(»connecting to %d sctp association %d»,
nodeid, (int)sn->sn_assoc_change.sac_assoc_id);

/* Send any pending writes */
clear_bit(CF_CONNECT_PENDING, &new_con->flags);
clear_bit(CF_INIT_PENDING, &con->flags);
if (!test_and_set_bit(CF_WRITE_PENDING, &new_con->flags)) {
queue_work(send_workqueue, &new_con->swork);
}
if (!test_and_set_bit(CF_READ_PENDING, &new_con->flags))
queue_work(recv_workqueue, &new_con->rwork);
}
break;

case SCTP_COMM_LOST:
case SCTP_SHUTDOWN_COMP:
{
con = assoc2con(sn->sn_assoc_change.sac_assoc_id);
if (con) {
con->sctp_assoc = 0;
}
}
break;

/* We don’t know which INIT failed, so clear the PENDING flags
* on them all. if assoc_id is zero then it will then try
* again */

case SCTP_CANT_STR_ASSOC:
{
log_print(»Can’t start SCTP association – retrying»);
sctp_init_failed();
}
break;

default:
log_print(»unexpected SCTP assoc change id=%d state=%d»,
(int)sn->sn_assoc_change.sac_assoc_id,
sn->sn_assoc_change.sac_state);
}
}
}

/* Data received from remote end */
static int receive_from_sock(struct connection *con)
{
int ret = 0;
struct msghdr msg = {};
struct kvec iov[2];
unsigned len;
int r;
int call_again_soon = 0;
int nvec;
char incmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))];

mutex_lock(&con->sock_mutex);

if (con->sock == NULL) {
ret = -EAGAIN;
goto out_close;
}

if (con->rx_page == NULL) {
/*
* This doesn’t need to be atomic, but I think it should
* improve performance if it is.
*/
con->rx_page = alloc_page(GFP_ATOMIC);
if (con->rx_page == NULL)
goto out_resched;
cbuf_init(&con->cb, PAGE_CACHE_SIZE);
}

/* Only SCTP needs these really */
memset(&incmsg, 0, sizeof(incmsg));
msg.msg_control = incmsg;
msg.msg_controllen = sizeof(incmsg);

/*
* iov[0] is the bit of the circular buffer between the current end
* point (cb.base + cb.len) and the end of the buffer.
*/
iov[0].iov_len = con->cb.base – cbuf_data(&con->cb);
iov[0].iov_base = page_address(con->rx_page) + cbuf_data(&con->cb);
iov[1].iov_len = 0;
nvec = 1;

/*
* iov[1] is the bit of the circular buffer between the start of the
* buffer and the start of the currently used section (cb.base)
*/
if (cbuf_data(&con->cb) >= con->cb.base) {
iov[0].iov_len = PAGE_CACHE_SIZE – cbuf_data(&con->cb);
iov[1].iov_len = con->cb.base;
iov[1].iov_base = page_address(con->rx_page);
nvec = 2;
}
len = iov[0].iov_len + iov[1].iov_len;

r = ret = kernel_recvmsg(con->sock, &msg, iov, nvec, len,
MSG_DONTWAIT | MSG_NOSIGNAL);
if (ret <= 0)
goto out_close;

/* Process SCTP notifications */
if (msg.msg_flags & MSG_NOTIFICATION) {
msg.msg_control = incmsg;
msg.msg_controllen = sizeof(incmsg);

process_sctp_notification(con, &msg,
page_address(con->rx_page) + con->cb.base);
mutex_unlock(&con->sock_mutex);
return 0;
}
BUG_ON(con->nodeid == 0);

if (ret == len)
call_again_soon = 1;
cbuf_add(&con->cb, ret);
ret = dlm_process_incoming_buffer(con->nodeid,
page_address(con->rx_page),
con->cb.base, con->cb.len,
PAGE_CACHE_SIZE);
if (ret == -EBADMSG) {
log_print(»lowcomms: addr=%p, base=%u, len=%u, »
«iov_len=%u, iov_base[0]=%p, read=%d»,
page_address(con->rx_page), con->cb.base, con->cb.len,
len, iov[0].iov_base, r);
}
if (ret < 0)
goto out_close;
cbuf_eat(&con->cb, ret);

if (cbuf_empty(&con->cb) && !call_again_soon) {
__free_page(con->rx_page);
con->rx_page = NULL;
}

if (call_again_soon)
goto out_resched;
mutex_unlock(&con->sock_mutex);
return 0;

out_resched:
if (!test_and_set_bit(CF_READ_PENDING, &con->flags))
queue_work(recv_workqueue, &con->rwork);
mutex_unlock(&con->sock_mutex);
return -EAGAIN;

out_close:
mutex_unlock(&con->sock_mutex);
if (ret != -EAGAIN) {
close_connection(con, false);
/* Reconnect when there is something to send */
}
/* Don’t return success if we really got EOF */
if (ret == 0)
ret = -EAGAIN;

return ret;
}

/* Listening socket is busy, accept a connection */
static int tcp_accept_from_sock(struct connection *con)
{
int result;
struct sockaddr_storage peeraddr;
struct socket *newsock;
int len;
int nodeid;
struct connection *newcon;
struct connection *addcon;

memset(&peeraddr, 0, sizeof(peeraddr));
result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM,
IPPROTO_TCP, &newsock);
if (result < 0)
return -ENOMEM;

mutex_lock_nested(&con->sock_mutex, 0);

result = -ENOTCONN;
if (con->sock == NULL)
goto accept_err;

newsock->type = con->sock->type;
newsock->ops = con->sock->ops;

result = con->sock->ops->accept(con->sock, newsock, O_NONBLOCK);
if (result < 0)
goto accept_err;

/* Get the connected socket's peer */
memset(&peeraddr, 0, sizeof(peeraddr));
if (newsock->ops->getname(newsock, (struct sockaddr *)&peeraddr,
&len, 2)) {
result = -ECONNABORTED;
goto accept_err;
}

/* Get the new node’s NODEID */
make_sockaddr(&peeraddr, 0, &len);
if (dlm_addr_to_nodeid(&peeraddr, &nodeid)) {
log_print(»connect from non cluster node»);
sock_release(newsock);
mutex_unlock(&con->sock_mutex);
return -1;
}

log_print(»got connection from %d», nodeid);

/* Check to see if we already have a connection to this node. This
* could happen if the two nodes initiate a connection at roughly
* the same time and the connections cross on the wire.
* In this case we store the incoming one in «othercon»
*/
newcon = nodeid2con(nodeid, GFP_NOFS);
if (!newcon) {
result = -ENOMEM;
goto accept_err;
}
mutex_lock_nested(&newcon->sock_mutex, 1);
if (newcon->sock) {
struct connection *othercon = newcon->othercon;

if (!othercon) {
othercon = kmem_cache_zalloc(con_cache, GFP_NOFS);
if (!othercon) {
log_print(»failed to allocate incoming socket»);
mutex_unlock(&newcon->sock_mutex);
result = -ENOMEM;
goto accept_err;
}
othercon->nodeid = nodeid;
othercon->rx_action = receive_from_sock;
mutex_init(&othercon->sock_mutex);
INIT_WORK(&othercon->swork, process_send_sockets);
INIT_WORK(&othercon->rwork, process_recv_sockets);
set_bit(CF_IS_OTHERCON, &othercon->flags);
}
if (!othercon->sock) {
newcon->othercon = othercon;
othercon->sock = newsock;
newsock->sk->sk_user_data = othercon;
add_sock(newsock, othercon);
addcon = othercon;
}
else {
printk(»Extra connection from node %d attempted\n», nodeid);
result = -EAGAIN;
mutex_unlock(&newcon->sock_mutex);
goto accept_err;
}
}
else {
newsock->sk->sk_user_data = newcon;
newcon->rx_action = receive_from_sock;
add_sock(newsock, newcon);
addcon = newcon;
}

mutex_unlock(&newcon->sock_mutex);

/*
* Add it to the active queue in case we got data
* beween processing the accept adding the socket
* to the read_sockets list
*/
if (!test_and_set_bit(CF_READ_PENDING, &addcon->flags))
queue_work(recv_workqueue, &addcon->rwork);
mutex_unlock(&con->sock_mutex);

return 0;

accept_err:
mutex_unlock(&con->sock_mutex);
sock_release(newsock);

if (result != -EAGAIN)
log_print(»error accepting connection from node: %d», result);
return result;
}

static void free_entry(struct writequeue_entry *e)
{
__free_page(e->page);
kfree(e);
}

/* Initiate an SCTP association.
This is a special case of send_to_sock() in that we don’t yet have a
peeled-off socket for this association, so we use the listening socket
and add the primary IP address of the remote node.
*/
static void sctp_init_assoc(struct connection *con)
{
struct sockaddr_storage rem_addr;
char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))];
struct msghdr outmessage;
struct cmsghdr *cmsg;
struct sctp_sndrcvinfo *sinfo;
struct connection *base_con;
struct writequeue_entry *e;
int len, offset;
int ret;
int addrlen;
struct kvec iov[1];

if (test_and_set_bit(CF_INIT_PENDING, &con->flags))
return;

if (con->retries++ > MAX_CONNECT_RETRIES)
return;

if (nodeid_to_addr(con->nodeid, (struct sockaddr *)&rem_addr)) {
log_print(»no address for nodeid %d», con->nodeid);
return;
}
base_con = nodeid2con(0, 0);
BUG_ON(base_con == NULL);

make_sockaddr(&rem_addr, dlm_config.ci_tcp_port, &addrlen);

outmessage.msg_name = &rem_addr;
outmessage.msg_namelen = addrlen;
outmessage.msg_control = outcmsg;
outmessage.msg_controllen = sizeof(outcmsg);
outmessage.msg_flags = MSG_EOR;

spin_lock(&con->writequeue_lock);

if (list_empty(&con->writequeue)) {
spin_unlock(&con->writequeue_lock);
log_print(»writequeue empty for nodeid %d», con->nodeid);
return;
}

e = list_first_entry(&con->writequeue, struct writequeue_entry, list);
len = e->len;
offset = e->offset;
spin_unlock(&con->writequeue_lock);

/* Send the first block off the write queue */
iov[0].iov_base = page_address(e->page)+offset;
iov[0].iov_len = len;

cmsg = CMSG_FIRSTHDR(&outmessage);
cmsg->cmsg_level = IPPROTO_SCTP;
cmsg->cmsg_type = SCTP_SNDRCV;
cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo));
sinfo = CMSG_DATA(cmsg);
memset(sinfo, 0×00, sizeof(struct sctp_sndrcvinfo));
sinfo->sinfo_ppid = cpu_to_le32(dlm_our_nodeid());
outmessage.msg_controllen = cmsg->cmsg_len;

ret = kernel_sendmsg(base_con->sock, &outmessage, iov, 1, len);
if (ret < 0) {
log_print("Send first packet to node %d failed: %d",
con->nodeid, ret);

/* Try again later */
clear_bit(CF_CONNECT_PENDING, &con->flags);
clear_bit(CF_INIT_PENDING, &con->flags);
}
else {
spin_lock(&con->writequeue_lock);
e->offset += ret;
e->len -= ret;

if (e->len == 0 && e->users == 0) {
list_del(&e->list);
free_entry(e);
}
spin_unlock(&con->writequeue_lock);
}
}

/* Connect a new socket to its peer */
static void tcp_connect_to_sock(struct connection *con)
{
int result = -EHOSTUNREACH;
struct sockaddr_storage saddr, src_addr;
int addr_len;
struct socket *sock = NULL;

if (con->nodeid == 0) {
log_print(»attempt to connect sock 0 foiled»);
return;
}

mutex_lock(&con->sock_mutex);
if (con->retries++ > MAX_CONNECT_RETRIES)
goto out;

/* Some odd races can cause double-connects, ignore them */
if (con->sock) {
result = 0;
goto out;
}

/* Create a socket to communicate with */
result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM,
IPPROTO_TCP, &sock);
if (result < 0)
goto out_err;

memset(&saddr, 0, sizeof(saddr));
if (dlm_nodeid_to_addr(con->nodeid, &saddr))
goto out_err;

sock->sk->sk_user_data = con;
con->rx_action = receive_from_sock;
con->connect_action = tcp_connect_to_sock;
add_sock(sock, con);

/* Bind to our cluster-known address connecting to avoid
routing problems */
memcpy(&src_addr, dlm_local_addr[0], sizeof(src_addr));
make_sockaddr(&src_addr, 0, &addr_len);
result = sock->ops->bind(sock, (struct sockaddr *) &src_addr,
addr_len);
if (result < 0) {
log_print("could not bind for connect: %d", result);
/* This *may* not indicate a critical error */
}

make_sockaddr(&saddr, dlm_config.ci_tcp_port, &addr_len);

log_print("connecting to %d", con->nodeid);
result =
sock->ops->connect(sock, (struct sockaddr *)&saddr, addr_len,
O_NONBLOCK);
if (result == -EINPROGRESS)
result = 0;
if (result == 0)
goto out;

out_err:
if (con->sock) {
sock_release(con->sock);
con->sock = NULL;
} else if (sock) {
sock_release(sock);
}
/*
* Some errors are fatal and this list might need adjusting. For other
* errors we try again until the max number of retries is reached.
*/
if (result != -EHOSTUNREACH && result != -ENETUNREACH &&
result != -ENETDOWN && result != -EINVAL
&& result != -EPROTONOSUPPORT) {
lowcomms_connect_sock(con);
result = 0;
}
out:
mutex_unlock(&con->sock_mutex);
return;
}

static struct socket *tcp_create_listen_sock(struct connection *con,
struct sockaddr_storage *saddr)
{
struct socket *sock = NULL;
int result = 0;
int one = 1;
int addr_len;

if (dlm_local_addr[0]->ss_family == AF_INET)
addr_len = sizeof(struct sockaddr_in);
else
addr_len = sizeof(struct sockaddr_in6);

/* Create a socket to communicate with */
result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_STREAM,
IPPROTO_TCP, &sock);
if (result < 0) {
log_print("Can't create listening comms socket");
goto create_out;
}

result = kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR,
(char *)&one, sizeof(one));

if (result < 0) {
log_print("Failed to set SO_REUSEADDR on socket: %d", result);
}
sock->sk->sk_user_data = con;
con->rx_action = tcp_accept_from_sock;
con->connect_action = tcp_connect_to_sock;
con->sock = sock;

/* Bind to our port */
make_sockaddr(saddr, dlm_config.ci_tcp_port, &addr_len);
result = sock->ops->bind(sock, (struct sockaddr *) saddr, addr_len);
if (result < 0) {
log_print("Can't bind to port %d", dlm_config.ci_tcp_port);
sock_release(sock);
sock = NULL;
con->sock = NULL;
goto create_out;
}
result = kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
(char *)&one, sizeof(one));
if (result < 0) {
log_print("Set keepalive failed: %d", result);
}

result = sock->ops->listen(sock, 5);
if (result < 0) {
log_print("Can't listen on port %d", dlm_config.ci_tcp_port);
sock_release(sock);
sock = NULL;
goto create_out;
}

create_out:
return sock;
}

/* Get local addresses */
static void init_local(void)
{
struct sockaddr_storage sas, *addr;
int i;

dlm_local_count = 0;
for (i = 0; i < DLM_MAX_ADDR_COUNT - 1; i++) {
if (dlm_our_addr(&sas, i))
break;

addr = kmalloc(sizeof(*addr), GFP_KERNEL);
if (!addr)
break;
memcpy(addr, &sas, sizeof(*addr));
dlm_local_addr[dlm_local_count++] = addr;
}
}

/* Bind to an IP address. SCTP allows multiple address so it can do
multi-homing */
static int add_sctp_bind_addr(struct connection *sctp_con,
struct sockaddr_storage *addr,
int addr_len, int num)
{
int result = 0;

if (num == 1)
result = kernel_bind(sctp_con->sock,
(struct sockaddr *) addr,
addr_len);
else
result = kernel_setsockopt(sctp_con->sock, SOL_SCTP,
SCTP_SOCKOPT_BINDX_ADD,
(char *)addr, addr_len);

if (result < 0)
log_print("Can't bind to port %d addr number %d",
dlm_config.ci_tcp_port, num);

return result;
}

/* Initialise SCTP socket and bind to all interfaces */
static int sctp_listen_for_all(void)
{
struct socket *sock = NULL;
struct sockaddr_storage localaddr;
struct sctp_event_subscribe subscribe;
int result = -EINVAL, num = 1, i, addr_len;
struct connection *con = nodeid2con(0, GFP_KERNEL);
int bufsize = NEEDED_RMEM;

if (!con)
return -ENOMEM;

log_print("Using SCTP for communications");

result = sock_create_kern(dlm_local_addr[0]->ss_family, SOCK_SEQPACKET,
IPPROTO_SCTP, &sock);
if (result < 0) {
log_print("Can't create comms socket, check SCTP is loaded");
goto out;
}

/* Listen for events */
memset(&subscribe, 0, sizeof(subscribe));
subscribe.sctp_data_io_event = 1;
subscribe.sctp_association_event = 1;
subscribe.sctp_send_failure_event = 1;
subscribe.sctp_shutdown_event = 1;
subscribe.sctp_partial_delivery_event = 1;

result = kernel_setsockopt(sock, SOL_SOCKET, SO_RCVBUFFORCE,
(char *)&bufsize, sizeof(bufsize));
if (result)
log_print("Error increasing buffer space on socket %d", result);

result = kernel_setsockopt(sock, SOL_SCTP, SCTP_EVENTS,
(char *)&subscribe, sizeof(subscribe));
if (result < 0) {
log_print("Failed to set SCTP_EVENTS on socket: result=%d",
result);
goto create_delsock;
}

/* Init con struct */
sock->sk->sk_user_data = con;
con->sock = sock;
con->sock->sk->sk_data_ready = lowcomms_data_ready;
con->rx_action = receive_from_sock;
con->connect_action = sctp_init_assoc;

/* Bind to all interfaces. */
for (i = 0; i < dlm_local_count; i++) {
memcpy(&localaddr, dlm_local_addr[i], sizeof(localaddr));
make_sockaddr(&localaddr, dlm_config.ci_tcp_port, &addr_len);

result = add_sctp_bind_addr(con, &localaddr, addr_len, num);
if (result)
goto create_delsock;
++num;
}

result = sock->ops->listen(sock, 5);
if (result < 0) {
log_print("Can't set socket listening");
goto create_delsock;
}

return 0;

create_delsock:
sock_release(sock);
con->sock = NULL;
out:
return result;
}

static int tcp_listen_for_all(void)
{
struct socket *sock = NULL;
struct connection *con = nodeid2con(0, GFP_KERNEL);
int result = -EINVAL;

if (!con)
return -ENOMEM;

/* We don’t support multi-homed hosts */
if (dlm_local_addr[1] != NULL) {
log_print(»TCP protocol can’t handle multi-homed hosts, »
«try SCTP»);
return -EINVAL;
}

log_print(»Using TCP for communications»);

sock = tcp_create_listen_sock(con, dlm_local_addr[0]);
if (sock) {
add_sock(sock, con);
result = 0;
}
else {
result = -EADDRINUSE;
}

return result;
}

static struct writequeue_entry *new_writequeue_entry(struct connection *con,
gfp_t allocation)
{
struct writequeue_entry *entry;

entry = kmalloc(sizeof(struct writequeue_entry), allocation);
if (!entry)
return NULL;

entry->page = alloc_page(allocation);
if (!entry->page) {
kfree(entry);
return NULL;
}

entry->offset = 0;
entry->len = 0;
entry->end = 0;
entry->users = 0;
entry->con = con;

return entry;
}

void *dlm_lowcomms_get_buffer(int nodeid, int len, gfp_t allocation, char **ppc)
{
struct connection *con;
struct writequeue_entry *e;
int offset = 0;
int users = 0;

con = nodeid2con(nodeid, allocation);
if (!con)
return NULL;

spin_lock(&con->writequeue_lock);
e = list_entry(con->writequeue.prev, struct writequeue_entry, list);
if ((&e->list == &con->writequeue) ||
(PAGE_CACHE_SIZE – e->end < len)) {
e = NULL;
} else {
offset = e->end;
e->end += len;
users = e->users++;
}
spin_unlock(&con->writequeue_lock);

if (e) {
got_one:
*ppc = page_address(e->page) + offset;
return e;
}

e = new_writequeue_entry(con, allocation);
if (e) {
spin_lock(&con->writequeue_lock);
offset = e->end;
e->end += len;
users = e->users++;
list_add_tail(&e->list, &con->writequeue);
spin_unlock(&con->writequeue_lock);
goto got_one;
}
return NULL;
}

void dlm_lowcomms_commit_buffer(void *mh)
{
struct writequeue_entry *e = (struct writequeue_entry *)mh;
struct connection *con = e->con;
int users;

spin_lock(&con->writequeue_lock);
users = –e->users;
if (users)
goto out;
e->len = e->end – e->offset;
spin_unlock(&con->writequeue_lock);

if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) {
queue_work(send_workqueue, &con->swork);
}
return;

out:
spin_unlock(&con->writequeue_lock);
return;
}

/* Send a message */
static void send_to_sock(struct connection *con)
{
int ret = 0;
const int msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
struct writequeue_entry *e;
int len, offset;

mutex_lock(&con->sock_mutex);
if (con->sock == NULL)
goto out_connect;

spin_lock(&con->writequeue_lock);
for (;;) {
e = list_entry(con->writequeue.next, struct writequeue_entry,
list);
if ((struct list_head *) e == &con->writequeue)
break;

len = e->len;
offset = e->offset;
BUG_ON(len == 0 && e->users == 0);
spin_unlock(&con->writequeue_lock);

ret = 0;
if (len) {
ret = kernel_sendpage(con->sock, e->page, offset, len,
msg_flags);
if (ret == -EAGAIN || ret == 0) {
cond_resched();
goto out;
}
if (ret <= 0)
goto send_error;
}
/* Don't starve people filling buffers */
cond_resched();

spin_lock(&con->writequeue_lock);
e->offset += ret;
e->len -= ret;

if (e->len == 0 && e->users == 0) {
list_del(&e->list);
free_entry(e);
continue;
}
}
spin_unlock(&con->writequeue_lock);
out:
mutex_unlock(&con->sock_mutex);
return;

send_error:
mutex_unlock(&con->sock_mutex);
close_connection(con, false);
lowcomms_connect_sock(con);
return;

out_connect:
mutex_unlock(&con->sock_mutex);
if (!test_bit(CF_INIT_PENDING, &con->flags))
lowcomms_connect_sock(con);
return;
}

static void clean_one_writequeue(struct connection *con)
{
struct writequeue_entry *e, *safe;

spin_lock(&con->writequeue_lock);
list_for_each_entry_safe(e, safe, &con->writequeue, list) {
list_del(&e->list);
free_entry(e);
}
spin_unlock(&con->writequeue_lock);
}

/* Called from recovery when it knows that a node has
left the cluster */
int dlm_lowcomms_close(int nodeid)
{
struct connection *con;

log_print(»closing connection to node %d», nodeid);
con = nodeid2con(nodeid, 0);
if (con) {
clear_bit(CF_CONNECT_PENDING, &con->flags);
clear_bit(CF_WRITE_PENDING, &con->flags);
set_bit(CF_CLOSE, &con->flags);
if (cancel_work_sync(&con->swork))
log_print(»canceled swork for node %d», nodeid);
if (cancel_work_sync(&con->rwork))
log_print(»canceled rwork for node %d», nodeid);
clean_one_writequeue(con);
close_connection(con, true);
}
return 0;
}

/* Receive workqueue function */
static void process_recv_sockets(struct work_struct *work)
{
struct connection *con = container_of(work, struct connection, rwork);
int err;

clear_bit(CF_READ_PENDING, &con->flags);
do {
err = con->rx_action(con);
} while (!err);
}

/* Send workqueue function */
static void process_send_sockets(struct work_struct *work)
{
struct connection *con = container_of(work, struct connection, swork);

if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) {
con->connect_action(con);
set_bit(CF_WRITE_PENDING, &con->flags);
}
if (test_and_clear_bit(CF_WRITE_PENDING, &con->flags))
send_to_sock(con);
}

/* Discard all entries on the write queues */
static void clean_writequeues(void)
{
foreach_conn(clean_one_writequeue);
}

static void work_stop(void)
{
destroy_workqueue(recv_workqueue);
destroy_workqueue(send_workqueue);
}

static int work_start(void)
{
int error;
recv_workqueue = create_workqueue(»dlm_recv»);
error = IS_ERR(recv_workqueue);
if (error) {
log_print(»can’t start dlm_recv %d», error);
return error;
}

send_workqueue = create_singlethread_workqueue(»dlm_send»);
error = IS_ERR(send_workqueue);
if (error) {
log_print(»can’t start dlm_send %d», error);
destroy_workqueue(recv_workqueue);
return error;
}

return 0;
}

static void stop_conn(struct connection *con)
{
con->flags |= 0×0F;
if (con->sock && con->sock->sk)
con->sock->sk->sk_user_data = NULL;
}

static void free_conn(struct connection *con)
{
close_connection(con, true);
if (con->othercon)
kmem_cache_free(con_cache, con->othercon);
hlist_del(&con->list);
kmem_cache_free(con_cache, con);
}

void dlm_lowcomms_stop(void)
{
/* Set all the flags to prevent any
socket activity.
*/
mutex_lock(&connections_lock);
foreach_conn(stop_conn);
mutex_unlock(&connections_lock);

work_stop();

mutex_lock(&connections_lock);
clean_writequeues();

foreach_conn(free_conn);

mutex_unlock(&connections_lock);
kmem_cache_destroy(con_cache);
}

int dlm_lowcomms_start(void)
{
int error = -EINVAL;
struct connection *con;
int i;

for (i = 0; i < CONN_HASH_SIZE; i++)
INIT_HLIST_HEAD(&connection_hash[i]);

init_local();
if (!dlm_local_count) {
error = -ENOTCONN;
log_print(»no local IP address has been set»);
goto out;
}

error = -ENOMEM;
con_cache = kmem_cache_create(»dlm_conn», sizeof(struct connection),
__alignof__(struct connection), 0,
NULL);
if (!con_cache)
goto out;

/* Start listening */
if (dlm_config.ci_protocol == 0)
error = tcp_listen_for_all();
else
error = sctp_listen_for_all();
if (error)
goto fail_unlisten;

error = work_start();
if (error)
goto fail_unlisten;

return 0;

fail_unlisten:
con = nodeid2con(0,0);
if (con) {
close_connection(con, false);
kmem_cache_free(con_cache, con);
}
kmem_cache_destroy(con_cache);

out:
return error;
}

#ifndef __LOCKSPACE_DOT_H__
#define __LOCKSPACE_DOT_H__

int dlm_lockspace_init(void);
void dlm_lockspace_exit(void);
struct dlm_ls *dlm_find_lockspace_global(uint32_t id);
struct dlm_ls *dlm_find_lockspace_local(void *id);
struct dlm_ls *dlm_find_lockspace_device(int minor);
void dlm_put_lockspace(struct dlm_ls *ls);
void dlm_stop_lockspaces(void);

#endif /* __LOCKSPACE_DOT_H__ */

#include «dlm_internal.h»
#include «lockspace.h»
#include «member.h»
#include «recoverd.h»
#include «ast.h»
#include «dir.h»
#include «lowcomms.h»
#include «config.h»
#include «memory.h»
#include «lock.h»
#include «recover.h»
#include «requestqueue.h»
#include «user.h»

static int ls_count;
static struct mutex ls_lock;
static struct list_head lslist;
static spinlock_t lslist_lock;
static struct task_struct * scand_task;

static ssize_t dlm_control_store(struct dlm_ls *ls, const char *buf, size_t len)
{
ssize_t ret = len;
int n = simple_strtol(buf, NULL, 0);

ls = dlm_find_lockspace_local(ls->ls_local_handle);
if (!ls)
return -EINVAL;

switch (n) {
case 0:
dlm_ls_stop(ls);
break;
case 1:
dlm_ls_start(ls);
break;
default:
ret = -EINVAL;
}
dlm_put_lockspace(ls);
return ret;
}

static ssize_t dlm_event_store(struct dlm_ls *ls, const char *buf, size_t len)
{
ls->ls_uevent_result = simple_strtol(buf, NULL, 0);
set_bit(LSFL_UEVENT_WAIT, &ls->ls_flags);
wake_up(&ls->ls_uevent_wait);
return len;
}

static ssize_t dlm_id_show(struct dlm_ls *ls, char *buf)
{
return snprintf(buf, PAGE_SIZE, «%u\n», ls->ls_global_id);
}

static ssize_t dlm_id_store(struct dlm_ls *ls, const char *buf, size_t len)
{
ls->ls_global_id = simple_strtoul(buf, NULL, 0);
return len;
}

static ssize_t dlm_recover_status_show(struct dlm_ls *ls, char *buf)
{
uint32_t status = dlm_recover_status(ls);
return snprintf(buf, PAGE_SIZE, «%x\n», status);
}

static ssize_t dlm_recover_nodeid_show(struct dlm_ls *ls, char *buf)
{
return snprintf(buf, PAGE_SIZE, «%d\n», ls->ls_recover_nodeid);
}

struct dlm_attr {
struct attribute attr;
ssize_t (*show)(struct dlm_ls *, char *);
ssize_t (*store)(struct dlm_ls *, const char *, size_t);
};

static struct dlm_attr dlm_attr_control = {
.attr = {.name = «control», .mode = S_IWUSR},
.store = dlm_control_store
};

static struct dlm_attr dlm_attr_event = {
.attr = {.name = «event_done», .mode = S_IWUSR},
.store = dlm_event_store
};

static struct dlm_attr dlm_attr_id = {
.attr = {.name = «id», .mode = S_IRUGO | S_IWUSR},
.show = dlm_id_show,
.store = dlm_id_store
};

static struct dlm_attr dlm_attr_recover_status = {
.attr = {.name = «recover_status», .mode = S_IRUGO},
.show = dlm_recover_status_show
};

static struct dlm_attr dlm_attr_recover_nodeid = {
.attr = {.name = «recover_nodeid», .mode = S_IRUGO},
.show = dlm_recover_nodeid_show
};

static struct attribute *dlm_attrs[] = {
&dlm_attr_control.attr,
&dlm_attr_event.attr,
&dlm_attr_id.attr,
&dlm_attr_recover_status.attr,
&dlm_attr_recover_nodeid.attr,
NULL,
};

static ssize_t dlm_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj);
struct dlm_attr *a = container_of(attr, struct dlm_attr, attr);
return a->show ? a->show(ls, buf) : 0;
}

static ssize_t dlm_attr_store(struct kobject *kobj, struct attribute *attr,
const char *buf, size_t len)
{
struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj);
struct dlm_attr *a = container_of(attr, struct dlm_attr, attr);
return a->store ? a->store(ls, buf, len) : len;
}

static void lockspace_kobj_release(struct kobject *k)
{
struct dlm_ls *ls = container_of(k, struct dlm_ls, ls_kobj);
kfree(ls);
}

static struct sysfs_ops dlm_attr_ops = {
.show = dlm_attr_show,
.store = dlm_attr_store,
};

static struct kobj_type dlm_ktype = {
.default_attrs = dlm_attrs,
.sysfs_ops = &dlm_attr_ops,
.release = lockspace_kobj_release,
};

static struct kset *dlm_kset;

static int do_uevent(struct dlm_ls *ls, int in)
{
int error;

if (in)
kobject_uevent(&ls->ls_kobj, KOBJ_ONLINE);
else
kobject_uevent(&ls->ls_kobj, KOBJ_OFFLINE);

log_debug(ls, «%s the lockspace group…», in ? «joining» : «leaving»);

/* dlm_controld will see the uevent, do the necessary group management
and then write to sysfs to wake us */

error = wait_event_interruptible(ls->ls_uevent_wait,
test_and_clear_bit(LSFL_UEVENT_WAIT, &ls->ls_flags));

log_debug(ls, «group event done %d %d», error, ls->ls_uevent_result);

if (error)
goto out;

error = ls->ls_uevent_result;
out:
if (error)
log_error(ls, «group %s failed %d %d», in ? «join» : «leave»,
error, ls->ls_uevent_result);
return error;
}

int __init dlm_lockspace_init(void)
{
ls_count = 0;
mutex_init(&ls_lock);
INIT_LIST_HEAD(&lslist);
spin_lock_init(&lslist_lock);

dlm_kset = kset_create_and_add(»dlm», NULL, kernel_kobj);
if (!dlm_kset) {
printk(KERN_WARNING «%s: can not create kset\n», __func__);
return -ENOMEM;
}
return 0;
}

void dlm_lockspace_exit(void)
{
kset_unregister(dlm_kset);
}

static struct dlm_ls *find_ls_to_scan(void)
{
struct dlm_ls *ls;

spin_lock(&lslist_lock);
list_for_each_entry(ls, &lslist, ls_list) {
if (time_after_eq(jiffies, ls->ls_scan_time +
dlm_config.ci_scan_secs * HZ)) {
spin_unlock(&lslist_lock);
return ls;
}
}
spin_unlock(&lslist_lock);
return NULL;
}

static int dlm_scand(void *data)
{
struct dlm_ls *ls;
int timeout_jiffies = dlm_config.ci_scan_secs * HZ;

while (!kthread_should_stop()) {
ls = find_ls_to_scan();
if (ls) {
if (dlm_lock_recovery_try(ls)) {
ls->ls_scan_time = jiffies;
dlm_scan_rsbs(ls);
dlm_scan_timeout(ls);
dlm_unlock_recovery(ls);
} else {
ls->ls_scan_time += HZ;
}
} else {
schedule_timeout_interruptible(timeout_jiffies);
}
}
return 0;
}

static int dlm_scand_start(void)
{
struct task_struct *p;
int error = 0;

p = kthread_run(dlm_scand, NULL, «dlm_scand»);
if (IS_ERR(p))
error = PTR_ERR(p);
else
scand_task = p;
return error;
}

static void dlm_scand_stop(void)
{
kthread_stop(scand_task);
}

struct dlm_ls *dlm_find_lockspace_global(uint32_t id)
{
struct dlm_ls *ls;

spin_lock(&lslist_lock);

list_for_each_entry(ls, &lslist, ls_list) {
if (ls->ls_global_id == id) {
ls->ls_count++;
goto out;
}
}
ls = NULL;
out:
spin_unlock(&lslist_lock);
return ls;
}

struct dlm_ls *dlm_find_lockspace_local(dlm_lockspace_t *lockspace)
{
struct dlm_ls *ls;

spin_lock(&lslist_lock);
list_for_each_entry(ls, &lslist, ls_list) {
if (ls->ls_local_handle == lockspace) {
ls->ls_count++;
goto out;
}
}
ls = NULL;
out:
spin_unlock(&lslist_lock);
return ls;
}

struct dlm_ls *dlm_find_lockspace_device(int minor)
{
struct dlm_ls *ls;

spin_lock(&lslist_lock);
list_for_each_entry(ls, &lslist, ls_list) {
if (ls->ls_device.minor == minor) {
ls->ls_count++;
goto out;
}
}
ls = NULL;
out:
spin_unlock(&lslist_lock);
return ls;
}

void dlm_put_lockspace(struct dlm_ls *ls)
{
spin_lock(&lslist_lock);
ls->ls_count–;
spin_unlock(&lslist_lock);
}

static void remove_lockspace(struct dlm_ls *ls)
{
for (;;) {
spin_lock(&lslist_lock);
if (ls->ls_count == 0) {
WARN_ON(ls->ls_create_count != 0);
list_del(&ls->ls_list);
spin_unlock(&lslist_lock);
return;
}
spin_unlock(&lslist_lock);
ssleep(1);
}
}

static int threads_start(void)
{
int error;

/* Thread which process lock requests for all lockspace’s */
error = dlm_astd_start();
if (error) {
log_print(»cannot start dlm_astd thread %d», error);
goto fail;
}

error = dlm_scand_start();
if (error) {
log_print(»cannot start dlm_scand thread %d», error);
goto astd_fail;
}

/* Thread for sending/receiving messages for all lockspace’s */
error = dlm_lowcomms_start();
if (error) {
log_print(»cannot start dlm lowcomms %d», error);
goto scand_fail;
}

return 0;

scand_fail:
dlm_scand_stop();
astd_fail:
dlm_astd_stop();
fail:
return error;
}

static void threads_stop(void)
{
dlm_scand_stop();
dlm_lowcomms_stop();
dlm_astd_stop();
}

static int new_lockspace(const char *name, int namelen, void **lockspace,
uint32_t flags, int lvblen)
{
struct dlm_ls *ls;
int i, size, error;
int do_unreg = 0;

if (namelen > DLM_LOCKSPACE_LEN)
return -EINVAL;

if (!lvblen || (lvblen % 8))
return -EINVAL;

if (!try_module_get(THIS_MODULE))
return -EINVAL;

if (!dlm_user_daemon_available()) {
module_put(THIS_MODULE);
return -EUNATCH;
}

error = 0;

spin_lock(&lslist_lock);
list_for_each_entry(ls, &lslist, ls_list) {
WARN_ON(ls->ls_create_count <= 0);
if (ls->ls_namelen != namelen)
continue;
if (memcmp(ls->ls_name, name, namelen))
continue;
if (flags & DLM_LSFL_NEWEXCL) {
error = -EEXIST;
break;
}
ls->ls_create_count++;
*lockspace = ls;
error = 1;
break;
}
spin_unlock(&lslist_lock);

if (error)
goto out;

error = -ENOMEM;

ls = kzalloc(sizeof(struct dlm_ls) + namelen, GFP_KERNEL);
if (!ls)
goto out;
memcpy(ls->ls_name, name, namelen);
ls->ls_namelen = namelen;
ls->ls_lvblen = lvblen;
ls->ls_count = 0;
ls->ls_flags = 0;
ls->ls_scan_time = jiffies;

if (flags & DLM_LSFL_TIMEWARN)
set_bit(LSFL_TIMEWARN, &ls->ls_flags);

if (flags & DLM_LSFL_FS)
ls->ls_allocation = GFP_NOFS;
else
ls->ls_allocation = GFP_KERNEL;

/* ls_exflags are forced to match among nodes, and we don’t
need to require all nodes to have some flags set */
ls->ls_exflags = (flags & ~(DLM_LSFL_TIMEWARN | DLM_LSFL_FS |
DLM_LSFL_NEWEXCL));

size = dlm_config.ci_rsbtbl_size;
ls->ls_rsbtbl_size = size;

ls->ls_rsbtbl = kmalloc(sizeof(struct dlm_rsbtable) * size, GFP_KERNEL);
if (!ls->ls_rsbtbl)
goto out_lsfree;
for (i = 0; i < size; i++) {
INIT_LIST_HEAD(&ls->ls_rsbtbl[i].list);
INIT_LIST_HEAD(&ls->ls_rsbtbl[i].toss);
spin_lock_init(&ls->ls_rsbtbl[i].lock);
}

size = dlm_config.ci_lkbtbl_size;
ls->ls_lkbtbl_size = size;

ls->ls_lkbtbl = kmalloc(sizeof(struct dlm_lkbtable) * size, GFP_KERNEL);
if (!ls->ls_lkbtbl)
goto out_rsbfree;
for (i = 0; i < size; i++) {
INIT_LIST_HEAD(&ls->ls_lkbtbl[i].list);
rwlock_init(&ls->ls_lkbtbl[i].lock);
ls->ls_lkbtbl[i].counter = 1;
}

size = dlm_config.ci_dirtbl_size;
ls->ls_dirtbl_size = size;

ls->ls_dirtbl = kmalloc(sizeof(struct dlm_dirtable) * size, GFP_KERNEL);
if (!ls->ls_dirtbl)
goto out_lkbfree;
for (i = 0; i < size; i++) {
INIT_LIST_HEAD(&ls->ls_dirtbl[i].list);
spin_lock_init(&ls->ls_dirtbl[i].lock);
}

INIT_LIST_HEAD(&ls->ls_waiters);
mutex_init(&ls->ls_waiters_mutex);
INIT_LIST_HEAD(&ls->ls_orphans);
mutex_init(&ls->ls_orphans_mutex);
INIT_LIST_HEAD(&ls->ls_timeout);
mutex_init(&ls->ls_timeout_mutex);

INIT_LIST_HEAD(&ls->ls_nodes);
INIT_LIST_HEAD(&ls->ls_nodes_gone);
ls->ls_num_nodes = 0;
ls->ls_low_nodeid = 0;
ls->ls_total_weight = 0;
ls->ls_node_array = NULL;

memset(&ls->ls_stub_rsb, 0, sizeof(struct dlm_rsb));
ls->ls_stub_rsb.res_ls = ls;

ls->ls_debug_rsb_dentry = NULL;
ls->ls_debug_waiters_dentry = NULL;

init_waitqueue_head(&ls->ls_uevent_wait);
ls->ls_uevent_result = 0;
init_completion(&ls->ls_members_done);
ls->ls_members_result = -1;

ls->ls_recoverd_task = NULL;
mutex_init(&ls->ls_recoverd_active);
spin_lock_init(&ls->ls_recover_lock);
spin_lock_init(&ls->ls_rcom_spin);
get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t));
ls->ls_recover_status = 0;
ls->ls_recover_seq = 0;
ls->ls_recover_args = NULL;
init_rwsem(&ls->ls_in_recovery);
init_rwsem(&ls->ls_recv_active);
INIT_LIST_HEAD(&ls->ls_requestqueue);
mutex_init(&ls->ls_requestqueue_mutex);
mutex_init(&ls->ls_clear_proc_locks);

ls->ls_recover_buf = kmalloc(dlm_config.ci_buffer_size, GFP_KERNEL);
if (!ls->ls_recover_buf)
goto out_dirfree;

INIT_LIST_HEAD(&ls->ls_recover_list);
spin_lock_init(&ls->ls_recover_list_lock);
ls->ls_recover_list_count = 0;
ls->ls_local_handle = ls;
init_waitqueue_head(&ls->ls_wait_general);
INIT_LIST_HEAD(&ls->ls_root_list);
init_rwsem(&ls->ls_root_sem);

down_write(&ls->ls_in_recovery);

spin_lock(&lslist_lock);
ls->ls_create_count = 1;
list_add(&ls->ls_list, &lslist);
spin_unlock(&lslist_lock);

/* needs to find ls in lslist */
error = dlm_recoverd_start(ls);
if (error) {
log_error(ls, «can’t start dlm_recoverd %d», error);
goto out_delist;
}

ls->ls_kobj.kset = dlm_kset;
error = kobject_init_and_add(&ls->ls_kobj, &dlm_ktype, NULL,
«%s», ls->ls_name);
if (error)
goto out_stop;
kobject_uevent(&ls->ls_kobj, KOBJ_ADD);

/* let kobject handle freeing of ls if there’s an error */
do_unreg = 1;

/* This uevent triggers dlm_controld in userspace to add us to the
group of nodes that are members of this lockspace (managed by the
cluster infrastructure.) Once it’s done that, it tells us who the
current lockspace members are (via configfs) and then tells the
lockspace to start running (via sysfs) in dlm_ls_start(). */

error = do_uevent(ls, 1);
if (error)
goto out_stop;

wait_for_completion(&ls->ls_members_done);
error = ls->ls_members_result;
if (error)
goto out_members;

dlm_create_debug_file(ls);

log_debug(ls, «join complete»);
*lockspace = ls;
return 0;

out_members:
do_uevent(ls, 0);
dlm_clear_members(ls);
kfree(ls->ls_node_array);
out_stop:
dlm_recoverd_stop(ls);
out_delist:
spin_lock(&lslist_lock);
list_del(&ls->ls_list);
spin_unlock(&lslist_lock);
kfree(ls->ls_recover_buf);
out_dirfree:
kfree(ls->ls_dirtbl);
out_lkbfree:
kfree(ls->ls_lkbtbl);
out_rsbfree:
kfree(ls->ls_rsbtbl);
out_lsfree:
if (do_unreg)
kobject_put(&ls->ls_kobj);
else
kfree(ls);
out:
module_put(THIS_MODULE);
return error;
}

int dlm_new_lockspace(const char *name, int namelen, void **lockspace,
uint32_t flags, int lvblen)
{
int error = 0;

mutex_lock(&ls_lock);
if (!ls_count)
error = threads_start();
if (error)
goto out;

error = new_lockspace(name, namelen, lockspace, flags, lvblen);
if (!error)
ls_count++;
if (error > 0)
error = 0;
if (!ls_count)
threads_stop();
out:
mutex_unlock(&ls_lock);
return error;
}

/* Return 1 if the lockspace still has active remote locks,
* 2 if the lockspace still has active local locks.
*/
static int lockspace_busy(struct dlm_ls *ls)
{
int i, lkb_found = 0;
struct dlm_lkb *lkb;

/* NOTE: We check the lockidtbl here rather than the resource table.
This is because there may be LKBs queued as ASTs that have been
unlinked from their RSBs and are pending deletion once the AST has
been delivered */

for (i = 0; i < ls->ls_lkbtbl_size; i++) {
read_lock(&ls->ls_lkbtbl[i].lock);
if (!list_empty(&ls->ls_lkbtbl[i].list)) {
lkb_found = 1;
list_for_each_entry(lkb, &ls->ls_lkbtbl[i].list,
lkb_idtbl_list) {
if (!lkb->lkb_nodeid) {
read_unlock(&ls->ls_lkbtbl[i].lock);
return 2;
}
}
}
read_unlock(&ls->ls_lkbtbl[i].lock);
}
return lkb_found;
}

static int release_lockspace(struct dlm_ls *ls, int force)
{
struct dlm_lkb *lkb;
struct dlm_rsb *rsb;
struct list_head *head;
int i, busy, rv;

busy = lockspace_busy(ls);

spin_lock(&lslist_lock);
if (ls->ls_create_count == 1) {
if (busy > force)
rv = -EBUSY;
else {
/* remove_lockspace takes ls off lslist */
ls->ls_create_count = 0;
rv = 0;
}
} else if (ls->ls_create_count > 1) {
rv = –ls->ls_create_count;
} else {
rv = -EINVAL;
}
spin_unlock(&lslist_lock);

if (rv) {
log_debug(ls, «release_lockspace no remove %d», rv);
return rv;
}

dlm_device_deregister(ls);

if (force < 3 && dlm_user_daemon_available())
do_uevent(ls, 0);

dlm_recoverd_stop(ls);

remove_lockspace(ls);

dlm_delete_debug_file(ls);

dlm_astd_suspend();

kfree(ls->ls_recover_buf);

/*
* Free direntry structs.
*/

dlm_dir_clear(ls);
kfree(ls->ls_dirtbl);

/*
* Free all lkb’s on lkbtbl[] lists.
*/

for (i = 0; i < ls->ls_lkbtbl_size; i++) {
head = &ls->ls_lkbtbl[i].list;
while (!list_empty(head)) {
lkb = list_entry(head->next, struct dlm_lkb,
lkb_idtbl_list);

list_del(&lkb->lkb_idtbl_list);

dlm_del_ast(lkb);

if (lkb->lkb_lvbptr && lkb->lkb_flags & DLM_IFL_MSTCPY)
dlm_free_lvb(lkb->lkb_lvbptr);

dlm_free_lkb(lkb);
}
}
dlm_astd_resume();

kfree(ls->ls_lkbtbl);

/*
* Free all rsb’s on rsbtbl[] lists
*/

for (i = 0; i < ls->ls_rsbtbl_size; i++) {
head = &ls->ls_rsbtbl[i].list;
while (!list_empty(head)) {
rsb = list_entry(head->next, struct dlm_rsb,
res_hashchain);

list_del(&rsb->res_hashchain);
dlm_free_rsb(rsb);
}

head = &ls->ls_rsbtbl[i].toss;
while (!list_empty(head)) {
rsb = list_entry(head->next, struct dlm_rsb,
res_hashchain);
list_del(&rsb->res_hashchain);
dlm_free_rsb(rsb);
}
}

kfree(ls->ls_rsbtbl);

/*
* Free structures on any other lists
*/

dlm_purge_requestqueue(ls);
kfree(ls->ls_recover_args);
dlm_clear_free_entries(ls);
dlm_clear_members(ls);
dlm_clear_members_gone(ls);
kfree(ls->ls_node_array);
log_debug(ls, «release_lockspace final free»);
kobject_put(&ls->ls_kobj);
/* The ls structure will be freed when the kobject is done with */

module_put(THIS_MODULE);
return 0;
}

/*
* Called when a system has released all its locks and is not going to use the
* lockspace any longer. We free everything we’re managing for this lockspace.
* Remaining nodes will go through the recovery process as if we’d died. The
* lockspace must continue to function as usual, participating in recoveries,
* until this returns.
*
* Force has 4 possible values:
* 0 – don’t destroy locksapce if it has any LKBs
* 1 – destroy lockspace if it has remote LKBs but not if it has local LKBs
* 2 – destroy lockspace regardless of LKBs
* 3 – destroy lockspace as part of a forced shutdown
*/

int dlm_release_lockspace(void *lockspace, int force)
{
struct dlm_ls *ls;
int error;

ls = dlm_find_lockspace_local(lockspace);
if (!ls)
return -EINVAL;
dlm_put_lockspace(ls);

mutex_lock(&ls_lock);
error = release_lockspace(ls, force);
if (!error)
ls_count–;
if (!ls_count)
threads_stop();
mutex_unlock(&ls_lock);

return error;
}

void dlm_stop_lockspaces(void)
{
struct dlm_ls *ls;

restart:
spin_lock(&lslist_lock);
list_for_each_entry(ls, &lslist, ls_list) {
if (!test_bit(LSFL_RUNNING, &ls->ls_flags))
continue;
spin_unlock(&lslist_lock);
log_error(ls, «no userland control daemon, stopping lockspace»);
dlm_ls_stop(ls);
goto restart;
}
spin_unlock(&lslist_lock);
}

#ifndef __LOCK_DOT_H__
#define __LOCK_DOT_H__

void dlm_dump_rsb(struct dlm_rsb *r);
void dlm_print_lkb(struct dlm_lkb *lkb);
void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms);
void dlm_receive_buffer(union dlm_packet *p, int nodeid);
int dlm_modes_compat(int mode1, int mode2);
void dlm_put_rsb(struct dlm_rsb *r);
void dlm_hold_rsb(struct dlm_rsb *r);
int dlm_put_lkb(struct dlm_lkb *lkb);
void dlm_scan_rsbs(struct dlm_ls *ls);
int dlm_lock_recovery_try(struct dlm_ls *ls);
void dlm_unlock_recovery(struct dlm_ls *ls);
void dlm_scan_timeout(struct dlm_ls *ls);
void dlm_adjust_timeouts(struct dlm_ls *ls);

int dlm_purge_locks(struct dlm_ls *ls);
void dlm_purge_mstcpy_locks(struct dlm_rsb *r);
void dlm_grant_after_purge(struct dlm_ls *ls);
int dlm_recover_waiters_post(struct dlm_ls *ls);
void dlm_recover_waiters_pre(struct dlm_ls *ls);
int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc);
int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc);

int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, int mode,
uint32_t flags, void *name, unsigned int namelen,
unsigned long timeout_cs);
int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
int mode, uint32_t flags, uint32_t lkid, char *lvb_in,
unsigned long timeout_cs);
int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
uint32_t flags, uint32_t lkid, char *lvb_in);
int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
uint32_t flags, uint32_t lkid);
int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc,
int nodeid, int pid);
int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid);
void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc);

static inline int is_master(struct dlm_rsb *r)
{
return !r->res_nodeid;
}

static inline void lock_rsb(struct dlm_rsb *r)
{
mutex_lock(&r->res_mutex);
}

static inline void unlock_rsb(struct dlm_rsb *r)
{
mutex_unlock(&r->res_mutex);
}

#endif

/* Central locking logic has four stages:

dlm_lock()
dlm_unlock()

request_lock(ls, lkb)
convert_lock(ls, lkb)
unlock_lock(ls, lkb)
cancel_lock(ls, lkb)

_request_lock(r, lkb)
_convert_lock(r, lkb)
_unlock_lock(r, lkb)
_cancel_lock(r, lkb)

do_request(r, lkb)
do_convert(r, lkb)
do_unlock(r, lkb)
do_cancel(r, lkb)

Stage 1 (lock, unlock) is mainly about checking input args and
splitting into one of the four main operations:

dlm_lock = request_lock
dlm_lock+CONVERT = convert_lock
dlm_unlock = unlock_lock
dlm_unlock+CANCEL = cancel_lock

Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is
provided to the next stage.

Stage 3, _xxxx_lock(), determines if the operation is local or remote.
When remote, it calls send_xxxx(), when local it calls do_xxxx().

Stage 4, do_xxxx(), is the guts of the operation. It manipulates the
given rsb and lkb and queues callbacks.

For remote operations, send_xxxx() results in the corresponding do_xxxx()
function being executed on the remote node. The connecting send/receive
calls on local (L) and remote (R) nodes:

L: send_xxxx() -> R: receive_xxxx()
R: do_xxxx()
L: receive_xxxx_reply() <- R: send_xxxx_reply()
*/
#include #include «dlm_internal.h»
#include #include «memory.h»
#include «lowcomms.h»
#include «requestqueue.h»
#include «util.h»
#include «dir.h»
#include «member.h»
#include «lockspace.h»
#include «ast.h»
#include «lock.h»
#include «rcom.h»
#include «recover.h»
#include «lvb_table.h»
#include «user.h»
#include «config.h»

static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode);
static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int send_remove(struct dlm_rsb *r);
static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb);
static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb);
static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
struct dlm_message *ms);
static int receive_extralen(struct dlm_message *ms);
static void do_purge(struct dlm_ls *ls, int nodeid, int pid);
static void del_timeout(struct dlm_lkb *lkb);

/*
* Lock compatibilty matrix – thanks Steve
* UN = Unlocked state. Not really a state, used as a flag
* PD = Padding. Used to make the matrix a nice power of two in size
* Other states are the same as the VMS DLM.
* Usage: matrix[grmode+1][rqmode+1] (although m[rq+1][gr+1] is the same)
*/

static const int __dlm_compat_matrix[8][8] = {
/* UN NL CR CW PR PW EX PD */
{1, 1, 1, 1, 1, 1, 1, 0}, /* UN */
{1, 1, 1, 1, 1, 1, 1, 0}, /* NL */
{1, 1, 1, 1, 1, 1, 0, 0}, /* CR */
{1, 1, 1, 1, 0, 0, 0, 0}, /* CW */
{1, 1, 1, 0, 1, 0, 0, 0}, /* PR */
{1, 1, 1, 0, 0, 0, 0, 0}, /* PW */
{1, 1, 0, 0, 0, 0, 0, 0}, /* EX */
{0, 0, 0, 0, 0, 0, 0, 0} /* PD */
};

/*
* This defines the direction of transfer of LVB data.
* Granted mode is the row; requested mode is the column.
* Usage: matrix[grmode+1][rqmode+1]
* 1 = LVB is returned to the caller
* 0 = LVB is written to the resource
* -1 = nothing happens to the LVB
*/

const int dlm_lvb_operations[8][8] = {
/* UN NL CR CW PR PW EX PD*/
{ -1, 1, 1, 1, 1, 1, 1, -1 }, /* UN */
{ -1, 1, 1, 1, 1, 1, 1, 0 }, /* NL */
{ -1, -1, 1, 1, 1, 1, 1, 0 }, /* CR */
{ -1, -1, -1, 1, 1, 1, 1, 0 }, /* CW */
{ -1, -1, -1, -1, 1, 1, 1, 0 }, /* PR */
{ -1, 0, 0, 0, 0, 0, 1, 0 }, /* PW */
{ -1, 0, 0, 0, 0, 0, 0, 0 }, /* EX */
{ -1, 0, 0, 0, 0, 0, 0, 0 } /* PD */
};

#define modes_compat(gr, rq) \
__dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1]

int dlm_modes_compat(int mode1, int mode2)
{
return __dlm_compat_matrix[mode1 + 1][mode2 + 1];
}

/*
* Compatibility matrix for conversions with QUECVT set.
* Granted mode is the row; requested mode is the column.
* Usage: matrix[grmode+1][rqmode+1]
*/

static const int __quecvt_compat_matrix[8][8] = {
/* UN NL CR CW PR PW EX PD */
{0, 0, 0, 0, 0, 0, 0, 0}, /* UN */
{0, 0, 1, 1, 1, 1, 1, 0}, /* NL */
{0, 0, 0, 1, 1, 1, 1, 0}, /* CR */
{0, 0, 0, 0, 1, 1, 1, 0}, /* CW */
{0, 0, 0, 1, 0, 1, 1, 0}, /* PR */
{0, 0, 0, 0, 0, 0, 1, 0}, /* PW */
{0, 0, 0, 0, 0, 0, 0, 0}, /* EX */
{0, 0, 0, 0, 0, 0, 0, 0} /* PD */
};

void dlm_print_lkb(struct dlm_lkb *lkb)
{
printk(KERN_ERR «lkb: nodeid %d id %x remid %x exflags %x flags %x\n»
» status %d rqmode %d grmode %d wait_type %d ast_type %d\n»,
lkb->lkb_nodeid, lkb->lkb_id, lkb->lkb_remid, lkb->lkb_exflags,
lkb->lkb_flags, lkb->lkb_status, lkb->lkb_rqmode,
lkb->lkb_grmode, lkb->lkb_wait_type, lkb->lkb_ast_type);
}

static void dlm_print_rsb(struct dlm_rsb *r)
{
printk(KERN_ERR «rsb: nodeid %d flags %lx first %x rlc %d name %s\n»,
r->res_nodeid, r->res_flags, r->res_first_lkid,
r->res_recover_locks_count, r->res_name);
}

void dlm_dump_rsb(struct dlm_rsb *r)
{
struct dlm_lkb *lkb;

dlm_print_rsb(r);

printk(KERN_ERR «rsb: root_list empty %d recover_list empty %d\n»,
list_empty(&r->res_root_list), list_empty(&r->res_recover_list));
printk(KERN_ERR «rsb lookup list\n»);
list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup)
dlm_print_lkb(lkb);
printk(KERN_ERR «rsb grant queue:\n»);
list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue)
dlm_print_lkb(lkb);
printk(KERN_ERR «rsb convert queue:\n»);
list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue)
dlm_print_lkb(lkb);
printk(KERN_ERR «rsb wait queue:\n»);
list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue)
dlm_print_lkb(lkb);
}

/* Threads cannot use the lockspace while it’s being recovered */

static inline void dlm_lock_recovery(struct dlm_ls *ls)
{
down_read(&ls->ls_in_recovery);
}

void dlm_unlock_recovery(struct dlm_ls *ls)
{
up_read(&ls->ls_in_recovery);
}

int dlm_lock_recovery_try(struct dlm_ls *ls)
{
return down_read_trylock(&ls->ls_in_recovery);
}

static inline int can_be_queued(struct dlm_lkb *lkb)
{
return !(lkb->lkb_exflags & DLM_LKF_NOQUEUE);
}

static inline int force_blocking_asts(struct dlm_lkb *lkb)
{
return (lkb->lkb_exflags & DLM_LKF_NOQUEUEBAST);
}

static inline int is_demoted(struct dlm_lkb *lkb)
{
return (lkb->lkb_sbflags & DLM_SBF_DEMOTED);
}

static inline int is_altmode(struct dlm_lkb *lkb)
{
return (lkb->lkb_sbflags & DLM_SBF_ALTMODE);
}

static inline int is_granted(struct dlm_lkb *lkb)
{
return (lkb->lkb_status == DLM_LKSTS_GRANTED);
}

static inline int is_remote(struct dlm_rsb *r)
{
DLM_ASSERT(r->res_nodeid >= 0, dlm_print_rsb(r););
return !!r->res_nodeid;
}

static inline int is_process_copy(struct dlm_lkb *lkb)
{
return (lkb->lkb_nodeid && !(lkb->lkb_flags & DLM_IFL_MSTCPY));
}

static inline int is_master_copy(struct dlm_lkb *lkb)
{
if (lkb->lkb_flags & DLM_IFL_MSTCPY)
DLM_ASSERT(lkb->lkb_nodeid, dlm_print_lkb(lkb););
return (lkb->lkb_flags & DLM_IFL_MSTCPY) ? 1 : 0;
}

static inline int middle_conversion(struct dlm_lkb *lkb)
{
if ((lkb->lkb_grmode==DLM_LOCK_PR && lkb->lkb_rqmode==DLM_LOCK_CW) ||
(lkb->lkb_rqmode==DLM_LOCK_PR && lkb->lkb_grmode==DLM_LOCK_CW))
return 1;
return 0;
}

static inline int down_conversion(struct dlm_lkb *lkb)
{
return (!middle_conversion(lkb) && lkb->lkb_rqmode < lkb->lkb_grmode);
}

static inline int is_overlap_unlock(struct dlm_lkb *lkb)
{
return lkb->lkb_flags & DLM_IFL_OVERLAP_UNLOCK;
}

static inline int is_overlap_cancel(struct dlm_lkb *lkb)
{
return lkb->lkb_flags & DLM_IFL_OVERLAP_CANCEL;
}

static inline int is_overlap(struct dlm_lkb *lkb)
{
return (lkb->lkb_flags & (DLM_IFL_OVERLAP_UNLOCK |
DLM_IFL_OVERLAP_CANCEL));
}

static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
{
if (is_master_copy(lkb))
return;

del_timeout(lkb);

DLM_ASSERT(lkb->lkb_lksb, dlm_print_lkb(lkb););

/* if the operation was a cancel, then return -DLM_ECANCEL, if a
timeout caused the cancel then return -ETIMEDOUT */
if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_TIMEOUT_CANCEL)) {
lkb->lkb_flags &= ~DLM_IFL_TIMEOUT_CANCEL;
rv = -ETIMEDOUT;
}

if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_DEADLOCK_CANCEL)) {
lkb->lkb_flags &= ~DLM_IFL_DEADLOCK_CANCEL;
rv = -EDEADLK;
}

lkb->lkb_lksb->sb_status = rv;
lkb->lkb_lksb->sb_flags = lkb->lkb_sbflags;

dlm_add_ast(lkb, AST_COMP, 0);
}

static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
queue_cast(r, lkb,
is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL);
}

static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode)
{
lkb->lkb_time_bast = ktime_get();

if (is_master_copy(lkb))
send_bast(r, lkb, rqmode);
else
dlm_add_ast(lkb, AST_BAST, rqmode);
}

/*
* Basic operations on rsb’s and lkb’s
*/

static struct dlm_rsb *create_rsb(struct dlm_ls *ls, char *name, int len)
{
struct dlm_rsb *r;

r = dlm_allocate_rsb(ls, len);
if (!r)
return NULL;

r->res_ls = ls;
r->res_length = len;
memcpy(r->res_name, name, len);
mutex_init(&r->res_mutex);

INIT_LIST_HEAD(&r->res_lookup);
INIT_LIST_HEAD(&r->res_grantqueue);
INIT_LIST_HEAD(&r->res_convertqueue);
INIT_LIST_HEAD(&r->res_waitqueue);
INIT_LIST_HEAD(&r->res_root_list);
INIT_LIST_HEAD(&r->res_recover_list);

return r;
}

static int search_rsb_list(struct list_head *head, char *name, int len,
unsigned int flags, struct dlm_rsb **r_ret)
{
struct dlm_rsb *r;
int error = 0;

list_for_each_entry(r, head, res_hashchain) {
if (len == r->res_length && !memcmp(name, r->res_name, len))
goto found;
}
*r_ret = NULL;
return -EBADR;

found:
if (r->res_nodeid && (flags & R_MASTER))
error = -ENOTBLK;
*r_ret = r;
return error;
}

static int _search_rsb(struct dlm_ls *ls, char *name, int len, int b,
unsigned int flags, struct dlm_rsb **r_ret)
{
struct dlm_rsb *r;
int error;

error = search_rsb_list(&ls->ls_rsbtbl[b].list, name, len, flags, &r);
if (!error) {
kref_get(&r->res_ref);
goto out;
}
error = search_rsb_list(&ls->ls_rsbtbl[b].toss, name, len, flags, &r);
if (error)
goto out;

list_move(&r->res_hashchain, &ls->ls_rsbtbl[b].list);

if (dlm_no_directory(ls))
goto out;

if (r->res_nodeid == -1) {
rsb_clear_flag(r, RSB_MASTER_UNCERTAIN);
r->res_first_lkid = 0;
} else if (r->res_nodeid > 0) {
rsb_set_flag(r, RSB_MASTER_UNCERTAIN);
r->res_first_lkid = 0;
} else {
DLM_ASSERT(r->res_nodeid == 0, dlm_print_rsb(r););
DLM_ASSERT(!rsb_flag(r, RSB_MASTER_UNCERTAIN),);
}
out:
*r_ret = r;
return error;
}

static int search_rsb(struct dlm_ls *ls, char *name, int len, int b,
unsigned int flags, struct dlm_rsb **r_ret)
{
int error;
spin_lock(&ls->ls_rsbtbl[b].lock);
error = _search_rsb(ls, name, len, b, flags, r_ret);
spin_unlock(&ls->ls_rsbtbl[b].lock);
return error;
}

/*
* Find rsb in rsbtbl and potentially create/add one
*
* Delaying the release of rsb’s has a similar benefit to applications keeping
* NL locks on an rsb, but without the guarantee that the cached master value
* will still be valid when the rsb is reused. Apps aren’t always smart enough
* to keep NL locks on an rsb that they may lock again shortly; this can lead
* to excessive master lookups and removals if we don’t delay the release.
*
* Searching for an rsb means looking through both the normal list and toss
* list. When found on the toss list the rsb is moved to the normal list with
* ref count of 1; when found on normal list the ref count is incremented.
*/

static int find_rsb(struct dlm_ls *ls, char *name, int namelen,
unsigned int flags, struct dlm_rsb **r_ret)
{
struct dlm_rsb *r = NULL, *tmp;
uint32_t hash, bucket;
int error = -EINVAL;

if (namelen > DLM_RESNAME_MAXLEN)
goto out;

if (dlm_no_directory(ls))
flags |= R_CREATE;

error = 0;
hash = jhash(name, namelen, 0);
bucket = hash & (ls->ls_rsbtbl_size – 1);

error = search_rsb(ls, name, namelen, bucket, flags, &r);
if (!error)
goto out;

if (error == -EBADR && !(flags & R_CREATE))
goto out;

/* the rsb was found but wasn’t a master copy */
if (error == -ENOTBLK)
goto out;

error = -ENOMEM;
r = create_rsb(ls, name, namelen);
if (!r)
goto out;

r->res_hash = hash;
r->res_bucket = bucket;
r->res_nodeid = -1;
kref_init(&r->res_ref);

/* With no directory, the master can be set immediately */
if (dlm_no_directory(ls)) {
int nodeid = dlm_dir_nodeid(r);
if (nodeid == dlm_our_nodeid())
nodeid = 0;
r->res_nodeid = nodeid;
}

spin_lock(&ls->ls_rsbtbl[bucket].lock);
error = _search_rsb(ls, name, namelen, bucket, 0, &tmp);
if (!error) {
spin_unlock(&ls->ls_rsbtbl[bucket].lock);
dlm_free_rsb(r);
r = tmp;
goto out;
}
list_add(&r->res_hashchain, &ls->ls_rsbtbl[bucket].list);
spin_unlock(&ls->ls_rsbtbl[bucket].lock);
error = 0;
out:
*r_ret = r;
return error;
}

/* This is only called to add a reference when the code already holds
a valid reference to the rsb, so there’s no need for locking. */

static inline void hold_rsb(struct dlm_rsb *r)
{
kref_get(&r->res_ref);
}

void dlm_hold_rsb(struct dlm_rsb *r)
{
hold_rsb(r);
}

static void toss_rsb(struct kref *kref)
{
struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref);
struct dlm_ls *ls = r->res_ls;

DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r););
kref_init(&r->res_ref);
list_move(&r->res_hashchain, &ls->ls_rsbtbl[r->res_bucket].toss);
r->res_toss_time = jiffies;
if (r->res_lvbptr) {
dlm_free_lvb(r->res_lvbptr);
r->res_lvbptr = NULL;
}
}

/* When all references to the rsb are gone it’s transfered to
the tossed list for later disposal. */

static void put_rsb(struct dlm_rsb *r)
{
struct dlm_ls *ls = r->res_ls;
uint32_t bucket = r->res_bucket;

spin_lock(&ls->ls_rsbtbl[bucket].lock);
kref_put(&r->res_ref, toss_rsb);
spin_unlock(&ls->ls_rsbtbl[bucket].lock);
}

void dlm_put_rsb(struct dlm_rsb *r)
{
put_rsb(r);
}

/* See comment for unhold_lkb */

static void unhold_rsb(struct dlm_rsb *r)
{
int rv;
rv = kref_put(&r->res_ref, toss_rsb);
DLM_ASSERT(!rv, dlm_dump_rsb(r););
}

static void kill_rsb(struct kref *kref)
{
struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref);

/* All work is done after the return from kref_put() so we
can release the write_lock before the remove and free. */

DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r););
DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r););
DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r););
DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r););
DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r););
DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r););
}

/* Attaching/detaching lkb’s from rsb’s is for rsb reference counting.
The rsb must exist as long as any lkb’s for it do. */

static void attach_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
hold_rsb(r);
lkb->lkb_resource = r;
}

static void detach_lkb(struct dlm_lkb *lkb)
{
if (lkb->lkb_resource) {
put_rsb(lkb->lkb_resource);
lkb->lkb_resource = NULL;
}
}

static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret)
{
struct dlm_lkb *lkb, *tmp;
uint32_t lkid = 0;
uint16_t bucket;

lkb = dlm_allocate_lkb(ls);
if (!lkb)
return -ENOMEM;

lkb->lkb_nodeid = -1;
lkb->lkb_grmode = DLM_LOCK_IV;
kref_init(&lkb->lkb_ref);
INIT_LIST_HEAD(&lkb->lkb_ownqueue);
INIT_LIST_HEAD(&lkb->lkb_rsb_lookup);
INIT_LIST_HEAD(&lkb->lkb_time_list);

get_random_bytes(&bucket, sizeof(bucket));
bucket &= (ls->ls_lkbtbl_size – 1);

write_lock(&ls->ls_lkbtbl[bucket].lock);

/* counter can roll over so we must verify lkid is not in use */

while (lkid == 0) {
lkid = (bucket << 16) | ls->ls_lkbtbl[bucket].counter++;

list_for_each_entry(tmp, &ls->ls_lkbtbl[bucket].list,
lkb_idtbl_list) {
if (tmp->lkb_id != lkid)
continue;
lkid = 0;
break;
}
}

lkb->lkb_id = lkid;
list_add(&lkb->lkb_idtbl_list, &ls->ls_lkbtbl[bucket].list);
write_unlock(&ls->ls_lkbtbl[bucket].lock);

*lkb_ret = lkb;
return 0;
}

static struct dlm_lkb *__find_lkb(struct dlm_ls *ls, uint32_t lkid)
{
struct dlm_lkb *lkb;
uint16_t bucket = (lkid >> 16);

list_for_each_entry(lkb, &ls->ls_lkbtbl[bucket].list, lkb_idtbl_list) {
if (lkb->lkb_id == lkid)
return lkb;
}
return NULL;
}

static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret)
{
struct dlm_lkb *lkb;
uint16_t bucket = (lkid >> 16);

if (bucket >= ls->ls_lkbtbl_size)
return -EBADSLT;

read_lock(&ls->ls_lkbtbl[bucket].lock);
lkb = __find_lkb(ls, lkid);
if (lkb)
kref_get(&lkb->lkb_ref);
read_unlock(&ls->ls_lkbtbl[bucket].lock);

*lkb_ret = lkb;
return lkb ? 0 : -ENOENT;
}

static void kill_lkb(struct kref *kref)
{
struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref);

/* All work is done after the return from kref_put() so we
can release the write_lock before the detach_lkb */

DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb););
}

/* __put_lkb() is used when an lkb may not have an rsb attached to
it so we need to provide the lockspace explicitly */

static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
uint16_t bucket = (lkb->lkb_id >> 16);

write_lock(&ls->ls_lkbtbl[bucket].lock);
if (kref_put(&lkb->lkb_ref, kill_lkb)) {
list_del(&lkb->lkb_idtbl_list);
write_unlock(&ls->ls_lkbtbl[bucket].lock);

detach_lkb(lkb);

/* for local/process lkbs, lvbptr points to caller’s lksb */
if (lkb->lkb_lvbptr && is_master_copy(lkb))
dlm_free_lvb(lkb->lkb_lvbptr);
dlm_free_lkb(lkb);
return 1;
} else {
write_unlock(&ls->ls_lkbtbl[bucket].lock);
return 0;
}
}

int dlm_put_lkb(struct dlm_lkb *lkb)
{
struct dlm_ls *ls;

DLM_ASSERT(lkb->lkb_resource, dlm_print_lkb(lkb););
DLM_ASSERT(lkb->lkb_resource->res_ls, dlm_print_lkb(lkb););

ls = lkb->lkb_resource->res_ls;
return __put_lkb(ls, lkb);
}

/* This is only called to add a reference when the code already holds
a valid reference to the lkb, so there’s no need for locking. */

static inline void hold_lkb(struct dlm_lkb *lkb)
{
kref_get(&lkb->lkb_ref);
}

/* This is called when we need to remove a reference and are certain
it’s not the last ref. e.g. del_lkb is always called between a
find_lkb/put_lkb and is always the inverse of a previous add_lkb.
put_lkb would work fine, but would involve unnecessary locking */

static inline void unhold_lkb(struct dlm_lkb *lkb)
{
int rv;
rv = kref_put(&lkb->lkb_ref, kill_lkb);
DLM_ASSERT(!rv, dlm_print_lkb(lkb););
}

static void lkb_add_ordered(struct list_head *new, struct list_head *head,
int mode)
{
struct dlm_lkb *lkb = NULL;

list_for_each_entry(lkb, head, lkb_statequeue)
if (lkb->lkb_rqmode < mode)
break;

if (!lkb)
list_add_tail(new, head);
else
__list_add(new, lkb->lkb_statequeue.prev, &lkb->lkb_statequeue);
}

/* add/remove lkb to rsb’s grant/convert/wait queue */

static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status)
{
kref_get(&lkb->lkb_ref);

DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb););

lkb->lkb_timestamp = ktime_get();

lkb->lkb_status = status;

switch (status) {
case DLM_LKSTS_WAITING:
if (lkb->lkb_exflags & DLM_LKF_HEADQUE)
list_add(&lkb->lkb_statequeue, &r->res_waitqueue);
else
list_add_tail(&lkb->lkb_statequeue, &r->res_waitqueue);
break;
case DLM_LKSTS_GRANTED:
/* convention says granted locks kept in order of grmode */
lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue,
lkb->lkb_grmode);
break;
case DLM_LKSTS_CONVERT:
if (lkb->lkb_exflags & DLM_LKF_HEADQUE)
list_add(&lkb->lkb_statequeue, &r->res_convertqueue);
else
list_add_tail(&lkb->lkb_statequeue,
&r->res_convertqueue);
break;
default:
DLM_ASSERT(0, dlm_print_lkb(lkb); printk(»sts=%d\n», status););
}
}

static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
lkb->lkb_status = 0;
list_del(&lkb->lkb_statequeue);
unhold_lkb(lkb);
}

static void move_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int sts)
{
hold_lkb(lkb);
del_lkb(r, lkb);
add_lkb(r, lkb, sts);
unhold_lkb(lkb);
}

static int msg_reply_type(int mstype)
{
switch (mstype) {
case DLM_MSG_REQUEST:
return DLM_MSG_REQUEST_REPLY;
case DLM_MSG_CONVERT:
return DLM_MSG_CONVERT_REPLY;
case DLM_MSG_UNLOCK:
return DLM_MSG_UNLOCK_REPLY;
case DLM_MSG_CANCEL:
return DLM_MSG_CANCEL_REPLY;
case DLM_MSG_LOOKUP:
return DLM_MSG_LOOKUP_REPLY;
}
return -1;
}

/* add/remove lkb from global waiters list of lkb’s waiting for
a reply from a remote node */

static int add_to_waiters(struct dlm_lkb *lkb, int mstype)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;
int error = 0;

mutex_lock(&ls->ls_waiters_mutex);

if (is_overlap_unlock(lkb) ||
(is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL))) {
error = -EINVAL;
goto out;
}

if (lkb->lkb_wait_type || is_overlap_cancel(lkb)) {
switch (mstype) {
case DLM_MSG_UNLOCK:
lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
break;
case DLM_MSG_CANCEL:
lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
break;
default:
error = -EBUSY;
goto out;
}
lkb->lkb_wait_count++;
hold_lkb(lkb);

log_debug(ls, «addwait %x cur %d overlap %d count %d f %x»,
lkb->lkb_id, lkb->lkb_wait_type, mstype,
lkb->lkb_wait_count, lkb->lkb_flags);
goto out;
}

DLM_ASSERT(!lkb->lkb_wait_count,
dlm_print_lkb(lkb);
printk(»wait_count %d\n», lkb->lkb_wait_count););

lkb->lkb_wait_count++;
lkb->lkb_wait_type = mstype;
hold_lkb(lkb);
list_add(&lkb->lkb_wait_reply, &ls->ls_waiters);
out:
if (error)
log_error(ls, «addwait error %x %d flags %x %d %d %s»,
lkb->lkb_id, error, lkb->lkb_flags, mstype,
lkb->lkb_wait_type, lkb->lkb_resource->res_name);
mutex_unlock(&ls->ls_waiters_mutex);
return error;
}

/* We clear the RESEND flag because we might be taking an lkb off the waiters
list as part of process_requestqueue (e.g. a lookup that has an optimized
request reply on the requestqueue) between dlm_recover_waiters_pre() which
set RESEND and dlm_recover_waiters_post() */

static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype,
struct dlm_message *ms)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;
int overlap_done = 0;

if (is_overlap_unlock(lkb) && (mstype == DLM_MSG_UNLOCK_REPLY)) {
log_debug(ls, «remwait %x unlock_reply overlap», lkb->lkb_id);
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
overlap_done = 1;
goto out_del;
}

if (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL_REPLY)) {
log_debug(ls, «remwait %x cancel_reply overlap», lkb->lkb_id);
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
overlap_done = 1;
goto out_del;
}

/* Cancel state was preemptively cleared by a successful convert,
see next comment, nothing to do. */

if ((mstype == DLM_MSG_CANCEL_REPLY) &&
(lkb->lkb_wait_type != DLM_MSG_CANCEL)) {
log_debug(ls, «remwait %x cancel_reply wait_type %d»,
lkb->lkb_id, lkb->lkb_wait_type);
return -1;
}

/* Remove for the convert reply, and premptively remove for the
cancel reply. A convert has been granted while there’s still
an outstanding cancel on it (the cancel is moot and the result
in the cancel reply should be 0). We preempt the cancel reply
because the app gets the convert result and then can follow up
with another op, like convert. This subsequent op would see the
lingering state of the cancel and fail with -EBUSY. */

if ((mstype == DLM_MSG_CONVERT_REPLY) &&
(lkb->lkb_wait_type == DLM_MSG_CONVERT) &&
is_overlap_cancel(lkb) && ms && !ms->m_result) {
log_debug(ls, «remwait %x convert_reply zap overlap_cancel»,
lkb->lkb_id);
lkb->lkb_wait_type = 0;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
lkb->lkb_wait_count–;
goto out_del;
}

/* N.B. type of reply may not always correspond to type of original
msg due to lookup->request optimization, verify others? */

if (lkb->lkb_wait_type) {
lkb->lkb_wait_type = 0;
goto out_del;
}

log_error(ls, «remwait error %x reply %d flags %x no wait_type»,
lkb->lkb_id, mstype, lkb->lkb_flags);
return -1;

out_del:
/* the force-unlock/cancel has completed and we haven’t recvd a reply
to the op that was in progress prior to the unlock/cancel; we
give up on any reply to the earlier op. FIXME: not sure when/how
this would happen */

if (overlap_done && lkb->lkb_wait_type) {
log_error(ls, «remwait error %x reply %d wait_type %d overlap»,
lkb->lkb_id, mstype, lkb->lkb_wait_type);
lkb->lkb_wait_count–;
lkb->lkb_wait_type = 0;
}

DLM_ASSERT(lkb->lkb_wait_count, dlm_print_lkb(lkb););

lkb->lkb_flags &= ~DLM_IFL_RESEND;
lkb->lkb_wait_count–;
if (!lkb->lkb_wait_count)
list_del_init(&lkb->lkb_wait_reply);
unhold_lkb(lkb);
return 0;
}

static int remove_from_waiters(struct dlm_lkb *lkb, int mstype)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;
int error;

mutex_lock(&ls->ls_waiters_mutex);
error = _remove_from_waiters(lkb, mstype, NULL);
mutex_unlock(&ls->ls_waiters_mutex);
return error;
}

/* Handles situations where we might be processing a «fake» or «stub» reply in
which we can’t try to take waiters_mutex again. */

static int remove_from_waiters_ms(struct dlm_lkb *lkb, struct dlm_message *ms)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;
int error;

if (ms != &ls->ls_stub_ms)
mutex_lock(&ls->ls_waiters_mutex);
error = _remove_from_waiters(lkb, ms->m_type, ms);
if (ms != &ls->ls_stub_ms)
mutex_unlock(&ls->ls_waiters_mutex);
return error;
}

static void dir_remove(struct dlm_rsb *r)
{
int to_nodeid;

if (dlm_no_directory(r->res_ls))
return;

to_nodeid = dlm_dir_nodeid(r);
if (to_nodeid != dlm_our_nodeid())
send_remove(r);
else
dlm_dir_remove_entry(r->res_ls, to_nodeid,
r->res_name, r->res_length);
}

/* FIXME: shouldn’t this be able to exit as soon as one non-due rsb is
found since they are in order of newest to oldest? */

static int shrink_bucket(struct dlm_ls *ls, int b)
{
struct dlm_rsb *r;
int count = 0, found;

for (;;) {
found = 0;
spin_lock(&ls->ls_rsbtbl[b].lock);
list_for_each_entry_reverse(r, &ls->ls_rsbtbl[b].toss,
res_hashchain) {
if (!time_after_eq(jiffies, r->res_toss_time +
dlm_config.ci_toss_secs * HZ))
continue;
found = 1;
break;
}

if (!found) {
spin_unlock(&ls->ls_rsbtbl[b].lock);
break;
}

if (kref_put(&r->res_ref, kill_rsb)) {
list_del(&r->res_hashchain);
spin_unlock(&ls->ls_rsbtbl[b].lock);

if (is_master(r))
dir_remove(r);
dlm_free_rsb(r);
count++;
} else {
spin_unlock(&ls->ls_rsbtbl[b].lock);
log_error(ls, «tossed rsb in use %s», r->res_name);
}
}

return count;
}

void dlm_scan_rsbs(struct dlm_ls *ls)
{
int i;

for (i = 0; i < ls->ls_rsbtbl_size; i++) {
shrink_bucket(ls, i);
if (dlm_locking_stopped(ls))
break;
cond_resched();
}
}

static void add_timeout(struct dlm_lkb *lkb)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;

if (is_master_copy(lkb))
return;

if (test_bit(LSFL_TIMEWARN, &ls->ls_flags) &&
!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
lkb->lkb_flags |= DLM_IFL_WATCH_TIMEWARN;
goto add_it;
}
if (lkb->lkb_exflags & DLM_LKF_TIMEOUT)
goto add_it;
return;

add_it:
DLM_ASSERT(list_empty(&lkb->lkb_time_list), dlm_print_lkb(lkb););
mutex_lock(&ls->ls_timeout_mutex);
hold_lkb(lkb);
list_add_tail(&lkb->lkb_time_list, &ls->ls_timeout);
mutex_unlock(&ls->ls_timeout_mutex);
}

static void del_timeout(struct dlm_lkb *lkb)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;

mutex_lock(&ls->ls_timeout_mutex);
if (!list_empty(&lkb->lkb_time_list)) {
list_del_init(&lkb->lkb_time_list);
unhold_lkb(lkb);
}
mutex_unlock(&ls->ls_timeout_mutex);
}

/* FIXME: is it safe to look at lkb_exflags, lkb_flags, lkb_timestamp, and
lkb_lksb_timeout without lock_rsb? Note: we can’t lock timeout_mutex
and then lock rsb because of lock ordering in add_timeout. We may need
to specify some special timeout-related bits in the lkb that are just to
be accessed under the timeout_mutex. */

void dlm_scan_timeout(struct dlm_ls *ls)
{
struct dlm_rsb *r;
struct dlm_lkb *lkb;
int do_cancel, do_warn;
s64 wait_us;

for (;;) {
if (dlm_locking_stopped(ls))
break;

do_cancel = 0;
do_warn = 0;
mutex_lock(&ls->ls_timeout_mutex);
list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) {

wait_us = ktime_to_us(ktime_sub(ktime_get(),
lkb->lkb_timestamp));

if ((lkb->lkb_exflags & DLM_LKF_TIMEOUT) &&
wait_us >= (lkb->lkb_timeout_cs * 10000))
do_cancel = 1;

if ((lkb->lkb_flags & DLM_IFL_WATCH_TIMEWARN) &&
wait_us >= dlm_config.ci_timewarn_cs * 10000)
do_warn = 1;

if (!do_cancel && !do_warn)
continue;
hold_lkb(lkb);
break;
}
mutex_unlock(&ls->ls_timeout_mutex);

if (!do_cancel && !do_warn)
break;

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

if (do_warn) {
/* clear flag so we only warn once */
lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN;
if (!(lkb->lkb_exflags & DLM_LKF_TIMEOUT))
del_timeout(lkb);
dlm_timeout_warn(lkb);
}

if (do_cancel) {
log_debug(ls, «timeout cancel %x node %d %s»,
lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN;
lkb->lkb_flags |= DLM_IFL_TIMEOUT_CANCEL;
del_timeout(lkb);
_cancel_lock(r, lkb);
}

unlock_rsb(r);
unhold_rsb(r);
dlm_put_lkb(lkb);
}
}

/* This is only called by dlm_recoverd, and we rely on dlm_ls_stop() stopping
dlm_recoverd before checking/setting ls_recover_begin. */

void dlm_adjust_timeouts(struct dlm_ls *ls)
{
struct dlm_lkb *lkb;
u64 adj_us = jiffies_to_usecs(jiffies – ls->ls_recover_begin);

ls->ls_recover_begin = 0;
mutex_lock(&ls->ls_timeout_mutex);
list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list)
lkb->lkb_timestamp = ktime_add_us(lkb->lkb_timestamp, adj_us);
mutex_unlock(&ls->ls_timeout_mutex);
}

/* lkb is master or local copy */

static void set_lvb_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int b, len = r->res_ls->ls_lvblen;

/* b=1 lvb returned to caller
b=0 lvb written to rsb or invalidated
b=-1 do nothing */

b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1];

if (b == 1) {
if (!lkb->lkb_lvbptr)
return;

if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
return;

if (!r->res_lvbptr)
return;

memcpy(lkb->lkb_lvbptr, r->res_lvbptr, len);
lkb->lkb_lvbseq = r->res_lvbseq;

} else if (b == 0) {
if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) {
rsb_set_flag(r, RSB_VALNOTVALID);
return;
}

if (!lkb->lkb_lvbptr)
return;

if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
return;

if (!r->res_lvbptr)
r->res_lvbptr = dlm_allocate_lvb(r->res_ls);

if (!r->res_lvbptr)
return;

memcpy(r->res_lvbptr, lkb->lkb_lvbptr, len);
r->res_lvbseq++;
lkb->lkb_lvbseq = r->res_lvbseq;
rsb_clear_flag(r, RSB_VALNOTVALID);
}

if (rsb_flag(r, RSB_VALNOTVALID))
lkb->lkb_sbflags |= DLM_SBF_VALNOTVALID;
}

static void set_lvb_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
if (lkb->lkb_grmode < DLM_LOCK_PW)
return;

if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) {
rsb_set_flag(r, RSB_VALNOTVALID);
return;
}

if (!lkb->lkb_lvbptr)
return;

if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
return;

if (!r->res_lvbptr)
r->res_lvbptr = dlm_allocate_lvb(r->res_ls);

if (!r->res_lvbptr)
return;

memcpy(r->res_lvbptr, lkb->lkb_lvbptr, r->res_ls->ls_lvblen);
r->res_lvbseq++;
rsb_clear_flag(r, RSB_VALNOTVALID);
}

/* lkb is process copy (pc) */

static void set_lvb_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
int b;

if (!lkb->lkb_lvbptr)
return;

if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
return;

b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1];
if (b == 1) {
int len = receive_extralen(ms);
if (len > DLM_RESNAME_MAXLEN)
len = DLM_RESNAME_MAXLEN;
memcpy(lkb->lkb_lvbptr, ms->m_extra, len);
lkb->lkb_lvbseq = ms->m_lvbseq;
}
}

/* Manipulate lkb’s on rsb’s convert/granted/waiting queues
remove_lock — used for unlock, removes lkb from granted
revert_lock — used for cancel, moves lkb from convert to granted
grant_lock — used for request and convert, adds lkb to granted or
moves lkb from convert or waiting to granted

Each of these is used for master or local copy lkb’s. There is
also a _pc() variation used to make the corresponding change on
a process copy (pc) lkb. */

static void _remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
del_lkb(r, lkb);
lkb->lkb_grmode = DLM_LOCK_IV;
/* this unhold undoes the original ref from create_lkb()
so this leads to the lkb being freed */
unhold_lkb(lkb);
}

static void remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
set_lvb_unlock(r, lkb);
_remove_lock(r, lkb);
}

static void remove_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
_remove_lock(r, lkb);
}

/* returns: 0 did nothing
1 moved lock to granted
-1 removed lock */

static int revert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int rv = 0;

lkb->lkb_rqmode = DLM_LOCK_IV;

switch (lkb->lkb_status) {
case DLM_LKSTS_GRANTED:
break;
case DLM_LKSTS_CONVERT:
move_lkb(r, lkb, DLM_LKSTS_GRANTED);
rv = 1;
break;
case DLM_LKSTS_WAITING:
del_lkb(r, lkb);
lkb->lkb_grmode = DLM_LOCK_IV;
/* this unhold undoes the original ref from create_lkb()
so this leads to the lkb being freed */
unhold_lkb(lkb);
rv = -1;
break;
default:
log_print(»invalid status for revert %d», lkb->lkb_status);
}
return rv;
}

static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
return revert_lock(r, lkb);
}

static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
if (lkb->lkb_grmode != lkb->lkb_rqmode) {
lkb->lkb_grmode = lkb->lkb_rqmode;
if (lkb->lkb_status)
move_lkb(r, lkb, DLM_LKSTS_GRANTED);
else
add_lkb(r, lkb, DLM_LKSTS_GRANTED);
}

lkb->lkb_rqmode = DLM_LOCK_IV;
}

static void grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
set_lvb_lock(r, lkb);
_grant_lock(r, lkb);
lkb->lkb_highbast = 0;
}

static void grant_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
set_lvb_lock_pc(r, lkb, ms);
_grant_lock(r, lkb);
}

/* called by grant_pending_locks() which means an async grant message must
be sent to the requesting node in addition to granting the lock if the
lkb belongs to a remote node. */

static void grant_lock_pending(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
grant_lock(r, lkb);
if (is_master_copy(lkb))
send_grant(r, lkb);
else
queue_cast(r, lkb, 0);
}

/* The special CONVDEADLK, ALTPR and ALTCW flags allow the master to
change the granted/requested modes. We’re munging things accordingly in
the process copy.
CONVDEADLK: our grmode may have been forced down to NL to resolve a
conversion deadlock
ALTPR/ALTCW: our rqmode may have been changed to PR or CW to become
compatible with other granted locks */

static void munge_demoted(struct dlm_lkb *lkb, struct dlm_message *ms)
{
if (ms->m_type != DLM_MSG_CONVERT_REPLY) {
log_print(»munge_demoted %x invalid reply type %d»,
lkb->lkb_id, ms->m_type);
return;
}

if (lkb->lkb_rqmode == DLM_LOCK_IV || lkb->lkb_grmode == DLM_LOCK_IV) {
log_print(»munge_demoted %x invalid modes gr %d rq %d»,
lkb->lkb_id, lkb->lkb_grmode, lkb->lkb_rqmode);
return;
}

lkb->lkb_grmode = DLM_LOCK_NL;
}

static void munge_altmode(struct dlm_lkb *lkb, struct dlm_message *ms)
{
if (ms->m_type != DLM_MSG_REQUEST_REPLY &&
ms->m_type != DLM_MSG_GRANT) {
log_print(»munge_altmode %x invalid reply type %d»,
lkb->lkb_id, ms->m_type);
return;
}

if (lkb->lkb_exflags & DLM_LKF_ALTPR)
lkb->lkb_rqmode = DLM_LOCK_PR;
else if (lkb->lkb_exflags & DLM_LKF_ALTCW)
lkb->lkb_rqmode = DLM_LOCK_CW;
else {
log_print(»munge_altmode invalid exflags %x», lkb->lkb_exflags);
dlm_print_lkb(lkb);
}
}

static inline int first_in_list(struct dlm_lkb *lkb, struct list_head *head)
{
struct dlm_lkb *first = list_entry(head->next, struct dlm_lkb,
lkb_statequeue);
if (lkb->lkb_id == first->lkb_id)
return 1;

return 0;
}

/* Check if the given lkb conflicts with another lkb on the queue. */

static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
{
struct dlm_lkb *this;

list_for_each_entry(this, head, lkb_statequeue) {
if (this == lkb)
continue;
if (!modes_compat(this, lkb))
return 1;
}
return 0;
}

/*
* «A conversion deadlock arises with a pair of lock requests in the converting
* queue for one resource. The granted mode of each lock blocks the requested
* mode of the other lock.»
*
* Part 2: if the granted mode of lkb is preventing an earlier lkb in the
* convert queue from being granted, then deadlk/demote lkb.
*
* Example:
* Granted Queue: empty
* Convert Queue: NL->EX (first lock)
* PR->EX (second lock)
*
* The first lock can’t be granted because of the granted mode of the second
* lock and the second lock can’t be granted because it’s not first in the
* list. We either cancel lkb’s conversion (PR->EX) and return EDEADLK, or we
* demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK
* flag set and return DEMOTED in the lksb flags.
*
* Originally, this function detected conv-deadlk in a more limited scope:
* – if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or
* – if lkb1 was the first entry in the queue (not just earlier), and was
* blocked by the granted mode of lkb2, and there was nothing on the
* granted queue preventing lkb1 from being granted immediately, i.e.
* lkb2 was the only thing preventing lkb1 from being granted.
*
* That second condition meant we’d only say there was conv-deadlk if
* resolving it (by demotion) would lead to the first lock on the convert
* queue being granted right away. It allowed conversion deadlocks to exist
* between locks on the convert queue while they couldn’t be granted anyway.
*
* Now, we detect and take action on conversion deadlocks immediately when
* they’re created, even if they may not be immediately consequential. If
* lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted
* mode that would prevent lkb1’s conversion from being granted, we do a
* deadlk/demote on lkb2 right away and don’t let it onto the convert queue.
* I think this means that the lkb_is_ahead condition below should always
* be zero, i.e. there will never be conv-deadlk between two locks that are
* both already on the convert queue.
*/

static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2)
{
struct dlm_lkb *lkb1;
int lkb_is_ahead = 0;

list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) {
if (lkb1 == lkb2) {
lkb_is_ahead = 1;
continue;
}

if (!lkb_is_ahead) {
if (!modes_compat(lkb2, lkb1))
return 1;
} else {
if (!modes_compat(lkb2, lkb1) &&
!modes_compat(lkb1, lkb2))
return 1;
}
}
return 0;
}

/*
* Return 1 if the lock can be granted, 0 otherwise.
* Also detect and resolve conversion deadlocks.
*
* lkb is the lock to be granted
*
* now is 1 if the function is being called in the context of the
* immediate request, it is 0 if called later, after the lock has been
* queued.
*
* References are from chapter 6 of «VAXcluster Principles» by Roy Davis
*/

static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
{
int8_t conv = (lkb->lkb_grmode != DLM_LOCK_IV);

/*
* 6-10: Version 5.4 introduced an option to address the phenomenon of
* a new request for a NL mode lock being blocked.
*
* 6-11: If the optional EXPEDITE flag is used with the new NL mode
* request, then it would be granted. In essence, the use of this flag
* tells the Lock Manager to expedite theis request by not considering
* what may be in the CONVERTING or WAITING queues… As of this
* writing, the EXPEDITE flag can be used only with new requests for NL
* mode locks. This flag is not valid for conversion requests.
*
* A shortcut. Earlier checks return an error if EXPEDITE is used in a
* conversion or used with a non-NL requested mode. We also know an
* EXPEDITE request is always granted immediately, so now must always
* be 1. The full condition to grant an expedite request: (now &&
* !conv && lkb->rqmode == DLM_LOCK_NL && (flags & EXPEDITE)) can
* therefore be shortened to just checking the flag.
*/

if (lkb->lkb_exflags & DLM_LKF_EXPEDITE)
return 1;

/*
* A shortcut. Without this, !queue_conflict(grantqueue, lkb) would be
* added to the remaining conditions.
*/

if (queue_conflict(&r->res_grantqueue, lkb))
goto out;

/*
* 6-3: By default, a conversion request is immediately granted if the
* requested mode is compatible with the modes of all other granted
* locks
*/

if (queue_conflict(&r->res_convertqueue, lkb))
goto out;

/*
* 6-5: But the default algorithm for deciding whether to grant or
* queue conversion requests does not by itself guarantee that such
* requests are serviced on a «first come first serve» basis. This, in
* turn, can lead to a phenomenon known as «indefinate postponement».
*
* 6-7: This issue is dealt with by using the optional QUECVT flag with
* the system service employed to request a lock conversion. This flag
* forces certain conversion requests to be queued, even if they are
* compatible with the granted modes of other locks on the same
* resource. Thus, the use of this flag results in conversion requests
* being ordered on a «first come first servce» basis.
*
* DCT: This condition is all about new conversions being able to occur
* «in place» while the lock remains on the granted queue (assuming
* nothing else conflicts.) IOW if QUECVT isn’t set, a conversion
* doesn’t _have_ to go onto the convert queue where it’s processed in
* order. The «now» variable is necessary to distinguish converts
* being received and processed for the first time now, because once a
* convert is moved to the conversion queue the condition below applies
* requiring fifo granting.
*/

if (now && conv && !(lkb->lkb_exflags & DLM_LKF_QUECVT))
return 1;

/*
* The NOORDER flag is set to avoid the standard vms rules on grant
* order.
*/

if (lkb->lkb_exflags & DLM_LKF_NOORDER)
return 1;

/*
* 6-3: Once in that queue [CONVERTING], a conversion request cannot be
* granted until all other conversion requests ahead of it are granted
* and/or canceled.
*/

if (!now && conv && first_in_list(lkb, &r->res_convertqueue))
return 1;

/*
* 6-4: By default, a new request is immediately granted only if all
* three of the following conditions are satisfied when the request is
* issued:
* – The queue of ungranted conversion requests for the resource is
* empty.
* – The queue of ungranted new requests for the resource is empty.
* – The mode of the new request is compatible with the most
* restrictive mode of all granted locks on the resource.
*/

if (now && !conv && list_empty(&r->res_convertqueue) &&
list_empty(&r->res_waitqueue))
return 1;

/*
* 6-4: Once a lock request is in the queue of ungranted new requests,
* it cannot be granted until the queue of ungranted conversion
* requests is empty, all ungranted new requests ahead of it are
* granted and/or canceled, and it is compatible with the granted mode
* of the most restrictive lock granted on the resource.
*/

if (!now && !conv && list_empty(&r->res_convertqueue) &&
first_in_list(lkb, &r->res_waitqueue))
return 1;
out:
return 0;
}

static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now,
int *err)
{
int rv;
int8_t alt = 0, rqmode = lkb->lkb_rqmode;
int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV);

if (err)
*err = 0;

rv = _can_be_granted(r, lkb, now);
if (rv)
goto out;

/*
* The CONVDEADLK flag is non-standard and tells the dlm to resolve
* conversion deadlocks by demoting grmode to NL, otherwise the dlm
* cancels one of the locks.
*/

if (is_convert && can_be_queued(lkb) &&
conversion_deadlock_detect(r, lkb)) {
if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) {
lkb->lkb_grmode = DLM_LOCK_NL;
lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
} else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
if (err)
*err = -EDEADLK;
else {
log_print(»can_be_granted deadlock %x now %d»,
lkb->lkb_id, now);
dlm_dump_rsb(r);
}
}
goto out;
}

/*
* The ALTPR and ALTCW flags are non-standard and tell the dlm to try
* to grant a request in a mode other than the normal rqmode. It’s a
* simple way to provide a big optimization to applications that can
* use them.
*/

if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR))
alt = DLM_LOCK_PR;
else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW))
alt = DLM_LOCK_CW;

if (alt) {
lkb->lkb_rqmode = alt;
rv = _can_be_granted(r, lkb, now);
if (rv)
lkb->lkb_sbflags |= DLM_SBF_ALTMODE;
else
lkb->lkb_rqmode = rqmode;
}
out:
return rv;
}

/* FIXME: I don’t think that can_be_granted() can/will demote or find deadlock
for locks pending on the convert list. Once verified (watch for these
log_prints), we should be able to just call _can_be_granted() and not
bother with the demote/deadlk cases here (and there’s no easy way to deal
with a deadlk here, we’d have to generate something like grant_lock with
the deadlk error.) */

/* Returns the highest requested mode of all blocked conversions; sets
cw if there’s a blocked conversion to DLM_LOCK_CW. */

static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw)
{
struct dlm_lkb *lkb, *s;
int hi, demoted, quit, grant_restart, demote_restart;
int deadlk;

quit = 0;
restart:
grant_restart = 0;
demote_restart = 0;
hi = DLM_LOCK_IV;

list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) {
demoted = is_demoted(lkb);
deadlk = 0;

if (can_be_granted(r, lkb, 0, &deadlk)) {
grant_lock_pending(r, lkb);
grant_restart = 1;
continue;
}

if (!demoted && is_demoted(lkb)) {
log_print(»WARN: pending demoted %x node %d %s»,
lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
demote_restart = 1;
continue;
}

if (deadlk) {
log_print(»WARN: pending deadlock %x node %d %s»,
lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
dlm_dump_rsb(r);
continue;
}

hi = max_t(int, lkb->lkb_rqmode, hi);

if (cw && lkb->lkb_rqmode == DLM_LOCK_CW)
*cw = 1;
}

if (grant_restart)
goto restart;
if (demote_restart && !quit) {
quit = 1;
goto restart;
}

return max_t(int, high, hi);
}

static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw)
{
struct dlm_lkb *lkb, *s;

list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) {
if (can_be_granted(r, lkb, 0, NULL))
grant_lock_pending(r, lkb);
else {
high = max_t(int, lkb->lkb_rqmode, high);
if (lkb->lkb_rqmode == DLM_LOCK_CW)
*cw = 1;
}
}

return high;
}

/* cw of 1 means there’s a lock with a rqmode of DLM_LOCK_CW that’s blocked
on either the convert or waiting queue.
high is the largest rqmode of all locks blocked on the convert or
waiting queue. */

static int lock_requires_bast(struct dlm_lkb *gr, int high, int cw)
{
if (gr->lkb_grmode == DLM_LOCK_PR && cw) {
if (gr->lkb_highbast < DLM_LOCK_EX)
return 1;
return 0;
}

if (gr->lkb_highbast < high &&
!__dlm_compat_matrix[gr->lkb_grmode+1][high+1])
return 1;
return 0;
}

static void grant_pending_locks(struct dlm_rsb *r)
{
struct dlm_lkb *lkb, *s;
int high = DLM_LOCK_IV;
int cw = 0;

DLM_ASSERT(is_master(r), dlm_dump_rsb(r););

high = grant_pending_convert(r, high, &cw);
high = grant_pending_wait(r, high, &cw);

if (high == DLM_LOCK_IV)
return;

/*
* If there are locks left on the wait/convert queue then send blocking
* ASTs to granted locks based on the largest requested mode (high)
* found above.
*/

list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) {
if (lkb->lkb_bastfn && lock_requires_bast(lkb, high, cw)) {
if (cw && high == DLM_LOCK_PR &&
lkb->lkb_grmode == DLM_LOCK_PR)
queue_bast(r, lkb, DLM_LOCK_CW);
else
queue_bast(r, lkb, high);
lkb->lkb_highbast = high;
}
}
}

static int modes_require_bast(struct dlm_lkb *gr, struct dlm_lkb *rq)
{
if ((gr->lkb_grmode == DLM_LOCK_PR && rq->lkb_rqmode == DLM_LOCK_CW) ||
(gr->lkb_grmode == DLM_LOCK_CW && rq->lkb_rqmode == DLM_LOCK_PR)) {
if (gr->lkb_highbast < DLM_LOCK_EX)
return 1;
return 0;
}

if (gr->lkb_highbast < rq->lkb_rqmode && !modes_compat(gr, rq))
return 1;
return 0;
}

static void send_bast_queue(struct dlm_rsb *r, struct list_head *head,
struct dlm_lkb *lkb)
{
struct dlm_lkb *gr;

list_for_each_entry(gr, head, lkb_statequeue) {
if (gr->lkb_bastfn && modes_require_bast(gr, lkb)) {
queue_bast(r, gr, lkb->lkb_rqmode);
gr->lkb_highbast = lkb->lkb_rqmode;
}
}
}

static void send_blocking_asts(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
send_bast_queue(r, &r->res_grantqueue, lkb);
}

static void send_blocking_asts_all(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
send_bast_queue(r, &r->res_grantqueue, lkb);
send_bast_queue(r, &r->res_convertqueue, lkb);
}

/* set_master(r, lkb) — set the master nodeid of a resource

The purpose of this function is to set the nodeid field in the given
lkb using the nodeid field in the given rsb. If the rsb’s nodeid is
known, it can just be copied to the lkb and the function will return
0. If the rsb’s nodeid is _not_ known, it needs to be looked up
before it can be copied to the lkb.

When the rsb nodeid is being looked up remotely, the initial lkb
causing the lookup is kept on the ls_waiters list waiting for the
lookup reply. Other lkb’s waiting for the same rsb lookup are kept
on the rsb’s res_lookup list until the master is verified.

Return values:
0: nodeid is set in rsb/lkb and the caller should go ahead and use it
1: the rsb master is not available and the lkb has been placed on
a wait queue
*/

static int set_master(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
struct dlm_ls *ls = r->res_ls;
int i, error, dir_nodeid, ret_nodeid, our_nodeid = dlm_our_nodeid();

if (rsb_flag(r, RSB_MASTER_UNCERTAIN)) {
rsb_clear_flag(r, RSB_MASTER_UNCERTAIN);
r->res_first_lkid = lkb->lkb_id;
lkb->lkb_nodeid = r->res_nodeid;
return 0;
}

if (r->res_first_lkid && r->res_first_lkid != lkb->lkb_id) {
list_add_tail(&lkb->lkb_rsb_lookup, &r->res_lookup);
return 1;
}

if (r->res_nodeid == 0) {
lkb->lkb_nodeid = 0;
return 0;
}

if (r->res_nodeid > 0) {
lkb->lkb_nodeid = r->res_nodeid;
return 0;
}

DLM_ASSERT(r->res_nodeid == -1, dlm_dump_rsb(r););

dir_nodeid = dlm_dir_nodeid(r);

if (dir_nodeid != our_nodeid) {
r->res_first_lkid = lkb->lkb_id;
send_lookup(r, lkb);
return 1;
}

for (i = 0; i < 2; i++) {
/* It's possible for dlm_scand to remove an old rsb for
this same resource from the toss list, us to create
a new one, look up the master locally, and find it
already exists just before dlm_scand does the
dir_remove() on the previous rsb. */

error = dlm_dir_lookup(ls, our_nodeid, r->res_name,
r->res_length, &ret_nodeid);
if (!error)
break;
log_debug(ls, «dir_lookup error %d %s», error, r->res_name);
schedule();
}
if (error && error != -EEXIST)
return error;

if (ret_nodeid == our_nodeid) {
r->res_first_lkid = 0;
r->res_nodeid = 0;
lkb->lkb_nodeid = 0;
} else {
r->res_first_lkid = lkb->lkb_id;
r->res_nodeid = ret_nodeid;
lkb->lkb_nodeid = ret_nodeid;
}
return 0;
}

static void process_lookup_list(struct dlm_rsb *r)
{
struct dlm_lkb *lkb, *safe;

list_for_each_entry_safe(lkb, safe, &r->res_lookup, lkb_rsb_lookup) {
list_del_init(&lkb->lkb_rsb_lookup);
_request_lock(r, lkb);
schedule();
}
}

/* confirm_master — confirm (or deny) an rsb’s master nodeid */

static void confirm_master(struct dlm_rsb *r, int error)
{
struct dlm_lkb *lkb;

if (!r->res_first_lkid)
return;

switch (error) {
case 0:
case -EINPROGRESS:
r->res_first_lkid = 0;
process_lookup_list(r);
break;

case -EAGAIN:
case -EBADR:
case -ENOTBLK:
/* the remote request failed and won’t be retried (it was
a NOQUEUE, or has been canceled/unlocked); make a waiting
lkb the first_lkid */

r->res_first_lkid = 0;

if (!list_empty(&r->res_lookup)) {
lkb = list_entry(r->res_lookup.next, struct dlm_lkb,
lkb_rsb_lookup);
list_del_init(&lkb->lkb_rsb_lookup);
r->res_first_lkid = lkb->lkb_id;
_request_lock(r, lkb);
}
break;

default:
log_error(r->res_ls, «confirm_master unknown error %d», error);
}
}

static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags,
int namelen, unsigned long timeout_cs,
void (*ast) (void *astparam),
void *astparam,
void (*bast) (void *astparam, int mode),
struct dlm_args *args)
{
int rv = -EINVAL;

/* check for invalid arg usage */

if (mode < 0 || mode > DLM_LOCK_EX)
goto out;

if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN))
goto out;

if (flags & DLM_LKF_CANCEL)
goto out;

if (flags & DLM_LKF_QUECVT && !(flags & DLM_LKF_CONVERT))
goto out;

if (flags & DLM_LKF_CONVDEADLK && !(flags & DLM_LKF_CONVERT))
goto out;

if (flags & DLM_LKF_CONVDEADLK && flags & DLM_LKF_NOQUEUE)
goto out;

if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_CONVERT)
goto out;

if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_QUECVT)
goto out;

if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_NOQUEUE)
goto out;

if (flags & DLM_LKF_EXPEDITE && mode != DLM_LOCK_NL)
goto out;

if (!ast || !lksb)
goto out;

if (flags & DLM_LKF_VALBLK && !lksb->sb_lvbptr)
goto out;

if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid)
goto out;

/* these args will be copied to the lkb in validate_lock_args,
it cannot be done now because when converting locks, fields in
an active lkb cannot be modified before locking the rsb */

args->flags = flags;
args->astfn = ast;
args->astparam = astparam;
args->bastfn = bast;
args->timeout = timeout_cs;
args->mode = mode;
args->lksb = lksb;
rv = 0;
out:
return rv;
}

static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args)
{
if (flags & ~(DLM_LKF_CANCEL | DLM_LKF_VALBLK | DLM_LKF_IVVALBLK |
DLM_LKF_FORCEUNLOCK))
return -EINVAL;

if (flags & DLM_LKF_CANCEL && flags & DLM_LKF_FORCEUNLOCK)
return -EINVAL;

args->flags = flags;
args->astparam = astarg;
return 0;
}

static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_args *args)
{
int rv = -EINVAL;

if (args->flags & DLM_LKF_CONVERT) {
if (lkb->lkb_flags & DLM_IFL_MSTCPY)
goto out;

if (args->flags & DLM_LKF_QUECVT &&
!__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1])
goto out;

rv = -EBUSY;
if (lkb->lkb_status != DLM_LKSTS_GRANTED)
goto out;

if (lkb->lkb_wait_type)
goto out;

if (is_overlap(lkb))
goto out;
}

lkb->lkb_exflags = args->flags;
lkb->lkb_sbflags = 0;
lkb->lkb_astfn = args->astfn;
lkb->lkb_astparam = args->astparam;
lkb->lkb_bastfn = args->bastfn;
lkb->lkb_rqmode = args->mode;
lkb->lkb_lksb = args->lksb;
lkb->lkb_lvbptr = args->lksb->sb_lvbptr;
lkb->lkb_ownpid = (int) current->pid;
lkb->lkb_timeout_cs = args->timeout;
rv = 0;
out:
if (rv)
log_debug(ls, «validate_lock_args %d %x %x %x %d %d %s»,
rv, lkb->lkb_id, lkb->lkb_flags, args->flags,
lkb->lkb_status, lkb->lkb_wait_type,
lkb->lkb_resource->res_name);
return rv;
}

/* when dlm_unlock() sees -EBUSY with CANCEL/FORCEUNLOCK it returns 0
for success */

/* note: it’s valid for lkb_nodeid/res_nodeid to be -1 when we get here
because there may be a lookup in progress and it’s valid to do
cancel/unlockf on it */

static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
{
struct dlm_ls *ls = lkb->lkb_resource->res_ls;
int rv = -EINVAL;

if (lkb->lkb_flags & DLM_IFL_MSTCPY) {
log_error(ls, «unlock on MSTCPY %x», lkb->lkb_id);
dlm_print_lkb(lkb);
goto out;
}

/* an lkb may still exist even though the lock is EOL’ed due to a
cancel, unlock or failed noqueue request; an app can’t use these
locks; return same error as if the lkid had not been found at all */

if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) {
log_debug(ls, «unlock on ENDOFLIFE %x», lkb->lkb_id);
rv = -ENOENT;
goto out;
}

/* an lkb may be waiting for an rsb lookup to complete where the
lookup was initiated by another lock */

if (!list_empty(&lkb->lkb_rsb_lookup)) {
if (args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) {
log_debug(ls, «unlock on rsb_lookup %x», lkb->lkb_id);
list_del_init(&lkb->lkb_rsb_lookup);
queue_cast(lkb->lkb_resource, lkb,
args->flags & DLM_LKF_CANCEL ?
-DLM_ECANCEL : -DLM_EUNLOCK);
unhold_lkb(lkb); /* undoes create_lkb() */
}
/* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */
rv = -EBUSY;
goto out;
}

/* cancel not allowed with another cancel/unlock in progress */

if (args->flags & DLM_LKF_CANCEL) {
if (lkb->lkb_exflags & DLM_LKF_CANCEL)
goto out;

if (is_overlap(lkb))
goto out;

/* don’t let scand try to do a cancel */
del_timeout(lkb);

if (lkb->lkb_flags & DLM_IFL_RESEND) {
lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
rv = -EBUSY;
goto out;
}

/* there’s nothing to cancel */
if (lkb->lkb_status == DLM_LKSTS_GRANTED &&
!lkb->lkb_wait_type) {
rv = -EBUSY;
goto out;
}

switch (lkb->lkb_wait_type) {
case DLM_MSG_LOOKUP:
case DLM_MSG_REQUEST:
lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
rv = -EBUSY;
goto out;
case DLM_MSG_UNLOCK:
case DLM_MSG_CANCEL:
goto out;
}
/* add_to_waiters() will set OVERLAP_CANCEL */
goto out_ok;
}

/* do we need to allow a force-unlock if there’s a normal unlock
already in progress? in what conditions could the normal unlock
fail such that we’d want to send a force-unlock to be sure? */

if (args->flags & DLM_LKF_FORCEUNLOCK) {
if (lkb->lkb_exflags & DLM_LKF_FORCEUNLOCK)
goto out;

if (is_overlap_unlock(lkb))
goto out;

/* don’t let scand try to do a cancel */
del_timeout(lkb);

if (lkb->lkb_flags & DLM_IFL_RESEND) {
lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
rv = -EBUSY;
goto out;
}

switch (lkb->lkb_wait_type) {
case DLM_MSG_LOOKUP:
case DLM_MSG_REQUEST:
lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
rv = -EBUSY;
goto out;
case DLM_MSG_UNLOCK:
goto out;
}
/* add_to_waiters() will set OVERLAP_UNLOCK */
goto out_ok;
}

/* normal unlock not allowed if there’s any op in progress */
rv = -EBUSY;
if (lkb->lkb_wait_type || lkb->lkb_wait_count)
goto out;

out_ok:
/* an overlapping op shouldn’t blow away exflags from other op */
lkb->lkb_exflags |= args->flags;
lkb->lkb_sbflags = 0;
lkb->lkb_astparam = args->astparam;
rv = 0;
out:
if (rv)
log_debug(ls, «validate_unlock_args %d %x %x %x %x %d %s», rv,
lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags,
args->flags, lkb->lkb_wait_type,
lkb->lkb_resource->res_name);
return rv;
}

/*
* Four stage 4 varieties:
* do_request(), do_convert(), do_unlock(), do_cancel()
* These are called on the master node for the given lock and
* from the central locking logic.
*/

static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error = 0;

if (can_be_granted(r, lkb, 1, NULL)) {
grant_lock(r, lkb);
queue_cast(r, lkb, 0);
goto out;
}

if (can_be_queued(lkb)) {
error = -EINPROGRESS;
add_lkb(r, lkb, DLM_LKSTS_WAITING);
send_blocking_asts(r, lkb);
add_timeout(lkb);
goto out;
}

error = -EAGAIN;
if (force_blocking_asts(lkb))
send_blocking_asts_all(r, lkb);
queue_cast(r, lkb, -EAGAIN);

out:
return error;
}

static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error = 0;
int deadlk = 0;

/* changing an existing lock may allow others to be granted */

if (can_be_granted(r, lkb, 1, &deadlk)) {
grant_lock(r, lkb);
queue_cast(r, lkb, 0);
grant_pending_locks(r);
goto out;
}

/* can_be_granted() detected that this lock would block in a conversion
deadlock, so we leave it on the granted queue and return EDEADLK in
the ast for the convert. */

if (deadlk) {
/* it’s left on the granted queue */
log_debug(r->res_ls, «deadlock %x node %d sts%d g%d r%d %s»,
lkb->lkb_id, lkb->lkb_nodeid, lkb->lkb_status,
lkb->lkb_grmode, lkb->lkb_rqmode, r->res_name);
revert_lock(r, lkb);
queue_cast(r, lkb, -EDEADLK);
error = -EDEADLK;
goto out;
}

/* is_demoted() means the can_be_granted() above set the grmode
to NL, and left us on the granted queue. This auto-demotion
(due to CONVDEADLK) might mean other locks, and/or this lock, are
now grantable. We have to try to grant other converting locks
before we try again to grant this one. */

if (is_demoted(lkb)) {
grant_pending_convert(r, DLM_LOCK_IV, NULL);
if (_can_be_granted(r, lkb, 1)) {
grant_lock(r, lkb);
queue_cast(r, lkb, 0);
grant_pending_locks(r);
goto out;
}
/* else fall through and move to convert queue */
}

if (can_be_queued(lkb)) {
error = -EINPROGRESS;
del_lkb(r, lkb);
add_lkb(r, lkb, DLM_LKSTS_CONVERT);
send_blocking_asts(r, lkb);
add_timeout(lkb);
goto out;
}

error = -EAGAIN;
if (force_blocking_asts(lkb))
send_blocking_asts_all(r, lkb);
queue_cast(r, lkb, -EAGAIN);

out:
return error;
}

static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
remove_lock(r, lkb);
queue_cast(r, lkb, -DLM_EUNLOCK);
grant_pending_locks(r);
return -DLM_EUNLOCK;
}

/* returns: 0 did nothing, -DLM_ECANCEL canceled lock */

static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

error = revert_lock(r, lkb);
if (error) {
queue_cast(r, lkb, -DLM_ECANCEL);
grant_pending_locks(r);
return -DLM_ECANCEL;
}
return 0;
}

/*
* Four stage 3 varieties:
* _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock()
*/

/* add a new lkb to a possibly new rsb, called by requesting process */

static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

/* set_master: sets lkb nodeid from r */

error = set_master(r, lkb);
if (error < 0)
goto out;
if (error) {
error = 0;
goto out;
}

if (is_remote(r))
/* receive_request() calls do_request() on remote node */
error = send_request(r, lkb);
else
error = do_request(r, lkb);
out:
return error;
}

/* change some property of an existing lkb, e.g. mode */

static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

if (is_remote(r))
/* receive_convert() calls do_convert() on remote node */
error = send_convert(r, lkb);
else
error = do_convert(r, lkb);

return error;
}

/* remove an existing lkb from the granted queue */

static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

if (is_remote(r))
/* receive_unlock() calls do_unlock() on remote node */
error = send_unlock(r, lkb);
else
error = do_unlock(r, lkb);

return error;
}

/* remove an existing lkb from the convert or wait queue */

static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

if (is_remote(r))
/* receive_cancel() calls do_cancel() on remote node */
error = send_cancel(r, lkb);
else
error = do_cancel(r, lkb);

return error;
}

/*
* Four stage 2 varieties:
* request_lock(), convert_lock(), unlock_lock(), cancel_lock()
*/

static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name,
int len, struct dlm_args *args)
{
struct dlm_rsb *r;
int error;

error = validate_lock_args(ls, lkb, args);
if (error)
goto out;

error = find_rsb(ls, name, len, R_CREATE, &r);
if (error)
goto out;

lock_rsb(r);

attach_lkb(r, lkb);
lkb->lkb_lksb->sb_lkid = lkb->lkb_id;

error = _request_lock(r, lkb);

unlock_rsb(r);
put_rsb(r);

out:
return error;
}

static int convert_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_args *args)
{
struct dlm_rsb *r;
int error;

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_lock_args(ls, lkb, args);
if (error)
goto out;

error = _convert_lock(r, lkb);
out:
unlock_rsb(r);
put_rsb(r);
return error;
}

static int unlock_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_args *args)
{
struct dlm_rsb *r;
int error;

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_unlock_args(lkb, args);
if (error)
goto out;

error = _unlock_lock(r, lkb);
out:
unlock_rsb(r);
put_rsb(r);
return error;
}

static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_args *args)
{
struct dlm_rsb *r;
int error;

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_unlock_args(lkb, args);
if (error)
goto out;

error = _cancel_lock(r, lkb);
out:
unlock_rsb(r);
put_rsb(r);
return error;
}

/*
* Two stage 1 varieties: dlm_lock() and dlm_unlock()
*/

int dlm_lock(dlm_lockspace_t *lockspace,
int mode,
struct dlm_lksb *lksb,
uint32_t flags,
void *name,
unsigned int namelen,
uint32_t parent_lkid,
void (*ast) (void *astarg),
void *astarg,
void (*bast) (void *astarg, int mode))
{
struct dlm_ls *ls;
struct dlm_lkb *lkb;
struct dlm_args args;
int error, convert = flags & DLM_LKF_CONVERT;

ls = dlm_find_lockspace_local(lockspace);
if (!ls)
return -EINVAL;

dlm_lock_recovery(ls);

if (convert)
error = find_lkb(ls, lksb->sb_lkid, &lkb);
else
error = create_lkb(ls, &lkb);

if (error)
goto out;

error = set_lock_args(mode, lksb, flags, namelen, 0, ast,
astarg, bast, &args);
if (error)
goto out_put;

if (convert)
error = convert_lock(ls, lkb, &args);
else
error = request_lock(ls, lkb, name, namelen, &args);

if (error == -EINPROGRESS)
error = 0;
out_put:
if (convert || error)
__put_lkb(ls, lkb);
if (error == -EAGAIN || error == -EDEADLK)
error = 0;
out:
dlm_unlock_recovery(ls);
dlm_put_lockspace(ls);
return error;
}

int dlm_unlock(dlm_lockspace_t *lockspace,
uint32_t lkid,
uint32_t flags,
struct dlm_lksb *lksb,
void *astarg)
{
struct dlm_ls *ls;
struct dlm_lkb *lkb;
struct dlm_args args;
int error;

ls = dlm_find_lockspace_local(lockspace);
if (!ls)
return -EINVAL;

dlm_lock_recovery(ls);

error = find_lkb(ls, lkid, &lkb);
if (error)
goto out;

error = set_unlock_args(flags, astarg, &args);
if (error)
goto out_put;

if (flags & DLM_LKF_CANCEL)
error = cancel_lock(ls, lkb, &args);
else
error = unlock_lock(ls, lkb, &args);

if (error == -DLM_EUNLOCK || error == -DLM_ECANCEL)
error = 0;
if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)))
error = 0;
out_put:
dlm_put_lkb(lkb);
out:
dlm_unlock_recovery(ls);
dlm_put_lockspace(ls);
return error;
}

/*
* send/receive routines for remote operations and replies
*
* send_args
* send_common
* send_request receive_request
* send_convert receive_convert
* send_unlock receive_unlock
* send_cancel receive_cancel
* send_grant receive_grant
* send_bast receive_bast
* send_lookup receive_lookup
* send_remove receive_remove
*
* send_common_reply
* receive_request_reply send_request_reply
* receive_convert_reply send_convert_reply
* receive_unlock_reply send_unlock_reply
* receive_cancel_reply send_cancel_reply
* receive_lookup_reply send_lookup_reply
*/

static int _create_message(struct dlm_ls *ls, int mb_len,
int to_nodeid, int mstype,
struct dlm_message **ms_ret,
struct dlm_mhandle **mh_ret)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
char *mb;

/* get_buffer gives us a message handle (mh) that we need to
pass into lowcomms_commit and a message buffer (mb) that we
write our data into */

mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, ls->ls_allocation, &mb);
if (!mh)
return -ENOBUFS;

memset(mb, 0, mb_len);

ms = (struct dlm_message *) mb;

ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
ms->m_header.h_lockspace = ls->ls_global_id;
ms->m_header.h_nodeid = dlm_our_nodeid();
ms->m_header.h_length = mb_len;
ms->m_header.h_cmd = DLM_MSG;

ms->m_type = mstype;

*mh_ret = mh;
*ms_ret = ms;
return 0;
}

static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb,
int to_nodeid, int mstype,
struct dlm_message **ms_ret,
struct dlm_mhandle **mh_ret)
{
int mb_len = sizeof(struct dlm_message);

switch (mstype) {
case DLM_MSG_REQUEST:
case DLM_MSG_LOOKUP:
case DLM_MSG_REMOVE:
mb_len += r->res_length;
break;
case DLM_MSG_CONVERT:
case DLM_MSG_UNLOCK:
case DLM_MSG_REQUEST_REPLY:
case DLM_MSG_CONVERT_REPLY:
case DLM_MSG_GRANT:
if (lkb && lkb->lkb_lvbptr)
mb_len += r->res_ls->ls_lvblen;
break;
}

return _create_message(r->res_ls, mb_len, to_nodeid, mstype,
ms_ret, mh_ret);
}

/* further lowcomms enhancements or alternate implementations may make
the return value from this function useful at some point */

static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms)
{
dlm_message_out(ms);
dlm_lowcomms_commit_buffer(mh);
return 0;
}

static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
ms->m_nodeid = lkb->lkb_nodeid;
ms->m_pid = lkb->lkb_ownpid;
ms->m_lkid = lkb->lkb_id;
ms->m_remid = lkb->lkb_remid;
ms->m_exflags = lkb->lkb_exflags;
ms->m_sbflags = lkb->lkb_sbflags;
ms->m_flags = lkb->lkb_flags;
ms->m_lvbseq = lkb->lkb_lvbseq;
ms->m_status = lkb->lkb_status;
ms->m_grmode = lkb->lkb_grmode;
ms->m_rqmode = lkb->lkb_rqmode;
ms->m_hash = r->res_hash;

/* m_result and m_bastmode are set from function args,
not from lkb fields */

if (lkb->lkb_bastfn)
ms->m_asts |= AST_BAST;
if (lkb->lkb_astfn)
ms->m_asts |= AST_COMP;

/* compare with switch in create_message; send_remove() doesn’t
use send_args() */

switch (ms->m_type) {
case DLM_MSG_REQUEST:
case DLM_MSG_LOOKUP:
memcpy(ms->m_extra, r->res_name, r->res_length);
break;
case DLM_MSG_CONVERT:
case DLM_MSG_UNLOCK:
case DLM_MSG_REQUEST_REPLY:
case DLM_MSG_CONVERT_REPLY:
case DLM_MSG_GRANT:
if (!lkb->lkb_lvbptr)
break;
memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen);
break;
}
}

static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

error = add_to_waiters(lkb, mstype);
if (error)
return error;

to_nodeid = r->res_nodeid;

error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh);
if (error)
goto fail;

send_args(r, lkb, ms);

error = send_message(mh, ms);
if (error)
goto fail;
return 0;

fail:
remove_from_waiters(lkb, msg_reply_type(mstype));
return error;
}

static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
return send_common(r, lkb, DLM_MSG_REQUEST);
}

static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
int error;

error = send_common(r, lkb, DLM_MSG_CONVERT);

/* down conversions go without a reply from the master */
if (!error && down_conversion(lkb)) {
remove_from_waiters(lkb, DLM_MSG_CONVERT_REPLY);
r->res_ls->ls_stub_ms.m_type = DLM_MSG_CONVERT_REPLY;
r->res_ls->ls_stub_ms.m_result = 0;
r->res_ls->ls_stub_ms.m_flags = lkb->lkb_flags;
__receive_convert_reply(r, lkb, &r->res_ls->ls_stub_ms);
}

return error;
}

/* FIXME: if this lkb is the only lock we hold on the rsb, then set
MASTER_UNCERTAIN to force the next request on the rsb to confirm
that the master is still correct. */

static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
return send_common(r, lkb, DLM_MSG_UNLOCK);
}

static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
return send_common(r, lkb, DLM_MSG_CANCEL);
}

static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

to_nodeid = lkb->lkb_nodeid;

error = create_message(r, lkb, to_nodeid, DLM_MSG_GRANT, &ms, &mh);
if (error)
goto out;

send_args(r, lkb, ms);

ms->m_result = 0;

error = send_message(mh, ms);
out:
return error;
}

static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

to_nodeid = lkb->lkb_nodeid;

error = create_message(r, NULL, to_nodeid, DLM_MSG_BAST, &ms, &mh);
if (error)
goto out;

send_args(r, lkb, ms);

ms->m_bastmode = mode;

error = send_message(mh, ms);
out:
return error;
}

static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

error = add_to_waiters(lkb, DLM_MSG_LOOKUP);
if (error)
return error;

to_nodeid = dlm_dir_nodeid(r);

error = create_message(r, NULL, to_nodeid, DLM_MSG_LOOKUP, &ms, &mh);
if (error)
goto fail;

send_args(r, lkb, ms);

error = send_message(mh, ms);
if (error)
goto fail;
return 0;

fail:
remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY);
return error;
}

static int send_remove(struct dlm_rsb *r)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

to_nodeid = dlm_dir_nodeid(r);

error = create_message(r, NULL, to_nodeid, DLM_MSG_REMOVE, &ms, &mh);
if (error)
goto out;

memcpy(ms->m_extra, r->res_name, r->res_length);
ms->m_hash = r->res_hash;

error = send_message(mh, ms);
out:
return error;
}

static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
int mstype, int rv)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int to_nodeid, error;

to_nodeid = lkb->lkb_nodeid;

error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh);
if (error)
goto out;

send_args(r, lkb, ms);

ms->m_result = rv;

error = send_message(mh, ms);
out:
return error;
}

static int send_request_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
{
return send_common_reply(r, lkb, DLM_MSG_REQUEST_REPLY, rv);
}

static int send_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
{
return send_common_reply(r, lkb, DLM_MSG_CONVERT_REPLY, rv);
}

static int send_unlock_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
{
return send_common_reply(r, lkb, DLM_MSG_UNLOCK_REPLY, rv);
}

static int send_cancel_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
{
return send_common_reply(r, lkb, DLM_MSG_CANCEL_REPLY, rv);
}

static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in,
int ret_nodeid, int rv)
{
struct dlm_rsb *r = &ls->ls_stub_rsb;
struct dlm_message *ms;
struct dlm_mhandle *mh;
int error, nodeid = ms_in->m_header.h_nodeid;

error = create_message(r, NULL, nodeid, DLM_MSG_LOOKUP_REPLY, &ms, &mh);
if (error)
goto out;

ms->m_lkid = ms_in->m_lkid;
ms->m_result = rv;
ms->m_nodeid = ret_nodeid;

error = send_message(mh, ms);
out:
return error;
}

/* which args we save from a received message depends heavily on the type
of message, unlike the send side where we can safely send everything about
the lkb for any type of message */

static void receive_flags(struct dlm_lkb *lkb, struct dlm_message *ms)
{
lkb->lkb_exflags = ms->m_exflags;
lkb->lkb_sbflags = ms->m_sbflags;
lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) |
(ms->m_flags & 0×0000FFFF);
}

static void receive_flags_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
{
lkb->lkb_sbflags = ms->m_sbflags;
lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) |
(ms->m_flags & 0×0000FFFF);
}

static int receive_extralen(struct dlm_message *ms)
{
return (ms->m_header.h_length – sizeof(struct dlm_message));
}

static int receive_lvb(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
int len;

if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
if (!lkb->lkb_lvbptr)
lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
if (!lkb->lkb_lvbptr)
return -ENOMEM;
len = receive_extralen(ms);
if (len > DLM_RESNAME_MAXLEN)
len = DLM_RESNAME_MAXLEN;
memcpy(lkb->lkb_lvbptr, ms->m_extra, len);
}
return 0;
}

static void fake_bastfn(void *astparam, int mode)
{
log_print(»fake_bastfn should not be called»);
}

static void fake_astfn(void *astparam)
{
log_print(»fake_astfn should not be called»);
}

static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
lkb->lkb_nodeid = ms->m_header.h_nodeid;
lkb->lkb_ownpid = ms->m_pid;
lkb->lkb_remid = ms->m_lkid;
lkb->lkb_grmode = DLM_LOCK_IV;
lkb->lkb_rqmode = ms->m_rqmode;

lkb->lkb_bastfn = (ms->m_asts & AST_BAST) ? &fake_bastfn : NULL;
lkb->lkb_astfn = (ms->m_asts & AST_COMP) ? &fake_astfn : NULL;

if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
/* lkb was just created so there won’t be an lvb yet */
lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
if (!lkb->lkb_lvbptr)
return -ENOMEM;
}

return 0;
}

static int receive_convert_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
if (lkb->lkb_status != DLM_LKSTS_GRANTED)
return -EBUSY;

if (receive_lvb(ls, lkb, ms))
return -ENOMEM;

lkb->lkb_rqmode = ms->m_rqmode;
lkb->lkb_lvbseq = ms->m_lvbseq;

return 0;
}

static int receive_unlock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
if (receive_lvb(ls, lkb, ms))
return -ENOMEM;
return 0;
}

/* We fill in the stub-lkb fields with the info that send_xxxx_reply()
uses to send a reply and that the remote end uses to process the reply. */

static void setup_stub_lkb(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb = &ls->ls_stub_lkb;
lkb->lkb_nodeid = ms->m_header.h_nodeid;
lkb->lkb_remid = ms->m_lkid;
}

/* This is called after the rsb is locked so that we can safely inspect
fields in the lkb. */

static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms)
{
int from = ms->m_header.h_nodeid;
int error = 0;

switch (ms->m_type) {
case DLM_MSG_CONVERT:
case DLM_MSG_UNLOCK:
case DLM_MSG_CANCEL:
if (!is_master_copy(lkb) || lkb->lkb_nodeid != from)
error = -EINVAL;
break;

case DLM_MSG_CONVERT_REPLY:
case DLM_MSG_UNLOCK_REPLY:
case DLM_MSG_CANCEL_REPLY:
case DLM_MSG_GRANT:
case DLM_MSG_BAST:
if (!is_process_copy(lkb) || lkb->lkb_nodeid != from)
error = -EINVAL;
break;

case DLM_MSG_REQUEST_REPLY:
if (!is_process_copy(lkb))
error = -EINVAL;
else if (lkb->lkb_nodeid != -1 && lkb->lkb_nodeid != from)
error = -EINVAL;
break;

default:
error = -EINVAL;
}

if (error)
log_error(lkb->lkb_resource->res_ls,
«ignore invalid message %d from %d %x %x %x %d»,
ms->m_type, from, lkb->lkb_id, lkb->lkb_remid,
lkb->lkb_flags, lkb->lkb_nodeid);
return error;
}

static void receive_request(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error, namelen;

error = create_lkb(ls, &lkb);
if (error)
goto fail;

receive_flags(lkb, ms);
lkb->lkb_flags |= DLM_IFL_MSTCPY;
error = receive_request_args(ls, lkb, ms);
if (error) {
__put_lkb(ls, lkb);
goto fail;
}

namelen = receive_extralen(ms);

error = find_rsb(ls, ms->m_extra, namelen, R_MASTER, &r);
if (error) {
__put_lkb(ls, lkb);
goto fail;
}

lock_rsb(r);

attach_lkb(r, lkb);
error = do_request(r, lkb);
send_request_reply(r, lkb, error);

unlock_rsb(r);
put_rsb(r);

if (error == -EINPROGRESS)
error = 0;
if (error)
dlm_put_lkb(lkb);
return;

fail:
setup_stub_lkb(ls, ms);
send_request_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
}

static void receive_convert(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error, reply = 1;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error)
goto fail;

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

receive_flags(lkb, ms);
error = receive_convert_args(ls, lkb, ms);
if (error)
goto out_reply;
reply = !down_conversion(lkb);

error = do_convert(r, lkb);
out_reply:
if (reply)
send_convert_reply(r, lkb, error);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
return;

fail:
setup_stub_lkb(ls, ms);
send_convert_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
}

static void receive_unlock(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error)
goto fail;

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

receive_flags(lkb, ms);
error = receive_unlock_args(ls, lkb, ms);
if (error)
goto out_reply;

error = do_unlock(r, lkb);
out_reply:
send_unlock_reply(r, lkb, error);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
return;

fail:
setup_stub_lkb(ls, ms);
send_unlock_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
}

static void receive_cancel(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error)
goto fail;

receive_flags(lkb, ms);

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

error = do_cancel(r, lkb);
send_cancel_reply(r, lkb, error);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
return;

fail:
setup_stub_lkb(ls, ms);
send_cancel_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
}

static void receive_grant(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_grant from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

receive_flags_reply(lkb, ms);
if (is_altmode(lkb))
munge_altmode(lkb, ms);
grant_lock_pc(r, lkb, ms);
queue_cast(r, lkb, 0);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
}

static void receive_bast(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_bast from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

r = lkb->lkb_resource;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

queue_bast(r, lkb, ms->m_bastmode);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
}

static void receive_lookup(struct dlm_ls *ls, struct dlm_message *ms)
{
int len, error, ret_nodeid, dir_nodeid, from_nodeid, our_nodeid;

from_nodeid = ms->m_header.h_nodeid;
our_nodeid = dlm_our_nodeid();

len = receive_extralen(ms);

dir_nodeid = dlm_hash2nodeid(ls, ms->m_hash);
if (dir_nodeid != our_nodeid) {
log_error(ls, «lookup dir_nodeid %d from %d»,
dir_nodeid, from_nodeid);
error = -EINVAL;
ret_nodeid = -1;
goto out;
}

error = dlm_dir_lookup(ls, from_nodeid, ms->m_extra, len, &ret_nodeid);

/* Optimization: we’re master so treat lookup as a request */
if (!error && ret_nodeid == our_nodeid) {
receive_request(ls, ms);
return;
}
out:
send_lookup_reply(ls, ms, ret_nodeid, error);
}

static void receive_remove(struct dlm_ls *ls, struct dlm_message *ms)
{
int len, dir_nodeid, from_nodeid;

from_nodeid = ms->m_header.h_nodeid;

len = receive_extralen(ms);

dir_nodeid = dlm_hash2nodeid(ls, ms->m_hash);
if (dir_nodeid != dlm_our_nodeid()) {
log_error(ls, «remove dir entry dir_nodeid %d from %d»,
dir_nodeid, from_nodeid);
return;
}

dlm_dir_remove_entry(ls, from_nodeid, ms->m_extra, len);
}

static void receive_purge(struct dlm_ls *ls, struct dlm_message *ms)
{
do_purge(ls, ms->m_nodeid, ms->m_pid);
}

static void receive_request_reply(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error, mstype, result;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_request_reply from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

mstype = lkb->lkb_wait_type;
error = remove_from_waiters(lkb, DLM_MSG_REQUEST_REPLY);
if (error)
goto out;

/* Optimization: the dir node was also the master, so it took our
lookup as a request and sent request reply instead of lookup reply */
if (mstype == DLM_MSG_LOOKUP) {
r->res_nodeid = ms->m_header.h_nodeid;
lkb->lkb_nodeid = r->res_nodeid;
}

/* this is the value returned from do_request() on the master */
result = ms->m_result;

switch (result) {
case -EAGAIN:
/* request would block (be queued) on remote master */
queue_cast(r, lkb, -EAGAIN);
confirm_master(r, -EAGAIN);
unhold_lkb(lkb); /* undoes create_lkb() */
break;

case -EINPROGRESS:
case 0:
/* request was queued or granted on remote master */
receive_flags_reply(lkb, ms);
lkb->lkb_remid = ms->m_lkid;
if (is_altmode(lkb))
munge_altmode(lkb, ms);
if (result) {
add_lkb(r, lkb, DLM_LKSTS_WAITING);
add_timeout(lkb);
} else {
grant_lock_pc(r, lkb, ms);
queue_cast(r, lkb, 0);
}
confirm_master(r, result);
break;

case -EBADR:
case -ENOTBLK:
/* find_rsb failed to find rsb or rsb wasn’t master */
log_debug(ls, «receive_request_reply %x %x master diff %d %d»,
lkb->lkb_id, lkb->lkb_flags, r->res_nodeid, result);
r->res_nodeid = -1;
lkb->lkb_nodeid = -1;

if (is_overlap(lkb)) {
/* we’ll ignore error in cancel/unlock reply */
queue_cast_overlap(r, lkb);
confirm_master(r, result);
unhold_lkb(lkb); /* undoes create_lkb() */
} else
_request_lock(r, lkb);
break;

default:
log_error(ls, «receive_request_reply %x error %d»,
lkb->lkb_id, result);
}

if (is_overlap_unlock(lkb) && (result == 0 || result == -EINPROGRESS)) {
log_debug(ls, «receive_request_reply %x result %d unlock»,
lkb->lkb_id, result);
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
send_unlock(r, lkb);
} else if (is_overlap_cancel(lkb) && (result == -EINPROGRESS)) {
log_debug(ls, «receive_request_reply %x cancel», lkb->lkb_id);
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
send_cancel(r, lkb);
} else {
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
}
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
}

static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
struct dlm_message *ms)
{
/* this is the value returned from do_convert() on the master */
switch (ms->m_result) {
case -EAGAIN:
/* convert would block (be queued) on remote master */
queue_cast(r, lkb, -EAGAIN);
break;

case -EDEADLK:
receive_flags_reply(lkb, ms);
revert_lock_pc(r, lkb);
queue_cast(r, lkb, -EDEADLK);
break;

case -EINPROGRESS:
/* convert was queued on remote master */
receive_flags_reply(lkb, ms);
if (is_demoted(lkb))
munge_demoted(lkb, ms);
del_lkb(r, lkb);
add_lkb(r, lkb, DLM_LKSTS_CONVERT);
add_timeout(lkb);
break;

case 0:
/* convert was granted on remote master */
receive_flags_reply(lkb, ms);
if (is_demoted(lkb))
munge_demoted(lkb, ms);
grant_lock_pc(r, lkb, ms);
queue_cast(r, lkb, 0);
break;

default:
log_error(r->res_ls, «receive_convert_reply %x error %d»,
lkb->lkb_id, ms->m_result);
}
}

static void _receive_convert_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
{
struct dlm_rsb *r = lkb->lkb_resource;
int error;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

/* stub reply can happen with waiters_mutex held */
error = remove_from_waiters_ms(lkb, ms);
if (error)
goto out;

__receive_convert_reply(r, lkb, ms);
out:
unlock_rsb(r);
put_rsb(r);
}

static void receive_convert_reply(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_convert_reply from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

_receive_convert_reply(lkb, ms);
dlm_put_lkb(lkb);
}

static void _receive_unlock_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
{
struct dlm_rsb *r = lkb->lkb_resource;
int error;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

/* stub reply can happen with waiters_mutex held */
error = remove_from_waiters_ms(lkb, ms);
if (error)
goto out;

/* this is the value returned from do_unlock() on the master */

switch (ms->m_result) {
case -DLM_EUNLOCK:
receive_flags_reply(lkb, ms);
remove_lock_pc(r, lkb);
queue_cast(r, lkb, -DLM_EUNLOCK);
break;
case -ENOENT:
break;
default:
log_error(r->res_ls, «receive_unlock_reply %x error %d»,
lkb->lkb_id, ms->m_result);
}
out:
unlock_rsb(r);
put_rsb(r);
}

static void receive_unlock_reply(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_unlock_reply from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

_receive_unlock_reply(lkb, ms);
dlm_put_lkb(lkb);
}

static void _receive_cancel_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
{
struct dlm_rsb *r = lkb->lkb_resource;
int error;

hold_rsb(r);
lock_rsb(r);

error = validate_message(lkb, ms);
if (error)
goto out;

/* stub reply can happen with waiters_mutex held */
error = remove_from_waiters_ms(lkb, ms);
if (error)
goto out;

/* this is the value returned from do_cancel() on the master */

switch (ms->m_result) {
case -DLM_ECANCEL:
receive_flags_reply(lkb, ms);
revert_lock_pc(r, lkb);
queue_cast(r, lkb, -DLM_ECANCEL);
break;
case 0:
break;
default:
log_error(r->res_ls, «receive_cancel_reply %x error %d»,
lkb->lkb_id, ms->m_result);
}
out:
unlock_rsb(r);
put_rsb(r);
}

static void receive_cancel_reply(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
int error;

error = find_lkb(ls, ms->m_remid, &lkb);
if (error) {
log_debug(ls, «receive_cancel_reply from %d no lkb %x»,
ms->m_header.h_nodeid, ms->m_remid);
return;
}

_receive_cancel_reply(lkb, ms);
dlm_put_lkb(lkb);
}

static void receive_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error, ret_nodeid;

error = find_lkb(ls, ms->m_lkid, &lkb);
if (error) {
log_error(ls, «receive_lookup_reply no lkb»);
return;
}

/* ms->m_result is the value returned by dlm_dir_lookup on dir node
FIXME: will a non-zero error ever be returned? */

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

error = remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY);
if (error)
goto out;

ret_nodeid = ms->m_nodeid;
if (ret_nodeid == dlm_our_nodeid()) {
r->res_nodeid = 0;
ret_nodeid = 0;
r->res_first_lkid = 0;
} else {
/* set_master() will copy res_nodeid to lkb_nodeid */
r->res_nodeid = ret_nodeid;
}

if (is_overlap(lkb)) {
log_debug(ls, «receive_lookup_reply %x unlock %x»,
lkb->lkb_id, lkb->lkb_flags);
queue_cast_overlap(r, lkb);
unhold_lkb(lkb); /* undoes create_lkb() */
goto out_list;
}

_request_lock(r, lkb);

out_list:
if (!ret_nodeid)
process_lookup_list(r);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
}

static void _receive_message(struct dlm_ls *ls, struct dlm_message *ms)
{
if (!dlm_is_member(ls, ms->m_header.h_nodeid)) {
log_debug(ls, «ignore non-member message %d from %d %x %x %d»,
ms->m_type, ms->m_header.h_nodeid, ms->m_lkid,
ms->m_remid, ms->m_result);
return;
}

switch (ms->m_type) {

/* messages sent to a master node */

case DLM_MSG_REQUEST:
receive_request(ls, ms);
break;

case DLM_MSG_CONVERT:
receive_convert(ls, ms);
break;

case DLM_MSG_UNLOCK:
receive_unlock(ls, ms);
break;

case DLM_MSG_CANCEL:
receive_cancel(ls, ms);
break;

/* messages sent from a master node (replies to above) */

case DLM_MSG_REQUEST_REPLY:
receive_request_reply(ls, ms);
break;

case DLM_MSG_CONVERT_REPLY:
receive_convert_reply(ls, ms);
break;

case DLM_MSG_UNLOCK_REPLY:
receive_unlock_reply(ls, ms);
break;

case DLM_MSG_CANCEL_REPLY:
receive_cancel_reply(ls, ms);
break;

/* messages sent from a master node (only two types of async msg) */

case DLM_MSG_GRANT:
receive_grant(ls, ms);
break;

case DLM_MSG_BAST:
receive_bast(ls, ms);
break;

/* messages sent to a dir node */

case DLM_MSG_LOOKUP:
receive_lookup(ls, ms);
break;

case DLM_MSG_REMOVE:
receive_remove(ls, ms);
break;

/* messages sent from a dir node (remove has no reply) */

case DLM_MSG_LOOKUP_REPLY:
receive_lookup_reply(ls, ms);
break;

/* other messages */

case DLM_MSG_PURGE:
receive_purge(ls, ms);
break;

default:
log_error(ls, «unknown message type %d», ms->m_type);
}

dlm_astd_wake();
}

/* If the lockspace is in recovery mode (locking stopped), then normal
messages are saved on the requestqueue for processing after recovery is
done. When not in recovery mode, we wait for dlm_recoverd to drain saved
messages off the requestqueue before we process new ones. This occurs right
after recovery completes when we transition from saving all messages on
requestqueue, to processing all the saved messages, to processing new
messages as they arrive. */

static void dlm_receive_message(struct dlm_ls *ls, struct dlm_message *ms,
int nodeid)
{
if (dlm_locking_stopped(ls)) {
dlm_add_requestqueue(ls, nodeid, ms);
} else {
dlm_wait_requestqueue(ls);
_receive_message(ls, ms);
}
}

/* This is called by dlm_recoverd to process messages that were saved on
the requestqueue. */

void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms)
{
_receive_message(ls, ms);
}

/* This is called by the midcomms layer when something is received for
the lockspace. It could be either a MSG (normal message sent as part of
standard locking activity) or an RCOM (recovery message sent as part of
lockspace recovery). */

void dlm_receive_buffer(union dlm_packet *p, int nodeid)
{
struct dlm_header *hd = &p->header;
struct dlm_ls *ls;
int type = 0;

switch (hd->h_cmd) {
case DLM_MSG:
dlm_message_in(&p->message);
type = p->message.m_type;
break;
case DLM_RCOM:
dlm_rcom_in(&p->rcom);
type = p->rcom.rc_type;
break;
default:
log_print(»invalid h_cmd %d from %u», hd->h_cmd, nodeid);
return;
}

if (hd->h_nodeid != nodeid) {
log_print(»invalid h_nodeid %d from %d lockspace %x»,
hd->h_nodeid, nodeid, hd->h_lockspace);
return;
}

ls = dlm_find_lockspace_global(hd->h_lockspace);
if (!ls) {
if (dlm_config.ci_log_debug)
log_print(»invalid lockspace %x from %d cmd %d type %d»,
hd->h_lockspace, nodeid, hd->h_cmd, type);

if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS)
dlm_send_ls_not_ready(nodeid, &p->rcom);
return;
}

/* this rwsem allows dlm_ls_stop() to wait for all dlm_recv threads to
be inactive (in this ls) before transitioning to recovery mode */

down_read(&ls->ls_recv_active);
if (hd->h_cmd == DLM_MSG)
dlm_receive_message(ls, &p->message, nodeid);
else
dlm_receive_rcom(ls, &p->rcom, nodeid);
up_read(&ls->ls_recv_active);

dlm_put_lockspace(ls);
}

static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
if (middle_conversion(lkb)) {
hold_lkb(lkb);
ls->ls_stub_ms.m_type = DLM_MSG_CONVERT_REPLY;
ls->ls_stub_ms.m_result = -EINPROGRESS;
ls->ls_stub_ms.m_flags = lkb->lkb_flags;
ls->ls_stub_ms.m_header.h_nodeid = lkb->lkb_nodeid;
_receive_convert_reply(lkb, &ls->ls_stub_ms);

/* Same special case as in receive_rcom_lock_args() */
lkb->lkb_grmode = DLM_LOCK_IV;
rsb_set_flag(lkb->lkb_resource, RSB_RECOVER_CONVERT);
unhold_lkb(lkb);

} else if (lkb->lkb_rqmode >= lkb->lkb_grmode) {
lkb->lkb_flags |= DLM_IFL_RESEND;
}

/* lkb->lkb_rqmode < lkb->lkb_grmode shouldn’t happen since down
conversions are async; there’s no reply from the remote master */
}

/* A waiting lkb needs recovery if the master node has failed, or
the master node is changing (only when no directory is used) */

static int waiter_needs_recovery(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
if (dlm_is_removed(ls, lkb->lkb_nodeid))
return 1;

if (!dlm_no_directory(ls))
return 0;

if (dlm_dir_nodeid(lkb->lkb_resource) != lkb->lkb_nodeid)
return 1;

return 0;
}

/* Recovery for locks that are waiting for replies from nodes that are now
gone. We can just complete unlocks and cancels by faking a reply from the
dead node. Requests and up-conversions we flag to be resent after
recovery. Down-conversions can just be completed with a fake reply like
unlocks. Conversions between PR and CW need special attention. */

void dlm_recover_waiters_pre(struct dlm_ls *ls)
{
struct dlm_lkb *lkb, *safe;
int wait_type, stub_unlock_result, stub_cancel_result;

mutex_lock(&ls->ls_waiters_mutex);

list_for_each_entry_safe(lkb, safe, &ls->ls_waiters, lkb_wait_reply) {
log_debug(ls, «pre recover waiter lkid %x type %d flags %x»,
lkb->lkb_id, lkb->lkb_wait_type, lkb->lkb_flags);

/* all outstanding lookups, regardless of destination will be
resent after recovery is done */

if (lkb->lkb_wait_type == DLM_MSG_LOOKUP) {
lkb->lkb_flags |= DLM_IFL_RESEND;
continue;
}

if (!waiter_needs_recovery(ls, lkb))
continue;

wait_type = lkb->lkb_wait_type;
stub_unlock_result = -DLM_EUNLOCK;
stub_cancel_result = -DLM_ECANCEL;

/* Main reply may have been received leaving a zero wait_type,
but a reply for the overlapping op may not have been
received. In that case we need to fake the appropriate
reply for the overlap op. */

if (!wait_type) {
if (is_overlap_cancel(lkb)) {
wait_type = DLM_MSG_CANCEL;
if (lkb->lkb_grmode == DLM_LOCK_IV)
stub_cancel_result = 0;
}
if (is_overlap_unlock(lkb)) {
wait_type = DLM_MSG_UNLOCK;
if (lkb->lkb_grmode == DLM_LOCK_IV)
stub_unlock_result = -ENOENT;
}

log_debug(ls, «rwpre overlap %x %x %d %d %d»,
lkb->lkb_id, lkb->lkb_flags, wait_type,
stub_cancel_result, stub_unlock_result);
}

switch (wait_type) {

case DLM_MSG_REQUEST:
lkb->lkb_flags |= DLM_IFL_RESEND;
break;

case DLM_MSG_CONVERT:
recover_convert_waiter(ls, lkb);
break;

case DLM_MSG_UNLOCK:
hold_lkb(lkb);
ls->ls_stub_ms.m_type = DLM_MSG_UNLOCK_REPLY;
ls->ls_stub_ms.m_result = stub_unlock_result;
ls->ls_stub_ms.m_flags = lkb->lkb_flags;
ls->ls_stub_ms.m_header.h_nodeid = lkb->lkb_nodeid;
_receive_unlock_reply(lkb, &ls->ls_stub_ms);
dlm_put_lkb(lkb);
break;

case DLM_MSG_CANCEL:
hold_lkb(lkb);
ls->ls_stub_ms.m_type = DLM_MSG_CANCEL_REPLY;
ls->ls_stub_ms.m_result = stub_cancel_result;
ls->ls_stub_ms.m_flags = lkb->lkb_flags;
ls->ls_stub_ms.m_header.h_nodeid = lkb->lkb_nodeid;
_receive_cancel_reply(lkb, &ls->ls_stub_ms);
dlm_put_lkb(lkb);
break;

default:
log_error(ls, «invalid lkb wait_type %d %d»,
lkb->lkb_wait_type, wait_type);
}
schedule();
}
mutex_unlock(&ls->ls_waiters_mutex);
}

static struct dlm_lkb *find_resend_waiter(struct dlm_ls *ls)
{
struct dlm_lkb *lkb;
int found = 0;

mutex_lock(&ls->ls_waiters_mutex);
list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) {
if (lkb->lkb_flags & DLM_IFL_RESEND) {
hold_lkb(lkb);
found = 1;
break;
}
}
mutex_unlock(&ls->ls_waiters_mutex);

if (!found)
lkb = NULL;
return lkb;
}

/* Deal with lookups and lkb’s marked RESEND from _pre. We may now be the
master or dir-node for r. Processing the lkb may result in it being placed
back on waiters. */

/* We do this after normal locking has been enabled and any saved messages
(in requestqueue) have been processed. We should be confident that at
this point we won’t get or process a reply to any of these waiting
operations. But, new ops may be coming in on the rsbs/locks here from
userspace or remotely. */

/* there may have been an overlap unlock/cancel prior to recovery or after
recovery. if before, the lkb may still have a pos wait_count; if after, the
overlap flag would just have been set and nothing new sent. we can be
confident here than any replies to either the initial op or overlap ops
prior to recovery have been received. */

int dlm_recover_waiters_post(struct dlm_ls *ls)
{
struct dlm_lkb *lkb;
struct dlm_rsb *r;
int error = 0, mstype, err, oc, ou;

while (1) {
if (dlm_locking_stopped(ls)) {
log_debug(ls, «recover_waiters_post aborted»);
error = -EINTR;
break;
}

lkb = find_resend_waiter(ls);
if (!lkb)
break;

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

mstype = lkb->lkb_wait_type;
oc = is_overlap_cancel(lkb);
ou = is_overlap_unlock(lkb);
err = 0;

log_debug(ls, «recover_waiters_post %x type %d flags %x %s»,
lkb->lkb_id, mstype, lkb->lkb_flags, r->res_name);

/* At this point we assume that we won’t get a reply to any
previous op or overlap op on this lock. First, do a big
remove_from_waiters() for all previous ops. */

lkb->lkb_flags &= ~DLM_IFL_RESEND;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
lkb->lkb_wait_type = 0;
lkb->lkb_wait_count = 0;
mutex_lock(&ls->ls_waiters_mutex);
list_del_init(&lkb->lkb_wait_reply);
mutex_unlock(&ls->ls_waiters_mutex);
unhold_lkb(lkb); /* for waiters list */

if (oc || ou) {
/* do an unlock or cancel instead of resending */
switch (mstype) {
case DLM_MSG_LOOKUP:
case DLM_MSG_REQUEST:
queue_cast(r, lkb, ou ? -DLM_EUNLOCK :
-DLM_ECANCEL);
unhold_lkb(lkb); /* undoes create_lkb() */
break;
case DLM_MSG_CONVERT:
if (oc) {
queue_cast(r, lkb, -DLM_ECANCEL);
} else {
lkb->lkb_exflags |= DLM_LKF_FORCEUNLOCK;
_unlock_lock(r, lkb);
}
break;
default:
err = 1;
}
} else {
switch (mstype) {
case DLM_MSG_LOOKUP:
case DLM_MSG_REQUEST:
_request_lock(r, lkb);
if (is_master(r))
confirm_master(r, 0);
break;
case DLM_MSG_CONVERT:
_convert_lock(r, lkb);
break;
default:
err = 1;
}
}

if (err)
log_error(ls, «recover_waiters_post %x %d %x %d %d»,
lkb->lkb_id, mstype, lkb->lkb_flags, oc, ou);
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);
}

return error;
}

static void purge_queue(struct dlm_rsb *r, struct list_head *queue,
int (*test)(struct dlm_ls *ls, struct dlm_lkb *lkb))
{
struct dlm_ls *ls = r->res_ls;
struct dlm_lkb *lkb, *safe;

list_for_each_entry_safe(lkb, safe, queue, lkb_statequeue) {
if (test(ls, lkb)) {
rsb_set_flag(r, RSB_LOCKS_PURGED);
del_lkb(r, lkb);
/* this put should free the lkb */
if (!dlm_put_lkb(lkb))
log_error(ls, «purged lkb not released»);
}
}
}

static int purge_dead_test(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
return (is_master_copy(lkb) && dlm_is_removed(ls, lkb->lkb_nodeid));
}

static int purge_mstcpy_test(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
return is_master_copy(lkb);
}

static void purge_dead_locks(struct dlm_rsb *r)
{
purge_queue(r, &r->res_grantqueue, &purge_dead_test);
purge_queue(r, &r->res_convertqueue, &purge_dead_test);
purge_queue(r, &r->res_waitqueue, &purge_dead_test);
}

void dlm_purge_mstcpy_locks(struct dlm_rsb *r)
{
purge_queue(r, &r->res_grantqueue, &purge_mstcpy_test);
purge_queue(r, &r->res_convertqueue, &purge_mstcpy_test);
purge_queue(r, &r->res_waitqueue, &purge_mstcpy_test);
}

/* Get rid of locks held by nodes that are gone. */

int dlm_purge_locks(struct dlm_ls *ls)
{
struct dlm_rsb *r;

log_debug(ls, «dlm_purge_locks»);

down_write(&ls->ls_root_sem);
list_for_each_entry(r, &ls->ls_root_list, res_root_list) {
hold_rsb(r);
lock_rsb(r);
if (is_master(r))
purge_dead_locks(r);
unlock_rsb(r);
unhold_rsb(r);

schedule();
}
up_write(&ls->ls_root_sem);

return 0;
}

static struct dlm_rsb *find_purged_rsb(struct dlm_ls *ls, int bucket)
{
struct dlm_rsb *r, *r_ret = NULL;

spin_lock(&ls->ls_rsbtbl[bucket].lock);
list_for_each_entry(r, &ls->ls_rsbtbl[bucket].list, res_hashchain) {
if (!rsb_flag(r, RSB_LOCKS_PURGED))
continue;
hold_rsb(r);
rsb_clear_flag(r, RSB_LOCKS_PURGED);
r_ret = r;
break;
}
spin_unlock(&ls->ls_rsbtbl[bucket].lock);
return r_ret;
}

void dlm_grant_after_purge(struct dlm_ls *ls)
{
struct dlm_rsb *r;
int bucket = 0;

while (1) {
r = find_purged_rsb(ls, bucket);
if (!r) {
if (bucket == ls->ls_rsbtbl_size – 1)
break;
bucket++;
continue;
}
lock_rsb(r);
if (is_master(r)) {
grant_pending_locks(r);
confirm_master(r, 0);
}
unlock_rsb(r);
put_rsb(r);
schedule();
}
}

static struct dlm_lkb *search_remid_list(struct list_head *head, int nodeid,
uint32_t remid)
{
struct dlm_lkb *lkb;

list_for_each_entry(lkb, head, lkb_statequeue) {
if (lkb->lkb_nodeid == nodeid && lkb->lkb_remid == remid)
return lkb;
}
return NULL;
}

static struct dlm_lkb *search_remid(struct dlm_rsb *r, int nodeid,
uint32_t remid)
{
struct dlm_lkb *lkb;

lkb = search_remid_list(&r->res_grantqueue, nodeid, remid);
if (lkb)
return lkb;
lkb = search_remid_list(&r->res_convertqueue, nodeid, remid);
if (lkb)
return lkb;
lkb = search_remid_list(&r->res_waitqueue, nodeid, remid);
if (lkb)
return lkb;
return NULL;
}

/* needs at least dlm_rcom + rcom_lock */
static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
struct dlm_rsb *r, struct dlm_rcom *rc)
{
struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf;

lkb->lkb_nodeid = rc->rc_header.h_nodeid;
lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid);
lkb->lkb_remid = le32_to_cpu(rl->rl_lkid);
lkb->lkb_exflags = le32_to_cpu(rl->rl_exflags);
lkb->lkb_flags = le32_to_cpu(rl->rl_flags) & 0×0000FFFF;
lkb->lkb_flags |= DLM_IFL_MSTCPY;
lkb->lkb_lvbseq = le32_to_cpu(rl->rl_lvbseq);
lkb->lkb_rqmode = rl->rl_rqmode;
lkb->lkb_grmode = rl->rl_grmode;
/* don’t set lkb_status because add_lkb wants to itself */

lkb->lkb_bastfn = (rl->rl_asts & AST_BAST) ? &fake_bastfn : NULL;
lkb->lkb_astfn = (rl->rl_asts & AST_COMP) ? &fake_astfn : NULL;

if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
int lvblen = rc->rc_header.h_length – sizeof(struct dlm_rcom) -
sizeof(struct rcom_lock);
if (lvblen > ls->ls_lvblen)
return -EINVAL;
lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
if (!lkb->lkb_lvbptr)
return -ENOMEM;
memcpy(lkb->lkb_lvbptr, rl->rl_lvb, lvblen);
}

/* Conversions between PR and CW (middle modes) need special handling.
The real granted mode of these converting locks cannot be determined
until all locks have been rebuilt on the rsb (recover_conversion) */

if (rl->rl_wait_type == cpu_to_le16(DLM_MSG_CONVERT) &&
middle_conversion(lkb)) {
rl->rl_status = DLM_LKSTS_CONVERT;
lkb->lkb_grmode = DLM_LOCK_IV;
rsb_set_flag(r, RSB_RECOVER_CONVERT);
}

return 0;
}

/* This lkb may have been recovered in a previous aborted recovery so we need
to check if the rsb already has an lkb with the given remote nodeid/lkid.
If so we just send back a standard reply. If not, we create a new lkb with
the given values and send back our lkid. We send back our lkid by sending
back the rcom_lock struct we got but with the remid field filled in. */

/* needs at least dlm_rcom + rcom_lock */
int dlm_recover_master_copy(struct dlm_ls *ls, struct dlm_rcom *rc)
{
struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf;
struct dlm_rsb *r;
struct dlm_lkb *lkb;
int error;

if (rl->rl_parent_lkid) {
error = -EOPNOTSUPP;
goto out;
}

error = find_rsb(ls, rl->rl_name, le16_to_cpu(rl->rl_namelen),
R_MASTER, &r);
if (error)
goto out;

lock_rsb(r);

lkb = search_remid(r, rc->rc_header.h_nodeid, le32_to_cpu(rl->rl_lkid));
if (lkb) {
error = -EEXIST;
goto out_remid;
}

error = create_lkb(ls, &lkb);
if (error)
goto out_unlock;

error = receive_rcom_lock_args(ls, lkb, r, rc);
if (error) {
__put_lkb(ls, lkb);
goto out_unlock;
}

attach_lkb(r, lkb);
add_lkb(r, lkb, rl->rl_status);
error = 0;

out_remid:
/* this is the new value returned to the lock holder for
saving in its process-copy lkb */
rl->rl_remid = cpu_to_le32(lkb->lkb_id);

out_unlock:
unlock_rsb(r);
put_rsb(r);
out:
if (error)
log_debug(ls, «recover_master_copy %d %x», error,
le32_to_cpu(rl->rl_lkid));
rl->rl_result = cpu_to_le32(error);
return error;
}

/* needs at least dlm_rcom + rcom_lock */
int dlm_recover_process_copy(struct dlm_ls *ls, struct dlm_rcom *rc)
{
struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf;
struct dlm_rsb *r;
struct dlm_lkb *lkb;
int error;

error = find_lkb(ls, le32_to_cpu(rl->rl_lkid), &lkb);
if (error) {
log_error(ls, «recover_process_copy no lkid %x»,
le32_to_cpu(rl->rl_lkid));
return error;
}

DLM_ASSERT(is_process_copy(lkb), dlm_print_lkb(lkb););

error = le32_to_cpu(rl->rl_result);

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

switch (error) {
case -EBADR:
/* There’s a chance the new master received our lock before
dlm_recover_master_reply(), this wouldn’t happen if we did
a barrier between recover_masters and recover_locks. */
log_debug(ls, «master copy not ready %x r %lx %s», lkb->lkb_id,
(unsigned long)r, r->res_name);
dlm_send_rcom_lock(r, lkb);
goto out;
case -EEXIST:
log_debug(ls, «master copy exists %x», lkb->lkb_id);
/* fall through */
case 0:
lkb->lkb_remid = le32_to_cpu(rl->rl_remid);
break;
default:
log_error(ls, «dlm_recover_process_copy unknown error %d %x»,
error, lkb->lkb_id);
}

/* an ack for dlm_recover_locks() which waits for replies from
all the locks it sends to new masters */
dlm_recovered_lock(r);
out:
unlock_rsb(r);
put_rsb(r);
dlm_put_lkb(lkb);

return 0;
}

int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
int mode, uint32_t flags, void *name, unsigned int namelen,
unsigned long timeout_cs)
{
struct dlm_lkb *lkb;
struct dlm_args args;
int error;

dlm_lock_recovery(ls);

error = create_lkb(ls, &lkb);
if (error) {
kfree(ua);
goto out;
}

if (flags & DLM_LKF_VALBLK) {
ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_KERNEL);
if (!ua->lksb.sb_lvbptr) {
kfree(ua);
__put_lkb(ls, lkb);
error = -ENOMEM;
goto out;
}
}

/* After ua is attached to lkb it will be freed by dlm_free_lkb().
When DLM_IFL_USER is set, the dlm knows that this is a userspace
lock and that lkb_astparam is the dlm_user_args structure. */

error = set_lock_args(mode, &ua->lksb, flags, namelen, timeout_cs,
fake_astfn, ua, fake_bastfn, &args);
lkb->lkb_flags |= DLM_IFL_USER;
ua->old_mode = DLM_LOCK_IV;

if (error) {
__put_lkb(ls, lkb);
goto out;
}

error = request_lock(ls, lkb, name, namelen, &args);

switch (error) {
case 0:
break;
case -EINPROGRESS:
error = 0;
break;
case -EAGAIN:
error = 0;
/* fall through */
default:
__put_lkb(ls, lkb);
goto out;
}

/* add this new lkb to the per-process list of locks */
spin_lock(&ua->proc->locks_spin);
hold_lkb(lkb);
list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks);
spin_unlock(&ua->proc->locks_spin);
out:
dlm_unlock_recovery(ls);
return error;
}

int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
int mode, uint32_t flags, uint32_t lkid, char *lvb_in,
unsigned long timeout_cs)
{
struct dlm_lkb *lkb;
struct dlm_args args;
struct dlm_user_args *ua;
int error;

dlm_lock_recovery(ls);

error = find_lkb(ls, lkid, &lkb);
if (error)
goto out;

/* user can change the params on its lock when it converts it, or
add an lvb that didn’t exist before */

ua = lkb->lkb_ua;

if (flags & DLM_LKF_VALBLK && !ua->lksb.sb_lvbptr) {
ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_KERNEL);
if (!ua->lksb.sb_lvbptr) {
error = -ENOMEM;
goto out_put;
}
}
if (lvb_in && ua->lksb.sb_lvbptr)
memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN);

ua->xid = ua_tmp->xid;
ua->castparam = ua_tmp->castparam;
ua->castaddr = ua_tmp->castaddr;
ua->bastparam = ua_tmp->bastparam;
ua->bastaddr = ua_tmp->bastaddr;
ua->user_lksb = ua_tmp->user_lksb;
ua->old_mode = lkb->lkb_grmode;

error = set_lock_args(mode, &ua->lksb, flags, 0, timeout_cs,
fake_astfn, ua, fake_bastfn, &args);
if (error)
goto out_put;

error = convert_lock(ls, lkb, &args);

if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK)
error = 0;
out_put:
dlm_put_lkb(lkb);
out:
dlm_unlock_recovery(ls);
kfree(ua_tmp);
return error;
}

int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
uint32_t flags, uint32_t lkid, char *lvb_in)
{
struct dlm_lkb *lkb;
struct dlm_args args;
struct dlm_user_args *ua;
int error;

dlm_lock_recovery(ls);

error = find_lkb(ls, lkid, &lkb);
if (error)
goto out;

ua = lkb->lkb_ua;

if (lvb_in && ua->lksb.sb_lvbptr)
memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN);
if (ua_tmp->castparam)
ua->castparam = ua_tmp->castparam;
ua->user_lksb = ua_tmp->user_lksb;

error = set_unlock_args(flags, ua, &args);
if (error)
goto out_put;

error = unlock_lock(ls, lkb, &args);

if (error == -DLM_EUNLOCK)
error = 0;
/* from validate_unlock_args() */
if (error == -EBUSY && (flags & DLM_LKF_FORCEUNLOCK))
error = 0;
if (error)
goto out_put;

spin_lock(&ua->proc->locks_spin);
/* dlm_user_add_ast() may have already taken lkb off the proc list */
if (!list_empty(&lkb->lkb_ownqueue))
list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking);
spin_unlock(&ua->proc->locks_spin);
out_put:
dlm_put_lkb(lkb);
out:
dlm_unlock_recovery(ls);
kfree(ua_tmp);
return error;
}

int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
uint32_t flags, uint32_t lkid)
{
struct dlm_lkb *lkb;
struct dlm_args args;
struct dlm_user_args *ua;
int error;

dlm_lock_recovery(ls);

error = find_lkb(ls, lkid, &lkb);
if (error)
goto out;

ua = lkb->lkb_ua;
if (ua_tmp->castparam)
ua->castparam = ua_tmp->castparam;
ua->user_lksb = ua_tmp->user_lksb;

error = set_unlock_args(flags, ua, &args);
if (error)
goto out_put;

error = cancel_lock(ls, lkb, &args);

if (error == -DLM_ECANCEL)
error = 0;
/* from validate_unlock_args() */
if (error == -EBUSY)
error = 0;
out_put:
dlm_put_lkb(lkb);
out:
dlm_unlock_recovery(ls);
kfree(ua_tmp);
return error;
}

int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid)
{
struct dlm_lkb *lkb;
struct dlm_args args;
struct dlm_user_args *ua;
struct dlm_rsb *r;
int error;

dlm_lock_recovery(ls);

error = find_lkb(ls, lkid, &lkb);
if (error)
goto out;

ua = lkb->lkb_ua;

error = set_unlock_args(flags, ua, &args);
if (error)
goto out_put;

/* same as cancel_lock(), but set DEADLOCK_CANCEL after lock_rsb */

r = lkb->lkb_resource;
hold_rsb(r);
lock_rsb(r);

error = validate_unlock_args(lkb, &args);
if (error)
goto out_r;
lkb->lkb_flags |= DLM_IFL_DEADLOCK_CANCEL;

error = _cancel_lock(r, lkb);
out_r:
unlock_rsb(r);
put_rsb(r);

if (error == -DLM_ECANCEL)
error = 0;
/* from validate_unlock_args() */
if (error == -EBUSY)
error = 0;
out_put:
dlm_put_lkb(lkb);
out:
dlm_unlock_recovery(ls);
return error;
}

/* lkb’s that are removed from the waiters list by revert are just left on the
orphans list with the granted orphan locks, to be freed by purge */

static int orphan_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
struct dlm_args args;
int error;

hold_lkb(lkb);
mutex_lock(&ls->ls_orphans_mutex);
list_add_tail(&lkb->lkb_ownqueue, &ls->ls_orphans);
mutex_unlock(&ls->ls_orphans_mutex);

set_unlock_args(0, lkb->lkb_ua, &args);

error = cancel_lock(ls, lkb, &args);
if (error == -DLM_ECANCEL)
error = 0;
return error;
}

/* The force flag allows the unlock to go ahead even if the lkb isn’t granted.
Regardless of what rsb queue the lock is on, it’s removed and freed. */

static int unlock_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb)
{
struct dlm_args args;
int error;

set_unlock_args(DLM_LKF_FORCEUNLOCK, lkb->lkb_ua, &args);

error = unlock_lock(ls, lkb, &args);
if (error == -DLM_EUNLOCK)
error = 0;
return error;
}

/* We have to release clear_proc_locks mutex before calling unlock_proc_lock()
(which does lock_rsb) due to deadlock with receiving a message that does
lock_rsb followed by dlm_user_add_ast() */

static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls,
struct dlm_user_proc *proc)
{
struct dlm_lkb *lkb = NULL;

mutex_lock(&ls->ls_clear_proc_locks);
if (list_empty(&proc->locks))
goto out;

lkb = list_entry(proc->locks.next, struct dlm_lkb, lkb_ownqueue);
list_del_init(&lkb->lkb_ownqueue);

if (lkb->lkb_exflags & DLM_LKF_PERSISTENT)
lkb->lkb_flags |= DLM_IFL_ORPHAN;
else
lkb->lkb_flags |= DLM_IFL_DEAD;
out:
mutex_unlock(&ls->ls_clear_proc_locks);
return lkb;
}

/* The ls_clear_proc_locks mutex protects against dlm_user_add_asts() which
1) references lkb->ua which we free here and 2) adds lkbs to proc->asts,
which we clear here. */

/* proc CLOSING flag is set so no more device_reads should look at proc->asts
list, and no more device_writes should add lkb’s to proc->locks list; so we
shouldn’t need to take asts_spin or locks_spin here. this assumes that
device reads/writes/closes are serialized — FIXME: we may need to serialize
them ourself. */

void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc)
{
struct dlm_lkb *lkb, *safe;

dlm_lock_recovery(ls);

while (1) {
lkb = del_proc_lock(ls, proc);
if (!lkb)
break;
del_timeout(lkb);
if (lkb->lkb_exflags & DLM_LKF_PERSISTENT)
orphan_proc_lock(ls, lkb);
else
unlock_proc_lock(ls, lkb);

/* this removes the reference for the proc->locks list
added by dlm_user_request, it may result in the lkb
being freed */

dlm_put_lkb(lkb);
}

mutex_lock(&ls->ls_clear_proc_locks);

/* in-progress unlocks */
list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) {
list_del_init(&lkb->lkb_ownqueue);
lkb->lkb_flags |= DLM_IFL_DEAD;
dlm_put_lkb(lkb);
}

list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_astqueue) {
lkb->lkb_ast_type = 0;
list_del(&lkb->lkb_astqueue);
dlm_put_lkb(lkb);
}

mutex_unlock(&ls->ls_clear_proc_locks);
dlm_unlock_recovery(ls);
}

static void purge_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc)
{
struct dlm_lkb *lkb, *safe;

while (1) {
lkb = NULL;
spin_lock(&proc->locks_spin);
if (!list_empty(&proc->locks)) {
lkb = list_entry(proc->locks.next, struct dlm_lkb,
lkb_ownqueue);
list_del_init(&lkb->lkb_ownqueue);
}
spin_unlock(&proc->locks_spin);

if (!lkb)
break;

lkb->lkb_flags |= DLM_IFL_DEAD;
unlock_proc_lock(ls, lkb);
dlm_put_lkb(lkb); /* ref from proc->locks list */
}

spin_lock(&proc->locks_spin);
list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) {
list_del_init(&lkb->lkb_ownqueue);
lkb->lkb_flags |= DLM_IFL_DEAD;
dlm_put_lkb(lkb);
}
spin_unlock(&proc->locks_spin);

spin_lock(&proc->asts_spin);
list_for_each_entry_safe(lkb, safe, &proc->asts, lkb_astqueue) {
list_del(&lkb->lkb_astqueue);
dlm_put_lkb(lkb);
}
spin_unlock(&proc->asts_spin);
}

/* pid of 0 means purge all orphans */

static void do_purge(struct dlm_ls *ls, int nodeid, int pid)
{
struct dlm_lkb *lkb, *safe;

mutex_lock(&ls->ls_orphans_mutex);
list_for_each_entry_safe(lkb, safe, &ls->ls_orphans, lkb_ownqueue) {
if (pid && lkb->lkb_ownpid != pid)
continue;
unlock_proc_lock(ls, lkb);
list_del_init(&lkb->lkb_ownqueue);
dlm_put_lkb(lkb);
}
mutex_unlock(&ls->ls_orphans_mutex);
}

static int send_purge(struct dlm_ls *ls, int nodeid, int pid)
{
struct dlm_message *ms;
struct dlm_mhandle *mh;
int error;

error = _create_message(ls, sizeof(struct dlm_message), nodeid,
DLM_MSG_PURGE, &ms, &mh);
if (error)
return error;
ms->m_nodeid = nodeid;
ms->m_pid = pid;

return send_message(mh, ms);
}

int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc,
int nodeid, int pid)
{
int error = 0;

if (nodeid != dlm_our_nodeid()) {
error = send_purge(ls, nodeid, pid);
} else {
dlm_lock_recovery(ls);
if (pid == current->pid)
purge_proc_locks(ls, proc);
else
do_purge(ls, nodeid, pid);
dlm_unlock_recovery(ls);
}
return error;
}

config DLM_DEBUG
bool «DLM debugging»
depends on DLM
help
Under the debugfs mount point, the name of each lockspace will
appear as a file in the «dlm» directory. The output is the
list of resource and locks the local node knows about.

#ifndef __DLM_INTERNAL_DOT_H__
#define __DLM_INTERNAL_DOT_H__

/*
* This is the main header file to be included in each DLM source file.
*/

#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include

#include #include «config.h»

/* Size of the temp buffer midcomms allocates on the stack.
We try to make this large enough so most messages fit.
FIXME: should sctp make this unnecessary? */

#define DLM_INBUF_LEN 148

struct dlm_ls;
struct dlm_lkb;
struct dlm_rsb;
struct dlm_member;
struct dlm_lkbtable;
struct dlm_rsbtable;
struct dlm_dirtable;
struct dlm_direntry;
struct dlm_recover;
struct dlm_header;
struct dlm_message;
struct dlm_rcom;
struct dlm_mhandle;

#define log_print(fmt, args…) \
printk(KERN_ERR «dlm: «fmt»\n» , ##args)
#define log_error(ls, fmt, args…) \
printk(KERN_ERR «dlm: %s: » fmt «\n», (ls)->ls_name , ##args)

#define log_debug(ls, fmt, args…) \
do { \
if (dlm_config.ci_log_debug) \
printk(KERN_DEBUG «dlm: %s: » fmt «\n», \
(ls)->ls_name , ##args); \
} while (0)

#define DLM_ASSERT(x, do) \
{ \
if (!(x)) \
{ \
printk(KERN_ERR «\nDLM: Assertion failed on line %d of file %s\n» \
«DLM: assertion: \»%s\»\n» \
«DLM: time = %lu\n», \
__LINE__, __FILE__, #x, jiffies); \
{do} \
printk(»\n»); \
BUG(); \
panic(»DLM: Record message above and reboot.\n»); \
} \
}

struct dlm_direntry {
struct list_head list;
uint32_t master_nodeid;
uint16_t length;
char name[1];
};

struct dlm_dirtable {
struct list_head list;
spinlock_t lock;
};

struct dlm_rsbtable {
struct list_head list;
struct list_head toss;
spinlock_t lock;
};

struct dlm_lkbtable {
struct list_head list;
rwlock_t lock;
uint16_t counter;
};

/*
* Lockspace member (per node in a ls)
*/

struct dlm_member {
struct list_head list;
int nodeid;
int weight;
};

/*
* Save and manage recovery state for a lockspace.
*/

struct dlm_recover {
struct list_head list;
int *nodeids; /* nodeids of all members */
int node_count;
int *new; /* nodeids of new members */
int new_count;
uint64_t seq;
};

/*
* Pass input args to second stage locking function.
*/

struct dlm_args {
uint32_t flags;
void (*astfn) (void *astparam);
void *astparam;
void (*bastfn) (void *astparam, int mode);
int mode;
struct dlm_lksb *lksb;
unsigned long timeout;
};

/*
* Lock block
*
* A lock can be one of three types:
*
* local copy lock is mastered locally
* (lkb_nodeid is zero and DLM_LKF_MSTCPY is not set)
* process copy lock is mastered on a remote node
* (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is not set)
* master copy master node’s copy of a lock owned by remote node
* (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is set)
*
* lkb_exflags: a copy of the most recent flags arg provided to dlm_lock or
* dlm_unlock. The dlm does not modify these or use any private flags in
* this field; it only contains DLM_LKF_ flags from dlm.h. These flags
* are sent as-is to the remote master when the lock is remote.
*
* lkb_flags: internal dlm flags (DLM_IFL_ prefix) from dlm_internal.h.
* Some internal flags are shared between the master and process nodes;
* these shared flags are kept in the lower two bytes. One of these
* flags set on the master copy will be propagated to the process copy
* and v.v. Other internal flags are private to the master or process
* node (e.g. DLM_IFL_MSTCPY). These are kept in the high two bytes.
*
* lkb_sbflags: status block flags. These flags are copied directly into
* the caller’s lksb.sb_flags prior to the dlm_lock/dlm_unlock completion
* ast. All defined in dlm.h with DLM_SBF_ prefix.
*
* lkb_status: the lock status indicates which rsb queue the lock is
* on, grant, convert, or wait. DLM_LKSTS_ WAITING/GRANTED/CONVERT
*
* lkb_wait_type: the dlm message type (DLM_MSG_ prefix) for which a
* reply is needed. Only set when the lkb is on the lockspace waiters
* list awaiting a reply from a remote node.
*
* lkb_nodeid: when the lkb is a local copy, nodeid is 0; when the lkb
* is a master copy, nodeid specifies the remote lock holder, when the
* lkb is a process copy, the nodeid specifies the lock master.
*/

/* lkb_ast_type */

#define AST_COMP 1
#define AST_BAST 2

/* lkb_status */

#define DLM_LKSTS_WAITING 1
#define DLM_LKSTS_GRANTED 2
#define DLM_LKSTS_CONVERT 3

/* lkb_flags */

#define DLM_IFL_MSTCPY 0×00010000
#define DLM_IFL_RESEND 0×00020000
#define DLM_IFL_DEAD 0×00040000
#define DLM_IFL_OVERLAP_UNLOCK 0×00080000
#define DLM_IFL_OVERLAP_CANCEL 0×00100000
#define DLM_IFL_ENDOFLIFE 0×00200000
#define DLM_IFL_WATCH_TIMEWARN 0×00400000
#define DLM_IFL_TIMEOUT_CANCEL 0×00800000
#define DLM_IFL_DEADLOCK_CANCEL 0×01000000
#define DLM_IFL_USER 0×00000001
#define DLM_IFL_ORPHAN 0×00000002

struct dlm_lkb {
struct dlm_rsb *lkb_resource; /* the rsb */
struct kref lkb_ref;
int lkb_nodeid; /* copied from rsb */
int lkb_ownpid; /* pid of lock owner */
uint32_t lkb_id; /* our lock ID */
uint32_t lkb_remid; /* lock ID on remote partner */
uint32_t lkb_exflags; /* external flags from caller */
uint32_t lkb_sbflags; /* lksb flags */
uint32_t lkb_flags; /* internal flags */
uint32_t lkb_lvbseq; /* lvb sequence number */

int8_t lkb_status; /* granted, waiting, convert */
int8_t lkb_rqmode; /* requested lock mode */
int8_t lkb_grmode; /* granted lock mode */
int8_t lkb_bastmode; /* requested mode */
int8_t lkb_highbast; /* highest mode bast sent for */
int8_t lkb_wait_type; /* type of reply waiting for */
int8_t lkb_wait_count;
int8_t lkb_ast_type; /* type of ast queued for */

struct list_head lkb_idtbl_list; /* lockspace lkbtbl */
struct list_head lkb_statequeue; /* rsb g/c/w list */
struct list_head lkb_rsb_lookup; /* waiting for rsb lookup */
struct list_head lkb_wait_reply; /* waiting for remote reply */
struct list_head lkb_astqueue; /* need ast to be sent */
struct list_head lkb_ownqueue; /* list of locks for a process */
struct list_head lkb_time_list;
ktime_t lkb_time_bast; /* for debugging */
ktime_t lkb_timestamp;
unsigned long lkb_timeout_cs;

char *lkb_lvbptr;
struct dlm_lksb *lkb_lksb; /* caller’s status block */
void (*lkb_astfn) (void *astparam);
void (*lkb_bastfn) (void *astparam, int mode);
union {
void *lkb_astparam; /* caller’s ast arg */
struct dlm_user_args *lkb_ua;
};
};

struct dlm_rsb {
struct dlm_ls *res_ls; /* the lockspace */
struct kref res_ref;
struct mutex res_mutex;
unsigned long res_flags;
int res_length; /* length of rsb name */
int res_nodeid;
uint32_t res_lvbseq;
uint32_t res_hash;
uint32_t res_bucket; /* rsbtbl */
unsigned long res_toss_time;
uint32_t res_first_lkid;
struct list_head res_lookup; /* lkbs waiting on first */
struct list_head res_hashchain; /* rsbtbl */
struct list_head res_grantqueue;
struct list_head res_convertqueue;
struct list_head res_waitqueue;

struct list_head res_root_list; /* used for recovery */
struct list_head res_recover_list; /* used for recovery */
int res_recover_locks_count;

char *res_lvbptr;
char res_name[1];
};

/* find_rsb() flags */

#define R_MASTER 1 /* only return rsb if it’s a master */
#define R_CREATE 2 /* create/add rsb if not found */

/* rsb_flags */

enum rsb_flags {
RSB_MASTER_UNCERTAIN,
RSB_VALNOTVALID,
RSB_VALNOTVALID_PREV,
RSB_NEW_MASTER,
RSB_NEW_MASTER2,
RSB_RECOVER_CONVERT,
RSB_LOCKS_PURGED,
};

static inline void rsb_set_flag(struct dlm_rsb *r, enum rsb_flags flag)
{
__set_bit(flag, &r->res_flags);
}

static inline void rsb_clear_flag(struct dlm_rsb *r, enum rsb_flags flag)
{
__clear_bit(flag, &r->res_flags);
}

static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag)
{
return test_bit(flag, &r->res_flags);
}

/* dlm_header is first element of all structs sent between nodes */

#define DLM_HEADER_MAJOR 0×00030000
#define DLM_HEADER_MINOR 0×00000000

#define DLM_MSG 1
#define DLM_RCOM 2

struct dlm_header {
uint32_t h_version;
uint32_t h_lockspace;
uint32_t h_nodeid; /* nodeid of sender */
uint16_t h_length;
uint8_t h_cmd; /* DLM_MSG, DLM_RCOM */
uint8_t h_pad;
};

#define DLM_MSG_REQUEST 1
#define DLM_MSG_CONVERT 2
#define DLM_MSG_UNLOCK 3
#define DLM_MSG_CANCEL 4
#define DLM_MSG_REQUEST_REPLY 5
#define DLM_MSG_CONVERT_REPLY 6
#define DLM_MSG_UNLOCK_REPLY 7
#define DLM_MSG_CANCEL_REPLY 8
#define DLM_MSG_GRANT 9
#define DLM_MSG_BAST 10
#define DLM_MSG_LOOKUP 11
#define DLM_MSG_REMOVE 12
#define DLM_MSG_LOOKUP_REPLY 13
#define DLM_MSG_PURGE 14

struct dlm_message {
struct dlm_header m_header;
uint32_t m_type; /* DLM_MSG_ */
uint32_t m_nodeid;
uint32_t m_pid;
uint32_t m_lkid; /* lkid on sender */
uint32_t m_remid; /* lkid on receiver */
uint32_t m_parent_lkid;
uint32_t m_parent_remid;
uint32_t m_exflags;
uint32_t m_sbflags;
uint32_t m_flags;
uint32_t m_lvbseq;
uint32_t m_hash;
int m_status;
int m_grmode;
int m_rqmode;
int m_bastmode;
int m_asts;
int m_result; /* 0 or -EXXX */
char m_extra[0]; /* name or lvb */
};

#define DLM_RS_NODES 0×00000001
#define DLM_RS_NODES_ALL 0×00000002
#define DLM_RS_DIR 0×00000004
#define DLM_RS_DIR_ALL 0×00000008
#define DLM_RS_LOCKS 0×00000010
#define DLM_RS_LOCKS_ALL 0×00000020
#define DLM_RS_DONE 0×00000040
#define DLM_RS_DONE_ALL 0×00000080

#define DLM_RCOM_STATUS 1
#define DLM_RCOM_NAMES 2
#define DLM_RCOM_LOOKUP 3
#define DLM_RCOM_LOCK 4
#define DLM_RCOM_STATUS_REPLY 5
#define DLM_RCOM_NAMES_REPLY 6
#define DLM_RCOM_LOOKUP_REPLY 7
#define DLM_RCOM_LOCK_REPLY 8

struct dlm_rcom {
struct dlm_header rc_header;
uint32_t rc_type; /* DLM_RCOM_ */
int rc_result; /* multi-purpose */
uint64_t rc_id; /* match reply with request */
uint64_t rc_seq; /* sender’s ls_recover_seq */
uint64_t rc_seq_reply; /* remote ls_recover_seq */
char rc_buf[0];
};

union dlm_packet {
struct dlm_header header; /* common to other two */
struct dlm_message message;
struct dlm_rcom rcom;
};

struct rcom_config {
__le32 rf_lvblen;
__le32 rf_lsflags;
__le64 rf_unused;
};

struct rcom_lock {
__le32 rl_ownpid;
__le32 rl_lkid;
__le32 rl_remid;
__le32 rl_parent_lkid;
__le32 rl_parent_remid;
__le32 rl_exflags;
__le32 rl_flags;
__le32 rl_lvbseq;
__le32 rl_result;
int8_t rl_rqmode;
int8_t rl_grmode;
int8_t rl_status;
int8_t rl_asts;
__le16 rl_wait_type;
__le16 rl_namelen;
char rl_name[DLM_RESNAME_MAXLEN];
char rl_lvb[0];
};

struct dlm_ls {
struct list_head ls_list; /* list of lockspaces */
dlm_lockspace_t *ls_local_handle;
uint32_t ls_global_id; /* global unique lockspace ID */
uint32_t ls_exflags;
int ls_lvblen;
int ls_count; /* refcount of processes in
the dlm using this ls */
int ls_create_count; /* create/release refcount */
unsigned long ls_flags; /* LSFL_ */
unsigned long ls_scan_time;
struct kobject ls_kobj;

struct dlm_rsbtable *ls_rsbtbl;
uint32_t ls_rsbtbl_size;

struct dlm_lkbtable *ls_lkbtbl;
uint32_t ls_lkbtbl_size;

struct dlm_dirtable *ls_dirtbl;
uint32_t ls_dirtbl_size;

struct mutex ls_waiters_mutex;
struct list_head ls_waiters; /* lkbs needing a reply */

struct mutex ls_orphans_mutex;
struct list_head ls_orphans;

struct mutex ls_timeout_mutex;
struct list_head ls_timeout;

struct list_head ls_nodes; /* current nodes in ls */
struct list_head ls_nodes_gone; /* dead node list, recovery */
int ls_num_nodes; /* number of nodes in ls */
int ls_low_nodeid;
int ls_total_weight;
int *ls_node_array;
gfp_t ls_allocation;

struct dlm_rsb ls_stub_rsb; /* for returning errors */
struct dlm_lkb ls_stub_lkb; /* for returning errors */
struct dlm_message ls_stub_ms; /* for faking a reply */

struct dentry *ls_debug_rsb_dentry; /* debugfs */
struct dentry *ls_debug_waiters_dentry; /* debugfs */
struct dentry *ls_debug_locks_dentry; /* debugfs */
struct dentry *ls_debug_all_dentry; /* debugfs */

wait_queue_head_t ls_uevent_wait; /* user part of join/leave */
int ls_uevent_result;
struct completion ls_members_done;
int ls_members_result;

struct miscdevice ls_device;

/* recovery related */

struct timer_list ls_timer;
struct task_struct *ls_recoverd_task;
struct mutex ls_recoverd_active;
spinlock_t ls_recover_lock;
unsigned long ls_recover_begin; /* jiffies timestamp */
uint32_t ls_recover_status; /* DLM_RS_ */
uint64_t ls_recover_seq;
struct dlm_recover *ls_recover_args;
struct rw_semaphore ls_in_recovery; /* block local requests */
struct rw_semaphore ls_recv_active; /* block dlm_recv */
struct list_head ls_requestqueue;/* queue remote requests */
struct mutex ls_requestqueue_mutex;
struct dlm_rcom *ls_recover_buf;
int ls_recover_nodeid; /* for debugging */
uint64_t ls_rcom_seq;
spinlock_t ls_rcom_spin;
struct list_head ls_recover_list;
spinlock_t ls_recover_list_lock;
int ls_recover_list_count;
wait_queue_head_t ls_wait_general;
struct mutex ls_clear_proc_locks;

struct list_head ls_root_list; /* root resources */
struct rw_semaphore ls_root_sem; /* protect root_list */

int ls_namelen;
char ls_name[1];
};

#define LSFL_WORK 0
#define LSFL_RUNNING 1
#define LSFL_RECOVERY_STOP 2
#define LSFL_RCOM_READY 3
#define LSFL_RCOM_WAIT 4
#define LSFL_UEVENT_WAIT 5
#define LSFL_TIMEWARN 6

/* much of this is just saving user space pointers associated with the
lock that we pass back to the user lib with an ast */

struct dlm_user_args {
struct dlm_user_proc *proc; /* each process that opens the lockspace
device has private data
(dlm_user_proc) on the struct file,
the process’s locks point back to it*/
struct dlm_lksb lksb;
int old_mode;
int update_user_lvb;
struct dlm_lksb __user *user_lksb;
void __user *castparam;
void __user *castaddr;
void __user *bastparam;
void __user *bastaddr;
uint64_t xid;
};

#define DLM_PROC_FLAGS_CLOSING 1
#define DLM_PROC_FLAGS_COMPAT 2

/* locks list is kept so we can remove all a process’s locks when it
exits (or orphan those that are persistent) */

struct dlm_user_proc {
dlm_lockspace_t *lockspace;
unsigned long flags; /* DLM_PROC_FLAGS */
struct list_head asts;
spinlock_t asts_spin;
struct list_head locks;
spinlock_t locks_spin;
struct list_head unlocking;
wait_queue_head_t wait;
};

static inline int dlm_locking_stopped(struct dlm_ls *ls)
{
return !test_bit(LSFL_RUNNING, &ls->ls_flags);
}

static inline int dlm_recovery_stopped(struct dlm_ls *ls)
{
return test_bit(LSFL_RECOVERY_STOP, &ls->ls_flags);
}

static inline int dlm_no_directory(struct dlm_ls *ls)
{
return (ls->ls_exflags & DLM_LSFL_NODIR) ? 1 : 0;
}

int dlm_netlink_init(void);
void dlm_netlink_exit(void);
void dlm_timeout_warn(struct dlm_lkb *lkb);
int dlm_plock_init(void);
void dlm_plock_exit(void);

#ifdef CONFIG_DLM_DEBUG
int dlm_register_debugfs(void);
void dlm_unregister_debugfs(void);
int dlm_create_debug_file(struct dlm_ls *ls);
void dlm_delete_debug_file(struct dlm_ls *ls);
#else
static inline int dlm_register_debugfs(void) { return 0; }
static inline void dlm_unregister_debugfs(void) { }
static inline int dlm_create_debug_file(struct dlm_ls *ls) { return 0; }
static inline void dlm_delete_debug_file(struct dlm_ls *ls) { }
#endif

#endif /* __DLM_INTERNAL_DOT_H__ */

#ifndef __DIR_DOT_H__
#define __DIR_DOT_H__

int dlm_dir_nodeid(struct dlm_rsb *rsb);
int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash);
void dlm_dir_remove_entry(struct dlm_ls *ls, int nodeid, char *name, int len);
void dlm_dir_clear(struct dlm_ls *ls);
void dlm_clear_free_entries(struct dlm_ls *ls);
int dlm_recover_directory(struct dlm_ls *ls);
int dlm_dir_lookup(struct dlm_ls *ls, int nodeid, char *name, int namelen,
int *r_nodeid);
void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen,
char *outbuf, int outlen, int nodeid);

#endif /* __DIR_DOT_H__ */

#include «dlm_internal.h»
#include «lockspace.h»
#include «member.h»
#include «lowcomms.h»
#include «rcom.h»
#include «config.h»
#include «memory.h»
#include «recover.h»
#include «util.h»
#include «lock.h»
#include «dir.h»

static void put_free_de(struct dlm_ls *ls, struct dlm_direntry *de)
{
spin_lock(&ls->ls_recover_list_lock);
list_add(&de->list, &ls->ls_recover_list);
spin_unlock(&ls->ls_recover_list_lock);
}

static struct dlm_direntry *get_free_de(struct dlm_ls *ls, int len)
{
int found = 0;
struct dlm_direntry *de;

spin_lock(&ls->ls_recover_list_lock);
list_for_each_entry(de, &ls->ls_recover_list, list) {
if (de->length == len) {
list_del(&de->list);
de->master_nodeid = 0;
memset(de->name, 0, len);
found = 1;
break;
}
}
spin_unlock(&ls->ls_recover_list_lock);

if (!found)
de = kzalloc(sizeof(struct dlm_direntry) + len,
ls->ls_allocation);
return de;
}

void dlm_clear_free_entries(struct dlm_ls *ls)
{
struct dlm_direntry *de;

spin_lock(&ls->ls_recover_list_lock);
while (!list_empty(&ls->ls_recover_list)) {
de = list_entry(ls->ls_recover_list.next, struct dlm_direntry,
list);
list_del(&de->list);
kfree(de);
}
spin_unlock(&ls->ls_recover_list_lock);
}

/*
* We use the upper 16 bits of the hash value to select the directory node.
* Low bits are used for distribution of rsb’s among hash buckets on each node.
*
* To give the exact range wanted (0 to num_nodes-1), we apply a modulus of
* num_nodes to the hash value. This value in the desired range is used as an
* offset into the sorted list of nodeid’s to give the particular nodeid.
*/

int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash)
{
struct list_head *tmp;
struct dlm_member *memb = NULL;
uint32_t node, n = 0;
int nodeid;

if (ls->ls_num_nodes == 1) {
nodeid = dlm_our_nodeid();
goto out;
}

if (ls->ls_node_array) {
node = (hash >> 16) % ls->ls_total_weight;
nodeid = ls->ls_node_array[node];
goto out;
}

/* make_member_array() failed to kmalloc ls_node_array… */

node = (hash >> 16) % ls->ls_num_nodes;

list_for_each(tmp, &ls->ls_nodes) {
if (n++ != node)
continue;
memb = list_entry(tmp, struct dlm_member, list);
break;
}

DLM_ASSERT(memb , printk(»num_nodes=%u n=%u node=%u\n»,
ls->ls_num_nodes, n, node););
nodeid = memb->nodeid;
out:
return nodeid;
}

int dlm_dir_nodeid(struct dlm_rsb *r)
{
return dlm_hash2nodeid(r->res_ls, r->res_hash);
}

static inline uint32_t dir_hash(struct dlm_ls *ls, char *name, int len)
{
uint32_t val;

val = jhash(name, len, 0);
val &= (ls->ls_dirtbl_size – 1);

return val;
}

static void add_entry_to_hash(struct dlm_ls *ls, struct dlm_direntry *de)
{
uint32_t bucket;

bucket = dir_hash(ls, de->name, de->length);
list_add_tail(&de->list, &ls->ls_dirtbl[bucket].list);
}

static struct dlm_direntry *search_bucket(struct dlm_ls *ls, char *name,
int namelen, uint32_t bucket)
{
struct dlm_direntry *de;

list_for_each_entry(de, &ls->ls_dirtbl[bucket].list, list) {
if (de->length == namelen && !memcmp(name, de->name, namelen))
goto out;
}
de = NULL;
out:
return de;
}

void dlm_dir_remove_entry(struct dlm_ls *ls, int nodeid, char *name, int namelen)
{
struct dlm_direntry *de;
uint32_t bucket;

bucket = dir_hash(ls, name, namelen);

spin_lock(&ls->ls_dirtbl[bucket].lock);

de = search_bucket(ls, name, namelen, bucket);

if (!de) {
log_error(ls, «remove fr %u none», nodeid);
goto out;
}

if (de->master_nodeid != nodeid) {
log_error(ls, «remove fr %u ID %u», nodeid, de->master_nodeid);
goto out;
}

list_del(&de->list);
kfree(de);
out:
spin_unlock(&ls->ls_dirtbl[bucket].lock);
}

void dlm_dir_clear(struct dlm_ls *ls)
{
struct list_head *head;
struct dlm_direntry *de;
int i;

DLM_ASSERT(list_empty(&ls->ls_recover_list), );

for (i = 0; i < ls->ls_dirtbl_size; i++) {
spin_lock(&ls->ls_dirtbl[i].lock);
head = &ls->ls_dirtbl[i].list;
while (!list_empty(head)) {
de = list_entry(head->next, struct dlm_direntry, list);
list_del(&de->list);
put_free_de(ls, de);
}
spin_unlock(&ls->ls_dirtbl[i].lock);
}
}

int dlm_recover_directory(struct dlm_ls *ls)
{
struct dlm_member *memb;
struct dlm_direntry *de;
char *b, *last_name = NULL;
int error = -ENOMEM, last_len, count = 0;
uint16_t namelen;

log_debug(ls, «dlm_recover_directory»);

if (dlm_no_directory(ls))
goto out_status;

dlm_dir_clear(ls);

last_name = kmalloc(DLM_RESNAME_MAXLEN, ls->ls_allocation);
if (!last_name)
goto out;

list_for_each_entry(memb, &ls->ls_nodes, list) {
memset(last_name, 0, DLM_RESNAME_MAXLEN);
last_len = 0;

for (;;) {
int left;
error = dlm_recovery_stopped(ls);
if (error)
goto out_free;

error = dlm_rcom_names(ls, memb->nodeid,
last_name, last_len);
if (error)
goto out_free;

schedule();

/*
* pick namelen/name pairs out of received buffer
*/

b = ls->ls_recover_buf->rc_buf;
left = ls->ls_recover_buf->rc_header.h_length;
left -= sizeof(struct dlm_rcom);

for (;;) {
__be16 v;

error = -EINVAL;
if (left < sizeof(__be16))
goto out_free;

memcpy(&v, b, sizeof(__be16));
namelen = be16_to_cpu(v);
b += sizeof(__be16);
left -= sizeof(__be16);

/* namelen of 0xFFFFF marks end of names for
this node; namelen of 0 marks end of the
buffer */

if (namelen == 0xFFFF)
goto done;
if (!namelen)
break;

if (namelen > left)
goto out_free;

if (namelen > DLM_RESNAME_MAXLEN)
goto out_free;

error = -ENOMEM;
de = get_free_de(ls, namelen);
if (!de)
goto out_free;

de->master_nodeid = memb->nodeid;
de->length = namelen;
last_len = namelen;
memcpy(de->name, b, namelen);
memcpy(last_name, b, namelen);
b += namelen;
left -= namelen;

add_entry_to_hash(ls, de);
count++;
}
}
done:
;
}

out_status:
error = 0;
dlm_set_recover_status(ls, DLM_RS_DIR);
log_debug(ls, «dlm_recover_directory %d entries», count);
out_free:
kfree(last_name);
out:
dlm_clear_free_entries(ls);
return error;
}

static int get_entry(struct dlm_ls *ls, int nodeid, char *name,
int namelen, int *r_nodeid)
{
struct dlm_direntry *de, *tmp;
uint32_t bucket;

bucket = dir_hash(ls, name, namelen);

spin_lock(&ls->ls_dirtbl[bucket].lock);
de = search_bucket(ls, name, namelen, bucket);
if (de) {
*r_nodeid = de->master_nodeid;
spin_unlock(&ls->ls_dirtbl[bucket].lock);
if (*r_nodeid == nodeid)
return -EEXIST;
return 0;
}

spin_unlock(&ls->ls_dirtbl[bucket].lock);

if (namelen > DLM_RESNAME_MAXLEN)
return -EINVAL;

de = kzalloc(sizeof(struct dlm_direntry) + namelen, ls->ls_allocation);
if (!de)
return -ENOMEM;

de->master_nodeid = nodeid;
de->length = namelen;
memcpy(de->name, name, namelen);

spin_lock(&ls->ls_dirtbl[bucket].lock);
tmp = search_bucket(ls, name, namelen, bucket);
if (tmp) {
kfree(de);
de = tmp;
} else {
list_add_tail(&de->list, &ls->ls_dirtbl[bucket].list);
}
*r_nodeid = de->master_nodeid;
spin_unlock(&ls->ls_dirtbl[bucket].lock);
return 0;
}

int dlm_dir_lookup(struct dlm_ls *ls, int nodeid, char *name, int namelen,
int *r_nodeid)
{
return get_entry(ls, nodeid, name, namelen, r_nodeid);
}

static struct dlm_rsb *find_rsb_root(struct dlm_ls *ls, char *name, int len)
{
struct dlm_rsb *r;

down_read(&ls->ls_root_sem);
list_for_each_entry(r, &ls->ls_root_list, res_root_list) {
if (len == r->res_length && !memcmp(name, r->res_name, len)) {
up_read(&ls->ls_root_sem);
return r;
}
}
up_read(&ls->ls_root_sem);
return NULL;
}

/* Find the rsb where we left off (or start again), then send rsb names
for rsb’s we’re master of and whose directory node matches the requesting
node. inbuf is the rsb name last sent, inlen is the name’s length */

void dlm_copy_master_names(struct dlm_ls *ls, char *inbuf, int inlen,
char *outbuf, int outlen, int nodeid)
{
struct list_head *list;
struct dlm_rsb *r;
int offset = 0, dir_nodeid;
__be16 be_namelen;

down_read(&ls->ls_root_sem);

if (inlen > 1) {
r = find_rsb_root(ls, inbuf, inlen);
if (!r) {
inbuf[inlen - 1] = ‘\0′;
log_error(ls, «copy_master_names from %d start %d %s»,
nodeid, inlen, inbuf);
goto out;
}
list = r->res_root_list.next;
} else {
list = ls->ls_root_list.next;
}

for (offset = 0; list != &ls->ls_root_list; list = list->next) {
r = list_entry(list, struct dlm_rsb, res_root_list);
if (r->res_nodeid)
continue;

dir_nodeid = dlm_dir_nodeid(r);
if (dir_nodeid != nodeid)
continue;

/*
* The block ends when we can’t fit the following in the
* remaining buffer space:
* namelen (uint16_t) +
* name (r->res_length) +
* end-of-block record 0×0000 (uint16_t)
*/

if (offset + sizeof(uint16_t)*2 + r->res_length > outlen) {
/* Write end-of-block record */
be_namelen = cpu_to_be16(0);
memcpy(outbuf + offset, &be_namelen, sizeof(__be16));
offset += sizeof(__be16);
goto out;
}

be_namelen = cpu_to_be16(r->res_length);
memcpy(outbuf + offset, &be_namelen, sizeof(__be16));
offset += sizeof(__be16);
memcpy(outbuf + offset, r->res_name, r->res_length);
offset += r->res_length;
}

/*
* If we’ve reached the end of the list (and there’s room) write a
* terminating record.
*/

if ((list == &ls->ls_root_list) &&
(offset + sizeof(uint16_t) <= outlen)) {
be_namelen = cpu_to_be16(0xFFFF);
memcpy(outbuf + offset, &be_namelen, sizeof(__be16));
offset += sizeof(__be16);
}